From 46f8141af356747962d7a7a886ac36fe6eaccba9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Mon, 19 Oct 2026 23:41:48 -0500 Subject: [PATCH 01/40] feat(game-macros): deny economy names on FactCommand fields Extend the FactCommand deny list with amount, coins, gold, currency, price, cost, balance, grant, payout, credit and debit. The list moves to a shared `trusted` module so ticket extension claims can reuse it, and every denied field (Rust name, serde rename or alias) now gets its own spanned error instead of only the first. [[tasks/gr-p26-fact-economy-denylist]] --- distributed_game/README.md | 8 +- distributed_game/src/facts.rs | 3 +- distributed_game_macros/src/fact.rs | 75 ++--------- distributed_game_macros/src/lib.rs | 7 +- distributed_game_macros/src/trusted.rs | 120 ++++++++++++++++++ .../tests/compile_fail/fact_economy_fields.rs | 26 ++++ .../compile_fail/fact_economy_fields.stderr | 77 +++++++++++ .../compile_fail/fact_identity_field.stderr | 2 +- .../compile_fail/fact_identity_variants.rs | 15 +++ .../fact_identity_variants.stderr | 23 ++++ .../tests/compile_fail/fact_reward_field.rs | 2 +- .../compile_fail/fact_reward_field.stderr | 2 +- 12 files changed, 284 insertions(+), 76 deletions(-) create mode 100644 distributed_game_macros/src/trusted.rs create mode 100644 distributed_game_macros/tests/compile_fail/fact_economy_fields.rs create mode 100644 distributed_game_macros/tests/compile_fail/fact_economy_fields.stderr create mode 100644 distributed_game_macros/tests/compile_fail/fact_identity_variants.rs create mode 100644 distributed_game_macros/tests/compile_fail/fact_identity_variants.stderr diff --git a/distributed_game/README.md b/distributed_game/README.md index 8f20ca5db..824162e39 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -663,9 +663,11 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), - The host attaches the killer's **verified identity** from their ticket. The zone cannot claim to be someone else, and the payload names no reward amount; the command handler decides the XP from what it trusts. - The derive refuses identity- or reward-looking fields (`principal`, - `user_id`, `owner`, `xp`, `reward`, … including serde renames) at - compile time. + The derive refuses identity-, reward- and economy-looking fields + (`principal`, `user_id`, `owner`, `xp`, `reward`, `amount`, `coins`, + `gold`, `currency`, `price`, `cost`, `balance`, `grant`, `payout`, + `credit`, `debit`; `_`/`-`/case ignored, serde renames and aliases + included) at compile time, one error per field. - Keys are restart-safe automatically: `ctx.emit(player, input)` uses the input's `#[fact(key(field, ..))]` parts, or the tick and emission order; `ctx.emit_keyed` takes explicit parts; `ctx.emit_zone` emits a zone diff --git a/distributed_game/src/facts.rs b/distributed_game/src/facts.rs index 9855304dc..844759133 100644 --- a/distributed_game/src/facts.rs +++ b/distributed_game/src/facts.rs @@ -80,7 +80,8 @@ pub fn fact_key(prefix: &str, zone: &str, epoch: u64, parts: &[&dyn std::fmt::Di /// Trust rules (enforced by the host, not by this trait): the principal is /// bound from the player's verified ticket, never from the input; the input /// reports *what happened* (which mob died) and the command handler decides -/// any reward. The derive rejects identity- and reward-looking field names. +/// any reward or economy amount. The derive rejects identity-, reward- and +/// economy-looking field names (`user_id`, `xp`, `amount`, `gold`, …). pub trait FactCommand: serde::Serialize { /// Durable command name. const COMMAND: &'static str; diff --git a/distributed_game_macros/src/fact.rs b/distributed_game_macros/src/fact.rs index 09fa5516d..3757108d3 100644 --- a/distributed_game_macros/src/fact.rs +++ b/distributed_game_macros/src/fact.rs @@ -4,43 +4,10 @@ use proc_macro2::TokenStream; use quote::quote; use syn::spanned::Spanned; -use syn::{DeriveInput, Expr, Ident, LitStr}; +use syn::{DeriveInput, Expr, Ident}; -use crate::shared::{field_name, find_attr, named_fields, no_generics, runtime}; - -/// Field names a zone-emitted fact must never carry: identity is bound by -/// the host from the verified ticket, rewards are decided by the handler. -const FORBIDDEN: &[(&str, &str)] = &[ - ("principal", "identity"), - ("user", "identity"), - ("user_id", "identity"), - ("userid", "identity"), - ("owner", "identity"), - ("owner_id", "identity"), - ("account", "identity"), - ("account_id", "identity"), - ("subject", "identity"), - ("sub", "identity"), - ("xp", "a reward"), - ("experience", "a reward"), - ("reward", "a reward"), - ("rewards", "a reward"), -]; - -fn normalized(name: &str) -> String { - name.chars() - .filter(|c| *c != '_' && *c != '-') - .flat_map(char::to_lowercase) - .collect() -} - -fn forbidden(name: &str) -> Option<&'static str> { - let n = normalized(name); - FORBIDDEN - .iter() - .find(|(bad, _)| normalized(bad) == n) - .map(|(_, why)| *why) -} +use crate::shared::{find_attr, named_fields, no_generics, runtime}; +use crate::trusted::reject_forbidden; pub(crate) fn expand(input: DeriveInput) -> syn::Result { no_generics(&input, "FactCommand")?; @@ -78,37 +45,11 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { }; let fields = named_fields(&input, "FactCommand")?; - for field in &fields { - let ident = field.ident.as_ref().expect("named"); - let mut names = vec![(field_name(ident), ident.span())]; - // Serde renames change what the handler sees. - for a in field.attrs.iter().filter(|a| a.path().is_ident("serde")) { - let _ = a.parse_nested_meta(|meta| { - if meta.path.is_ident("rename") || meta.path.is_ident("alias") { - if let Ok(value) = meta.value() { - if let Ok(lit) = value.parse::() { - names.push((lit.value(), lit.span())); - } - } - } else if meta.input.peek(syn::Token![=]) { - let _ = meta.value()?.parse::(); - } else if meta.input.peek(syn::token::Paren) { - let _ = meta.parse_nested_meta(|_| Ok(())); - } - Ok(()) - }); - } - for (name, span) in names { - if let Some(why) = forbidden(&name) { - return Err(syn::Error::new( - span, - format!( - "fact input field `{name}` looks like {why}: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards" - ), - )); - } - } - } + reject_forbidden( + &fields, + "fact input", + "a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values", + )?; let parts = match &key { None => quote!(::std::vec::Vec::new()), diff --git a/distributed_game_macros/src/lib.rs b/distributed_game_macros/src/lib.rs index b3bd97a75..9b3487054 100644 --- a/distributed_game_macros/src/lib.rs +++ b/distributed_game_macros/src/lib.rs @@ -31,6 +31,7 @@ mod predict; mod pure; mod replicated; mod shared; +mod trusted; use proc_macro::TokenStream; use syn::{parse_macro_input, DeriveInput, ItemFn}; @@ -81,8 +82,10 @@ pub fn derive_game_record(input: TokenStream) -> TokenStream { /// Bind a cell command input to its command: /// `#[fact(command = RECORD_KILL, key(mob, generation))]`. Requires -/// `serde::Serialize`. Identity- and reward-looking field names are -/// rejected. +/// `serde::Serialize`. Identity-, reward- and economy-looking field names +/// (`principal`, `user_id`, `xp`, `reward`, `amount`, `gold`, `price`, +/// `balance`, `payout`, …; `_`/`-`/case ignored, serde renames included) +/// are rejected. #[proc_macro_derive(FactCommand, attributes(fact))] pub fn derive_fact_command(input: TokenStream) -> TokenStream { finish(fact::expand(parse_macro_input!(input as DeriveInput))) diff --git a/distributed_game_macros/src/trusted.rs b/distributed_game_macros/src/trusted.rs new file mode 100644 index 000000000..e4930c23c --- /dev/null +++ b/distributed_game_macros/src/trusted.rs @@ -0,0 +1,120 @@ +//! The deny list for fields a game must not let untrusted data carry. +//! +//! Zone facts (`FactCommand`) and ticket extension claims share it: identity +//! is bound by the host from the verified ticket, and rewards and economy +//! amounts are decided by the deterministic command handler, never by a +//! field the zone or the browser filled in. + +use proc_macro2::Span; +use syn::{Expr, Field, LitStr}; + +use crate::shared::field_name; + +/// Denied names and what they look like. Matching ignores `_`, `-` and case, +/// so `user_id`, `userId`, `USER-ID` and `UserID` all hit `user_id`. +const FORBIDDEN: &[(&str, &str)] = &[ + ("principal", "identity"), + ("user", "identity"), + ("user_id", "identity"), + ("owner", "identity"), + ("owner_id", "identity"), + ("account", "identity"), + ("account_id", "identity"), + ("subject", "identity"), + ("sub", "identity"), + ("xp", "a reward"), + ("experience", "a reward"), + ("reward", "a reward"), + ("rewards", "a reward"), + ("amount", "an economy value"), + ("coins", "an economy value"), + ("gold", "an economy value"), + ("currency", "an economy value"), + ("price", "an economy value"), + ("cost", "an economy value"), + ("balance", "an economy value"), + ("grant", "an economy value"), + ("payout", "an economy value"), + ("credit", "an economy value"), + ("debit", "an economy value"), +]; + +fn normalized(name: &str) -> String { + name.chars() + .filter(|c| *c != '_' && *c != '-') + .flat_map(char::to_lowercase) + .collect() +} + +/// What `name` looks like, if it is denied. +pub(crate) fn forbidden(name: &str) -> Option<&'static str> { + let n = normalized(name); + FORBIDDEN + .iter() + .find(|(bad, _)| normalized(bad) == n) + .map(|(_, why)| *why) +} + +/// Every name a field is seen under by a serde consumer: the Rust name plus +/// any `#[serde(rename = ..)]` / `#[serde(alias = ..)]`. +fn seen_names(field: &Field) -> Vec<(String, Span)> { + let ident = field.ident.as_ref().expect("named"); + let mut names = vec![(field_name(ident), ident.span())]; + for a in field.attrs.iter().filter(|a| a.path().is_ident("serde")) { + let _ = a.parse_nested_meta(|meta| { + if meta.path.is_ident("rename") || meta.path.is_ident("alias") { + if let Ok(value) = meta.value() { + if let Ok(lit) = value.parse::() { + names.push((lit.value(), lit.span())); + } + } + } else if meta.input.peek(syn::Token![=]) { + let _ = meta.value()?.parse::(); + } else if meta.input.peek(syn::token::Paren) { + let _ = meta.parse_nested_meta(|_| Ok(())); + } + Ok(()) + }); + } + names +} + +/// Reject every denied field name (one spanned error each, combined). +/// `what` names the declaration ("fact input", "ticket claim") and `rule` +/// ends the message with why the value must come from elsewhere. +pub(crate) fn reject_forbidden(fields: &[&Field], what: &str, rule: &str) -> syn::Result<()> { + let mut errors: Option = None; + for field in fields { + for (name, span) in seen_names(field) { + if let Some(why) = forbidden(&name) { + let error = syn::Error::new( + span, + format!("{what} field `{name}` looks like {why}: {rule}"), + ); + match &mut errors { + Some(all) => all.combine(error), + None => errors = Some(error), + } + } + } + } + errors.map_or(Ok(()), Err) +} + +#[cfg(test)] +mod tests { + use super::forbidden; + + #[test] + fn matches_ignore_separators_and_case() { + for name in ["userId", "USER-ID", "user_id", "UserID", "Owner_Id"] { + assert_eq!(forbidden(name), Some("identity"), "{name}"); + } + for name in ["Gold", "GOLD", "pay_out", "Pay-Out", "c_o_i_n_s", "Balance"] { + assert_eq!(forbidden(name), Some("an economy value"), "{name}"); + } + for name in ["mob", "generation", "amounts", "coin", "costs", "kills"] { + assert_eq!(forbidden(name), None, "{name}"); + } + } +} diff --git a/distributed_game_macros/tests/compile_fail/fact_economy_fields.rs b/distributed_game_macros/tests/compile_fail/fact_economy_fields.rs new file mode 100644 index 000000000..dcec8fe77 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_economy_fields.rs @@ -0,0 +1,26 @@ +use distributed_game::FactCommand; + +// Every economy name is denied, whatever its case or separators, and under +// a serde rename or alias too. Each one gets its own error. +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "crawler.loot_found")] +struct LootFound { + chest: String, + amount: u32, + Coins: u32, + GOLD: u32, + currency_: String, + price: u32, + cost: u32, + bal_ance: u64, + grant: u32, + pay_out: u32, + CREDIT: u32, + debit: u32, + #[serde(rename = "Pay-Out")] + a: u32, + #[serde(alias = "goldAmount", alias = "gold-")] + b: u32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/fact_economy_fields.stderr b/distributed_game_macros/tests/compile_fail/fact_economy_fields.stderr new file mode 100644 index 000000000..5c1e92975 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_economy_fields.stderr @@ -0,0 +1,77 @@ +error: fact input field `amount` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:9:5 + | +9 | amount: u32, + | ^^^^^^ + +error: fact input field `Coins` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:10:5 + | +10 | Coins: u32, + | ^^^^^ + +error: fact input field `GOLD` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:11:5 + | +11 | GOLD: u32, + | ^^^^ + +error: fact input field `currency_` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:12:5 + | +12 | currency_: String, + | ^^^^^^^^^ + +error: fact input field `price` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:13:5 + | +13 | price: u32, + | ^^^^^ + +error: fact input field `cost` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:14:5 + | +14 | cost: u32, + | ^^^^ + +error: fact input field `bal_ance` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:15:5 + | +15 | bal_ance: u64, + | ^^^^^^^^ + +error: fact input field `grant` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:16:5 + | +16 | grant: u32, + | ^^^^^ + +error: fact input field `pay_out` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:17:5 + | +17 | pay_out: u32, + | ^^^^^^^ + +error: fact input field `CREDIT` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:18:5 + | +18 | CREDIT: u32, + | ^^^^^^ + +error: fact input field `debit` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:19:5 + | +19 | debit: u32, + | ^^^^^ + +error: fact input field `Pay-Out` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:20:22 + | +20 | #[serde(rename = "Pay-Out")] + | ^^^^^^^^^ + +error: fact input field `gold-` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:22:43 + | +22 | #[serde(alias = "goldAmount", alias = "gold-")] + | ^^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr b/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr index 3707c3aac..80a24fe32 100644 --- a/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr +++ b/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr @@ -1,4 +1,4 @@ -error: fact input field `principal` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards +error: fact input field `principal` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values --> tests/compile_fail/fact_identity_field.rs:7:5 | 7 | principal: String, diff --git a/distributed_game_macros/tests/compile_fail/fact_identity_variants.rs b/distributed_game_macros/tests/compile_fail/fact_identity_variants.rs new file mode 100644 index 000000000..c2c6e7f88 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_identity_variants.rs @@ -0,0 +1,15 @@ +use distributed_game::FactCommand; + +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "crawler.record_kill")] +struct RecordKill { + mob: String, + UserId: String, + #[serde(rename = "owner-id")] + who: String, + ACCOUNT_ID: String, + #[serde(rename = "Experience")] + gained: u32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/fact_identity_variants.stderr b/distributed_game_macros/tests/compile_fail/fact_identity_variants.stderr new file mode 100644 index 000000000..d1c969bd5 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_identity_variants.stderr @@ -0,0 +1,23 @@ +error: fact input field `UserId` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:7:5 + | +7 | UserId: String, + | ^^^^^^ + +error: fact input field `owner-id` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:8:22 + | +8 | #[serde(rename = "owner-id")] + | ^^^^^^^^^^ + +error: fact input field `ACCOUNT_ID` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:10:5 + | +10 | ACCOUNT_ID: String, + | ^^^^^^^^^^ + +error: fact input field `Experience` looks like a reward: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:11:22 + | +11 | #[serde(rename = "Experience")] + | ^^^^^^^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/fact_reward_field.rs b/distributed_game_macros/tests/compile_fail/fact_reward_field.rs index 20944d60b..33a404f30 100644 --- a/distributed_game_macros/tests/compile_fail/fact_reward_field.rs +++ b/distributed_game_macros/tests/compile_fail/fact_reward_field.rs @@ -5,7 +5,7 @@ use distributed_game::FactCommand; struct RecordKill { mob: String, #[serde(rename = "XP")] - amount: u32, + count: u32, } fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr b/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr index 4bf6b46a6..62e0d6d0c 100644 --- a/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr +++ b/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr @@ -1,4 +1,4 @@ -error: fact input field `XP` looks like a reward: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards +error: fact input field `XP` looks like a reward: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values --> tests/compile_fail/fact_reward_field.rs:7:22 | 7 | #[serde(rename = "XP")] From 61dc71b9032a1b7628b7df1892ddc0657f1d7ba4 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Mon, 19 Oct 2026 23:54:43 -0500 Subject: [PATCH 02/40] feat(game): stamp facts with wall time, activation and config FactStamp gains emitted_at_ms (Clock::unix_ms, new required Clock method; ManualClock takes a Unix origin), activation (boot epoch) and config_version (HostConfig::config_version, settable from the host or a hook). Session vars and event metadata carry all six fields; cell::session_stamp and FactStamp::from_metadata read them back, and FactStamp::emission_order orders facts independent of commit order. Handoffs carry the source's departure time in the transfer ticket (`left`): source on_leave facts are pinned to it and the target stamps everything after it, so a join sorts after the leave even with clock skew. Replay format bumps to DGR3 (config version, departure time, SetConfigVersion event). [[tasks/gr-p25-fact-stamp-ordering]] --- distributed_game/README.md | 16 ++- distributed_game/src/cell.rs | 8 ++ distributed_game/src/clock.rs | 66 ++++++++- distributed_game/src/handoff.rs | 7 + distributed_game/src/host.rs | 87 ++++++++--- distributed_game/src/net/mod.rs | 1 + distributed_game/src/net/ticket.rs | 9 +- distributed_game/src/replay.rs | 40 +++++- distributed_game/src/stamp.rs | 168 +++++++++++++++++++--- distributed_game/tests/cell_sink/main.rs | 25 +++- distributed_game/tests/world/main.rs | 1 + distributed_game/tests/world/realm.rs | 17 ++- distributed_game/tests/world_host/main.rs | 125 ++++++++++++++++ 13 files changed, 522 insertions(+), 48 deletions(-) diff --git a/distributed_game/README.md b/distributed_game/README.md index 824162e39..f03dab693 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -682,6 +682,18 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), activation's boot epoch for you (see [Durable facts](#durable-facts)). - If the process crashes, each in-flight fact is either fully saved or not saved at all. Mob Pit shows this with `kill -9`. +- Every fact carries a `FactStamp`: zone, activation (boot epoch), tick, + host wall time `emitted_at_ms` (`Clock::unix_ms`; `ManualClock` in + tests), content version and config version (`HostConfig::config_version`, + `ctx.set_config_version`). The cell sink passes it to handlers as + trusted `x-game-*` session variables (`cell::session_stamp`), and + `stamp_metadata` / `FactStamp::from_metadata` copy it onto events. +- **Commit order is not emission order.** Order facts with + `FactStamp::emission_order` (same zone: activation, then tick; across + zones: wall time). After a handoff the target's stamps start after the + source's departure time, which the transfer ticket carries, so "joined + B" sorts after "left A" even when it commits first or the two hosts' + clocks disagree. ### Scaling out: many zones, many servers @@ -871,7 +883,7 @@ Settlements: `settlement.is::()`, and > do it for you. The host binds the principal from the player's **verified ticket** and -stamps zone/tick/content version. A `FactSink` commits facts on a worker +stamps zone, activation, tick, wall time, content and config version. A `FactSink` commits facts on a worker with bounded retries/backoff; `Zone::on_fact_settled` receives the outcome on a later tick. `cell::CellFactSink` dispatches into an `AggregateCell` with a deterministic command id per fact key, so retries replay instead of @@ -920,7 +932,7 @@ principals commit concurrently; settlements arrive in completion order. stored history. The interpolation delay travels in the game's input payload (no wire change). - **Record/replay** (`replay`): drive a host through `Recorder` (same - methods as `ZoneHost`), save the compact `Recording` (`DGR2`), and + methods as `ZoneHost`), save the compact `Recording` (`DGR3`), and `replay::(&recording, config)` re-runs it on a `ManualClock`, checking a per-tick state hash (`Zone::state_hash`, or the replicated `WorldView`). A mismatch returns a `Divergence` naming the tick and the diff --git a/distributed_game/src/cell.rs b/distributed_game/src/cell.rs index 62e363df0..f962400ff 100644 --- a/distributed_game/src/cell.rs +++ b/distributed_game/src/cell.rs @@ -87,6 +87,14 @@ pub fn trusted_session(principal: &str, fact: &Fact) -> Session { session } +/// The [`FactStamp`](crate::FactStamp) a [`trusted_session`] carries +/// (None for a session that did not come from a fact). Handlers use it to +/// order facts ([`FactStamp::emission_order`](crate::FactStamp::emission_order)) +/// and to [`stamp_metadata`](crate::stamp_metadata) their events. +pub fn session_stamp(session: &Session) -> Option { + crate::FactStamp::from_session_vars(session.variables()) +} + fn classify(error: CellDispatchError) -> FactError { match error { CellDispatchError::InProgress | CellDispatchError::Internal(_) => { diff --git a/distributed_game/src/clock.rs b/distributed_game/src/clock.rs index 1554b7158..a1b60758c 100644 --- a/distributed_game/src/clock.rs +++ b/distributed_game/src/clock.rs @@ -8,21 +8,32 @@ //! A clock reports elapsed time since its own origin as a [`Duration`]. Only //! differences matter to the runtime; values are also sent to clients as //! "server time" in milliseconds for clock synchronization. +//! +//! [`Clock::unix_ms`] is the clock's wall time (Unix milliseconds), used to +//! stamp facts ([`FactStamp::emitted_at_ms`](crate::FactStamp::emitted_at_ms)) +//! so facts from different zones and hosts can be ordered. It advances with +//! [`now`](Clock::now) and never goes backwards either. use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; /// Monotonic time source. pub trait Clock: Send + Sync + 'static { /// Time elapsed since this clock's origin. Must never go backwards. fn now(&self) -> Duration; + + /// Wall time in Unix milliseconds. Must never go backwards. + fn unix_ms(&self) -> u64; } -/// Real monotonic clock backed by [`Instant`]. +/// Real monotonic clock backed by [`Instant`]. Its wall time is the system +/// time read once at creation plus the monotonic elapsed time, so a system +/// clock step (NTP) never moves fact stamps backwards. #[derive(Debug, Clone)] pub struct SystemClock { origin: Instant, + origin_unix: Duration, } impl SystemClock { @@ -30,6 +41,9 @@ impl SystemClock { pub fn new() -> Self { Self { origin: Instant::now(), + origin_unix: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(), } } } @@ -44,12 +58,19 @@ impl Clock for SystemClock { fn now(&self) -> Duration { self.origin.elapsed() } + + fn unix_ms(&self) -> u64 { + (self.origin_unix + self.origin.elapsed()).as_millis() as u64 + } } -/// Virtual clock advanced explicitly; clones share the same time. +/// Virtual clock advanced explicitly; clones share the same time. Its wall +/// time is a fixed Unix origin (0 unless set with +/// [`with_unix_origin_ms`](Self::with_unix_origin_ms)) plus the virtual time. #[derive(Debug, Clone, Default)] pub struct ManualClock { nanos: Arc, + unix_origin_ms: u64, } impl ManualClock { @@ -58,6 +79,14 @@ impl ManualClock { Self::default() } + /// The same clock reporting wall time `origin_ms + now()` (e.g. two + /// hosts whose wall clocks disagree). Clones made before this call keep + /// their own origin but share the virtual time. + pub fn with_unix_origin_ms(mut self, origin_ms: u64) -> Self { + self.unix_origin_ms = origin_ms; + self + } + /// Move time forward by `by`. pub fn advance(&self, by: Duration) { self.nanos.fetch_add(by.as_nanos() as u64, Ordering::SeqCst); @@ -73,4 +102,35 @@ impl Clock for ManualClock { fn now(&self) -> Duration { Duration::from_nanos(self.nanos.load(Ordering::SeqCst)) } + + fn unix_ms(&self) -> u64 { + self.unix_origin_ms + self.now().as_millis() as u64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manual_wall_time_follows_virtual_time() { + let clock = ManualClock::new().with_unix_origin_ms(1_000); + let shared = clock.clone(); + assert_eq!(clock.unix_ms(), 1_000); + shared.advance(Duration::from_millis(250)); + assert_eq!(clock.unix_ms(), 1_250); + let skewed = shared.clone().with_unix_origin_ms(500); + assert_eq!(skewed.unix_ms(), 750); + } + + #[test] + fn system_wall_time_is_close_to_system_time() { + let clock = SystemClock::new(); + let system = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + assert!(clock.unix_ms().abs_diff(system) < 1_000); + assert!(clock.unix_ms() <= clock.unix_ms()); + } } diff --git a/distributed_game/src/handoff.rs b/distributed_game/src/handoff.rs index e8f67d081..d50bcaf8a 100644 --- a/distributed_game/src/handoff.rs +++ b/distributed_game/src/handoff.rs @@ -28,6 +28,10 @@ pub struct HandoffRequest { pub target_zone: String, /// Game state carried along (inventory, buffs, position hint...). pub carried: Vec, + /// Source stamp time of the departure (Unix ms): the facts the source + /// emits while the player leaves carry it, and the target stamps + /// everything after it ([`FactStamp`](crate::FactStamp) ordering). + pub departed_at_ms: u64, } /// Verified transfer ticket contents. @@ -43,6 +47,9 @@ pub struct Transfer { pub generation: u64, /// Carried state. pub carried: Vec, + /// Source stamp time of the departure (Unix ms, see + /// [`HandoffRequest::departed_at_ms`]). + pub departed_at_ms: u64, } /// Why [`Ctx::handoff`](crate::Ctx::handoff) refused. diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index a68a42cdb..7ccb1f6fd 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -92,6 +92,11 @@ pub struct HostConfig { /// Unix time in microseconds when the host is created. The `net` server /// sets it to the activation's fencing token. pub boot_epoch: u64, + /// Remote config version the zone runs (opaque; e.g. `"combat@12"`): + /// stamped on every fact ([`FactStamp::config_version`]). Change it at + /// runtime with [`ZoneHost::set_config_version`] / + /// [`Ctx::set_config_version`]. + pub config_version: String, } impl Default for HostConfig { @@ -114,6 +119,7 @@ impl Default for HostConfig { rewind: RewindConfig::default(), stale_policy: StalePolicy::Strict, boot_epoch: 0, + config_version: String::new(), } } } @@ -247,12 +253,20 @@ pub(crate) struct Core { pending_kicks: Vec<(PlayerId, String)>, rewind: Rewind, transfer_issuer: Option>, - pending_handoffs: Vec<(PlayerId, ServerMessage)>, + /// Handoffs to finish after the hook: player, `Handoff` message and the + /// departure stamp time signed into the transfer ticket. + pending_handoffs: Vec<(PlayerId, ServerMessage, u64)>, upward: Vec, child_inbox: Vec, dropped_frozen: u64, /// Typed facts keyed without dedupe parts this tick: `(tick, count)`. fact_seq: (u64, u32), + /// Stamps are never earlier than this (Unix ms): raised past a handoff + /// source's departure time when a transfer is admitted. + stamp_floor_ms: u64, + /// Stamp time forced while a handoff departure runs `on_leave`, so the + /// source's departure facts carry exactly the time in the ticket. + pinned_stamp_ms: Option, } impl Core { @@ -284,6 +298,23 @@ impl Core { fn in_flight(&self) -> usize { self.facts.as_ref().map_or(0, FactQueue::in_flight) } + + /// Wall time stamped on anything produced now (Unix ms). + fn stamp_ms(&self) -> u64 { + self.pinned_stamp_ms + .unwrap_or_else(|| self.clock.unix_ms().max(self.stamp_floor_ms)) + } + + fn stamp(&self) -> FactStamp { + FactStamp { + zone_id: self.zone_id.clone(), + tick: self.tick, + content_version: self.config.content_version.clone(), + emitted_at_ms: self.stamp_ms(), + activation: self.config.boot_epoch, + config_version: self.config.config_version.clone(), + } + } } /// Hook context: everything a zone may touch besides its own state. @@ -332,6 +363,17 @@ impl Ctx<'_, Z> { &self.core.config.content_version } + /// Remote config version ([`HostConfig::config_version`]). + pub fn config_version(&self) -> &str { + &self.core.config.config_version + } + + /// Switch the remote config version (e.g. on a config-changed + /// message); facts and stamps produced afterwards carry it. + pub fn set_config_version(&mut self, version: impl Into) { + self.core.config.config_version = version.into(); + } + /// Epoch of this zone activation ([`HostConfig::boot_epoch`]): unique /// per activation and increasing across restarts and hibernation. /// Tick counts and entity generations restart with the process, so a @@ -509,11 +551,7 @@ impl Ctx<'_, Z> { } fn emit_bound(&mut self, fact: Fact, principal: Option) -> Result<(), EmitError> { - let stamp = FactStamp { - zone_id: self.core.zone_id.clone(), - tick: self.core.tick, - content_version: self.core.config.content_version.clone(), - }; + let stamp = self.core.stamp(); self.core .facts .as_mut() @@ -582,15 +620,11 @@ impl Ctx<'_, Z> { self.core.pending_kicks.push((player, message.into())); } - /// Provenance of anything produced now (zone, tick, content version); - /// embed it in events so clients and logs can tell which content - /// version produced them. + /// Provenance of anything produced now (zone, activation, tick, wall + /// time, content and config version); embed it in events so clients + /// and logs can tell which content version produced them. pub fn stamp(&self) -> FactStamp { - FactStamp { - zone_id: self.core.zone_id.clone(), - tick: self.core.tick, - content_version: self.core.config.content_version.clone(), - } + self.core.stamp() } /// Move `player` to `target_zone` with `carried` state. A signed @@ -621,16 +655,18 @@ impl Ctx<'_, Z> { .core .pending_handoffs .iter() - .any(|(queued, _)| *queued == player) + .any(|(queued, _, _)| *queued == player) { return Err(HandoffError::Refused); } + let departed_at_ms = self.core.stamp_ms(); let request = HandoffRequest { principal, name, source_zone: self.core.zone_id.clone(), target_zone: target_zone.to_string(), carried, + departed_at_ms, }; let issuer = self .core @@ -644,6 +680,7 @@ impl Ctx<'_, Z> { target_zone: request.target_zone, transfer_ticket: ticket, }, + departed_at_ms, )); Ok(()) } @@ -729,6 +766,8 @@ impl ZoneHost { child_inbox: Vec::new(), dropped_frozen: 0, fact_seq: (0, 0), + stamp_floor_ms: 0, + pinned_stamp_ms: None, config, }; let zone = Z::on_create(&mut Ctx { core: &mut core }, zone_config); @@ -922,6 +961,10 @@ impl ZoneHost { /// Admit a player arriving with a verified transfer ticket; the zone /// sees [`Zone::on_handoff_join`] with the carried state (also when the /// principal is still present, e.g. inside its reconnect grace). + /// + /// From now on this zone stamps facts later than the source stamped the + /// departure ([`Transfer::departed_at_ms`]), whatever the two hosts' + /// clocks say. pub fn join_transfer( &mut self, transfer: &Transfer, @@ -934,6 +977,10 @@ impl ZoneHost { "transfers are for players", )); } + self.core.stamp_floor_ms = self + .core + .stamp_floor_ms + .max(transfer.departed_at_ms.saturating_add(1)); self.admit( &transfer.claims, capabilities, @@ -1209,6 +1256,12 @@ impl ZoneHost { .map(|slot| slot.content_version.as_str()) } + /// Switch the remote config version ([`HostConfig::config_version`]); + /// facts and stamps produced afterwards carry it. + pub fn set_config_version(&mut self, version: impl Into) { + self.core.config.config_version = version.into(); + } + /// Switch the zone to a new content version /// ([`Zone::on_content_version_changed`]); facts and stamps produced /// afterwards carry it. @@ -1505,8 +1558,10 @@ impl ZoneHost { } fn process_kicks(&mut self) { - while let Some((player, message)) = self.core.pending_handoffs.pop() { + while let Some((player, message, departed_at_ms)) = self.core.pending_handoffs.pop() { + self.core.pinned_stamp_ms = Some(departed_at_ms); self.remove_player(player, LeaveReason::Handoff, Some(message)); + self.core.pinned_stamp_ms = None; } while let Some((player, message)) = self.core.pending_kicks.pop() { let kick = ServerMessage::Kick { diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index ff5827c08..f65a0f080 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -1081,6 +1081,7 @@ async fn run_zone( issuer: inner.host_id.clone(), generation, carried: request.carried.clone(), + departed_at_ms: request.departed_at_ms, })) }); if let Some(saved) = &saved { diff --git a/distributed_game/src/net/ticket.rs b/distributed_game/src/net/ticket.rs index 2122bdf31..c2ef15e75 100644 --- a/distributed_game/src/net/ticket.rs +++ b/distributed_game/src/net/ticket.rs @@ -9,7 +9,8 @@ //! //! **Transfer tickets** (zone handoff) are the same format with //! `"kind":"transfer"` plus `src` (source zone), `iss` (issuing host id), -//! `gen` (source ownership token) and `carry` (base64url carried state). +//! `gen` (source ownership token), `left` (source departure stamp time, +//! Unix ms) and `carry` (base64url carried state). //! [`TicketKey::verify`] rejects them ([`TicketError::WrongKind`]); use //! [`TicketKey::verify_ticket`] to accept either kind. //! @@ -185,6 +186,7 @@ impl TicketKey { "src": transfer.source_zone, "iss": transfer.issuer, "gen": transfer.generation, + "left": transfer.departed_at_ms, "carry": URL_SAFE_NO_PAD.encode(&transfer.carried), }); self.sign_body(&with_name(body, claims)) @@ -238,6 +240,10 @@ impl TicketKey { .get("gen") .and_then(Value::as_u64) .ok_or(TicketError::MissingClaim("gen"))?, + departed_at_ms: body + .get("left") + .and_then(Value::as_u64) + .ok_or(TicketError::MissingClaim("left"))?, carried, claims, })) @@ -411,6 +417,7 @@ mod tests { issuer: "h1".into(), generation: 9, carried: vec![1, 2, 3], + departed_at_ms: 1_700_000_000_123, }; let ticket = key.sign_transfer(&transfer); assert_eq!( diff --git a/distributed_game/src/replay.rs b/distributed_game/src/replay.rs index 3caf4b262..b9cd61db4 100644 --- a/distributed_game/src/replay.rs +++ b/distributed_game/src/replay.rs @@ -50,7 +50,7 @@ use crate::schema::FieldValue; use crate::world::{ChildEvent, StalePolicy}; use crate::zone::{ClientId, PlayerId, TicketClaims, Zone}; -const MAGIC: &[u8; 4] = b"DGR2"; +const MAGIC: &[u8; 4] = b"DGR3"; /// FNV digest of a replicated world (entity order, ids, kinds, positions, /// priorities and field values). @@ -216,6 +216,13 @@ pub enum Event { /// New version. version: String, }, + /// [`ZoneHost::set_config_version`]. + SetConfigVersion { + /// Clock. + at: Duration, + /// New version. + version: String, + }, /// [`ZoneHost::request_freeze`]. RequestFreeze { /// Clock. @@ -269,6 +276,7 @@ impl Event { | Self::JoinTransfer { at, .. } | Self::ReconnectVersion { at, .. } | Self::SetContentVersion { at, .. } + | Self::SetConfigVersion { at, .. } | Self::RequestFreeze { at, .. } | Self::CancelFreeze { at, .. } | Self::ReloadStale { at } @@ -336,6 +344,9 @@ impl fmt::Display for Event { Self::SetContentVersion { version, .. } => { write!(f, "@{ms:.1}ms content version {version}") } + Self::SetConfigVersion { version, .. } => { + write!(f, "@{ms:.1}ms config version {version}") + } Self::RequestFreeze { notice, .. } => { write!(f, "@{ms:.1}ms request freeze {}", notice.reason) } @@ -375,7 +386,7 @@ pub struct Recording { /// File decode failure. #[derive(Debug, Clone, PartialEq, Eq)] pub enum RecordingError { - /// Not a `DGR2` file. + /// Not a `DGR3` file. BadMagic, /// Malformed content. Decode(String), @@ -552,6 +563,7 @@ fn put_config(w: &mut Writer, c: &HostConfig) { StalePolicy::SafeRoom { roam: true } => 2, }); put_u64(w, c.boot_epoch); + w.string(&c.config_version); } fn put_claims(w: &mut Writer, claims: &TicketClaims) { @@ -624,12 +636,13 @@ fn get_config(r: &mut Reader<'_>) -> Result { _ => StalePolicy::SafeRoom { roam: true }, }, boot_epoch: get_u64(r)?, + config_version: r.string()?, }) } impl Recording { /// Compact binary encoding: - /// `"DGR2" ‖ zone_id ‖ config ‖ created_at ‖ fact_capacity? ‖ zone_config ‖ + /// `"DGR3" ‖ zone_id ‖ config ‖ created_at ‖ fact_capacity? ‖ zone_config ‖ /// varuint(event count) ‖ events`. Integers are LEB128 varints, /// durations are nanosecond varints, clock readings are delta-coded /// against the previous event, hashes are 8 raw bytes. @@ -741,6 +754,7 @@ impl Recording { put_claims(&mut w, &transfer.claims); w.string(&transfer.source_zone).string(&transfer.issuer); put_u64(&mut w, transfer.generation); + put_u64(&mut w, transfer.departed_at_ms); w.bytes(&transfer.carried) .varuint(*capabilities) .string(content_version); @@ -789,6 +803,9 @@ impl Recording { } => { w.u8(16).bytes(state).string(saved_version); } + Event::SetConfigVersion { version, .. } => { + w.u8(17).string(version); + } } } w.into_bytes() @@ -885,6 +902,7 @@ impl Recording { let source_zone = r.string()?; let issuer = r.string()?; let generation = get_u64(&mut r)?; + let departed_at_ms = get_u64(&mut r)?; let carried = r.bytes()?.to_vec(); Event::JoinTransfer { at, @@ -894,6 +912,7 @@ impl Recording { issuer, generation, carried, + departed_at_ms, }, capabilities: r.varuint()?, content_version: r.string()?, @@ -937,6 +956,10 @@ impl Recording { state: r.bytes()?.to_vec(), saved_version: r.string()?, }, + 17 => Event::SetConfigVersion { + at, + version: r.string()?, + }, _ => return Err(bad("event tag")), }; events.push(event); @@ -1241,6 +1264,16 @@ impl Recorder { }); } + /// [`ZoneHost::set_config_version`]. + pub fn set_config_version(&mut self, version: &str) { + let at = self.now(); + self.host.set_config_version(version); + self.recording.events.push(Event::SetConfigVersion { + at, + version: version.into(), + }); + } + /// [`ZoneHost::request_freeze`]. pub fn request_freeze(&mut self, notice: Notice) -> crate::world::FreezeState { let at = self.now(); @@ -1571,6 +1604,7 @@ pub fn replay_with( } } Event::SetContentVersion { version, .. } => host.set_content_version(version.clone()), + Event::SetConfigVersion { version, .. } => host.set_config_version(version.clone()), Event::RequestFreeze { notice, .. } => { host.request_freeze(notice.clone()); } diff --git a/distributed_game/src/stamp.rs b/distributed_game/src/stamp.rs index a9d7401ab..cd237c528 100644 --- a/distributed_game/src/stamp.rs +++ b/distributed_game/src/stamp.rs @@ -1,12 +1,25 @@ //! Tick stamping: provenance of committed facts. //! -//! Every [`Fact`](crate::Fact) carries a [`FactStamp`] (zone id, tick, -//! content version) filled in by the host when the zone emits it. The cell -//! adapter forwards the stamp as trusted session variables; command handlers -//! copy it into `EventRecord.metadata` with [`stamp_metadata`] (or -//! `stamp_event_record` with the `cell` feature) so every durable event -//! records exactly which simulation tick produced it. +//! Every [`Fact`](crate::Fact) carries a [`FactStamp`] filled in by the host +//! when the zone emits it: zone id, activation, tick, host wall time, +//! content version and config version. The cell adapter forwards the stamp +//! as trusted session variables; command handlers copy it into +//! `EventRecord.metadata` with [`stamp_metadata`] (or `stamp_event_record` +//! with the `cell` feature) so every durable event records exactly which +//! simulation tick produced it. +//! +//! **Ordering.** Commit order is not emission order: facts travel through +//! per-principal lanes and retries, and two zones commit independently. A +//! handler that must order facts (presence: "left A" vs "joined B" after a +//! handoff) compares stamps with [`FactStamp::emission_order`]. Within one +//! zone activation the tick decides; across activations of one zone the +//! activation (it grows across restarts); across zones the host wall time +//! `emitted_at_ms`. A handoff target never stamps the arriving player's +//! facts earlier than the source stamped its departure: the transfer ticket +//! carries the departure time and the target's stamps start after it, even +//! when the two hosts' clocks disagree. +use std::cmp::Ordering; use std::collections::HashMap; /// Metadata key for the zone id. @@ -15,6 +28,12 @@ pub const META_ZONE: &str = "game.zone"; pub const META_TICK: &str = "game.tick"; /// Metadata key for the content version. pub const META_CONTENT_VERSION: &str = "game.content_version"; +/// Metadata key for the host wall time at emission (Unix ms, decimal). +pub const META_EMITTED_AT_MS: &str = "game.emitted_at_ms"; +/// Metadata key for the zone activation (boot epoch, decimal). +pub const META_ACTIVATION: &str = "game.activation"; +/// Metadata key for the config version. +pub const META_CONFIG_VERSION: &str = "game.config_version"; /// Session-variable prefix used to carry a stamp through a trusted Session. pub const SESSION_PREFIX: &str = "x-game-"; @@ -28,20 +47,36 @@ pub struct FactStamp { pub tick: u64, /// Content version the zone was running. pub content_version: String, + /// Host wall time at emission, Unix milliseconds + /// ([`Clock::unix_ms`](crate::Clock::unix_ms), never earlier than a + /// handoff source's departure stamp). + pub emitted_at_ms: u64, + /// Zone activation ([`HostConfig::boot_epoch`](crate::HostConfig::boot_epoch): + /// the ownership fencing token under the `net` server); grows across + /// restarts and hibernation. + pub activation: u64, + /// Remote config version the zone was running + /// ([`HostConfig::config_version`](crate::HostConfig::config_version)). + pub config_version: String, } impl FactStamp { /// Metadata entries for this stamp. - pub fn entries(&self) -> [(&'static str, String); 3] { + pub fn entries(&self) -> [(&'static str, String); 6] { [ (META_ZONE, self.zone_id.clone()), (META_TICK, self.tick.to_string()), (META_CONTENT_VERSION, self.content_version.clone()), + (META_EMITTED_AT_MS, self.emitted_at_ms.to_string()), + (META_ACTIVATION, self.activation.to_string()), + (META_CONFIG_VERSION, self.config_version.clone()), ] } - /// Session variables (`x-game-zone`, `x-game-tick`, `x-game-content-version`). - pub fn session_vars(&self) -> [(String, String); 3] { + /// Session variables (`x-game-zone`, `x-game-tick`, + /// `x-game-content-version`, `x-game-emitted-at-ms`, + /// `x-game-activation`, `x-game-config-version`). + pub fn session_vars(&self) -> [(String, String); 6] { [ (format!("{SESSION_PREFIX}zone"), self.zone_id.clone()), (format!("{SESSION_PREFIX}tick"), self.tick.to_string()), @@ -49,19 +84,72 @@ impl FactStamp { format!("{SESSION_PREFIX}content-version"), self.content_version.clone(), ), + ( + format!("{SESSION_PREFIX}emitted-at-ms"), + self.emitted_at_ms.to_string(), + ), + ( + format!("{SESSION_PREFIX}activation"), + self.activation.to_string(), + ), + ( + format!("{SESSION_PREFIX}config-version"), + self.config_version.clone(), + ), ] } - /// Read a stamp back from session variables (None if absent/malformed). + /// Read a stamp back from session variables (None if any is + /// absent/malformed). pub fn from_session_vars(vars: &HashMap) -> Option { + let get = |name: &str| vars.get(&format!("{SESSION_PREFIX}{name}")); Some(Self { - zone_id: vars.get(&format!("{SESSION_PREFIX}zone"))?.clone(), - tick: vars.get(&format!("{SESSION_PREFIX}tick"))?.parse().ok()?, - content_version: vars - .get(&format!("{SESSION_PREFIX}content-version"))? - .clone(), + zone_id: get("zone")?.clone(), + tick: get("tick")?.parse().ok()?, + content_version: get("content-version")?.clone(), + emitted_at_ms: get("emitted-at-ms")?.parse().ok()?, + activation: get("activation")?.parse().ok()?, + config_version: get("config-version")?.clone(), }) } + + /// Read a stamp back from event metadata written by [`stamp_metadata`] + /// (None if any entry is absent/malformed). + pub fn from_metadata(metadata: &HashMap) -> Option { + Some(Self { + zone_id: metadata.get(META_ZONE)?.clone(), + tick: metadata.get(META_TICK)?.parse().ok()?, + content_version: metadata.get(META_CONTENT_VERSION)?.clone(), + emitted_at_ms: metadata.get(META_EMITTED_AT_MS)?.parse().ok()?, + activation: metadata.get(META_ACTIVATION)?.parse().ok()?, + config_version: metadata.get(META_CONFIG_VERSION)?.clone(), + }) + } + + /// Emission order of two stamps: same zone and activation by tick (then + /// wall time); same zone across activations by activation; different + /// zones by wall time (then zone id, activation and tick, so the order + /// is total and deterministic). Commit order says nothing about this. + pub fn emission_order(&self, other: &Self) -> Ordering { + if self.zone_id == other.zone_id { + self.activation + .cmp(&other.activation) + .then(self.tick.cmp(&other.tick)) + .then(self.emitted_at_ms.cmp(&other.emitted_at_ms)) + } else { + self.emitted_at_ms + .cmp(&other.emitted_at_ms) + .then_with(|| self.zone_id.cmp(&other.zone_id)) + .then(self.activation.cmp(&other.activation)) + .then(self.tick.cmp(&other.tick)) + } + } + + /// True when this stamp was emitted after `other` + /// ([`emission_order`](Self::emission_order)). + pub fn emitted_after(&self, other: &Self) -> bool { + self.emission_order(other) == Ordering::Greater + } } /// Insert the stamp into an event metadata map. @@ -81,18 +169,58 @@ pub fn stamp_event_record(record: &mut distributed::EventRecord, stamp: &FactSta mod tests { use super::*; + fn stamp(zone: &str, activation: u64, tick: u64, at: u64) -> FactStamp { + FactStamp { + zone_id: zone.into(), + tick, + content_version: "c7".into(), + emitted_at_ms: at, + activation, + config_version: "combat@12".into(), + } + } + #[test] fn stamp_round_trips_through_session_vars_and_metadata() { - let stamp = FactStamp { - zone_id: "z1".into(), - tick: 42, - content_version: "c7".into(), - }; + let stamp = stamp("z1", 9, 42, 1_700_000_000_123); let vars: HashMap<_, _> = stamp.session_vars().into_iter().collect(); assert_eq!(FactStamp::from_session_vars(&vars), Some(stamp.clone())); let mut meta = HashMap::new(); stamp_metadata(&mut meta, &stamp); assert_eq!(meta[META_TICK], "42"); assert_eq!(meta[META_ZONE], "z1"); + assert_eq!(meta[META_EMITTED_AT_MS], "1700000000123"); + assert_eq!(meta[META_ACTIVATION], "9"); + assert_eq!(meta[META_CONFIG_VERSION], "combat@12"); + assert_eq!(FactStamp::from_metadata(&meta), Some(stamp)); + } + + #[test] + fn missing_or_malformed_fields_are_rejected() { + let stamp = stamp("z1", 9, 42, 5); + let mut vars: HashMap<_, _> = stamp.session_vars().into_iter().collect(); + vars.remove("x-game-activation"); + assert_eq!(FactStamp::from_session_vars(&vars), None); + let mut vars: HashMap<_, _> = stamp.session_vars().into_iter().collect(); + vars.insert("x-game-emitted-at-ms".into(), "soon".into()); + assert_eq!(FactStamp::from_session_vars(&vars), None); + let mut meta = HashMap::new(); + stamp_metadata(&mut meta, &stamp); + meta.remove(META_CONFIG_VERSION); + assert_eq!(FactStamp::from_metadata(&meta), None); + } + + #[test] + fn emission_order_by_tick_activation_then_wall_time() { + // One activation: the tick wins over a wall clock read out of order. + assert!(stamp("a", 1, 11, 100).emitted_after(&stamp("a", 1, 10, 200))); + // A later activation of the same zone is later, ticks restart. + assert!(stamp("a", 2, 1, 50).emitted_after(&stamp("a", 1, 900, 60))); + // Different zones: wall time. + assert!(stamp("b", 1, 1, 201).emitted_after(&stamp("a", 7, 99, 200))); + assert!(!stamp("a", 7, 99, 200).emitted_after(&stamp("b", 1, 1, 201))); + // Ties are broken deterministically. + let (x, y) = (stamp("a", 1, 1, 5), stamp("b", 1, 1, 5)); + assert_eq!(x.emission_order(&y), y.emission_order(&x).reverse()); } } diff --git a/distributed_game/tests/cell_sink/main.rs b/distributed_game/tests/cell_sink/main.rs index 9cf78fc0f..1eca5c85f 100644 --- a/distributed_game/tests/cell_sink/main.rs +++ b/distributed_game/tests/cell_sink/main.rs @@ -10,9 +10,11 @@ use distributed::microsvc::{ CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, }; use distributed::{sourced, Entity}; -use distributed_game::cell::{command_id_for, CellFactSink}; +use std::collections::HashMap; + +use distributed_game::cell::{command_id_for, session_stamp, CellFactSink}; use distributed_game::{ - fact_channel, spawn_fact_worker, Fact, FactOutcome, FactStamp, RetryPolicy, + fact_channel, spawn_fact_worker, stamp_metadata, Fact, FactOutcome, FactStamp, RetryPolicy, }; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -42,6 +44,11 @@ struct CreditPayload { hero: String, xp: u32, tick: String, + /// The stamp the handler read from the trusted session, after a trip + /// through event metadata. + emitted_at_ms: u64, + activation: u64, + config_version: String, } struct CreditXp; @@ -69,6 +76,11 @@ async fn handle_credit( // Identity comes from the trusted session, never from the input. let principal = ctx.session().user_id().unwrap_or_default().to_string(); let tick = ctx.session().get("x-game-tick").unwrap_or("").to_string(); + let stamp = session_stamp(ctx.session()) + .ok_or_else(|| HandlerError::Rejected("fact sessions carry a stamp".into()))?; + let mut metadata = HashMap::new(); + stamp_metadata(&mut metadata, &stamp); + let stamp = FactStamp::from_metadata(&metadata).expect("metadata round trip"); let repo = ctx.repo(); let mut hero = match repo.get(&principal).await? { Some(hero) => hero, @@ -81,6 +93,9 @@ async fn handle_credit( hero: principal, xp, tick, + emitted_at_ms: stamp.emitted_at_ms, + activation: stamp.activation, + config_version: stamp.config_version, }) } @@ -104,6 +119,9 @@ async fn facts_dispatch_idempotently_with_trusted_principal() { zone_id: "z1".into(), tick: 99, content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_456, + activation: 77, + config_version: "combat@12".into(), }; let fact = Fact::trusted( "coin:z1:7", @@ -131,6 +149,9 @@ async fn facts_dispatch_idempotently_with_trusted_principal() { assert_eq!(first.payload["hero"], "alice"); assert_eq!(first.payload["xp"], 10); assert_eq!(first.payload["tick"], "99"); + assert_eq!(first.payload["emitted_at_ms"], 1_700_000_000_456u64); + assert_eq!(first.payload["activation"], 77); + assert_eq!(first.payload["config_version"], "combat@12"); assert_eq!(cell.load().await.unwrap().unwrap().xp, 10, "credited once"); // Payload-supplied identity is not a declared input: permanent failure. diff --git a/distributed_game/tests/world/main.rs b/distributed_game/tests/world/main.rs index 6283d11f4..b5b0ff1cf 100644 --- a/distributed_game/tests/world/main.rs +++ b/distributed_game/tests/world/main.rs @@ -243,6 +243,7 @@ fn realm_factory( reject_all: rejecting.contains(&zone), gate: zone.ends_with(".inn"), freeze_ready: !zone.ends_with(".stuck"), + move_facts: false, }) }) } diff --git a/distributed_game/tests/world/realm.rs b/distributed_game/tests/world/realm.rs index 0ec711f33..1609c0ef2 100644 --- a/distributed_game/tests/world/realm.rs +++ b/distributed_game/tests/world/realm.rs @@ -20,6 +20,8 @@ pub struct RealmConfig { pub freeze_ready: bool, pub neighbour: Option, pub reject_all: bool, + /// Emit `realm.left` / `realm.arrived` facts for handoffs. + pub move_facts: bool, } impl RealmConfig { @@ -94,6 +96,15 @@ impl Zone for Realm { .player(player) .map(|p| p.principal.clone()) .unwrap_or_default(); + if self.cfg.move_facts { + let key = format!("arrived:{}:{principal}", ctx.zone_id()); + let fact = Fact::new( + key, + "realm.arrived", + serde_json::json!({ "from": from_zone }), + ); + let _ = ctx.emit_fact_for(player, fact); + } let mut payload = carried.clone(); payload.extend_from_slice(b"|"); payload.extend_from_slice(from_zone.as_bytes()); @@ -105,7 +116,11 @@ impl Zone for Realm { }); } - fn on_leave(&mut self, _ctx: &mut Ctx<'_, Self>, player: PlayerId, reason: LeaveReason) { + fn on_leave(&mut self, ctx: &mut Ctx<'_, Self>, player: PlayerId, reason: LeaveReason) { + if self.cfg.move_facts && reason == LeaveReason::Handoff { + let key = format!("left:{}:{}", ctx.zone_id(), player.0); + let _ = ctx.emit_fact(Fact::new(key, "realm.left", serde_json::json!({}))); + } self.left.push((player, reason)); } diff --git a/distributed_game/tests/world_host/main.rs b/distributed_game/tests/world_host/main.rs index 1699da6bc..92650bbd3 100644 --- a/distributed_game/tests/world_host/main.rs +++ b/distributed_game/tests/world_host/main.rs @@ -58,6 +58,7 @@ fn transfer(principal: &str, from: &str, to: &str) -> Transfer { issuer: "h1".into(), generation: 1, carried: b"gold=7".to_vec(), + departed_at_ms: 0, } } @@ -409,6 +410,7 @@ fn world_entry_points_record_and_replay_deterministically() { clock.advance(Duration::from_millis(40)); rec.advance(); rec.set_content_version("v2"); + rec.set_config_version("combat@2"); rec.resume(notice); rec.reload_stale_players(); let out = rec.drain_outbound(); @@ -421,6 +423,7 @@ fn world_entry_points_record_and_replay_deterministically() { clock.advance(Duration::from_millis(40)); rec.advance(); assert_eq!(rec.host().zone().counter, 6); + assert_eq!(rec.host().config().config_version, "combat@2"); assert_eq!(rec.host().zone().child_events.len(), 1); assert_eq!(rec.host().player_count(), 1, "bo handed off"); @@ -428,3 +431,125 @@ fn world_entry_points_record_and_replay_deterministically() { let summary = replay::(&recording, cfg).unwrap(); assert!(summary.ticks >= 2, "{summary:?}"); } + +#[test] +fn handoff_target_stamps_after_source_departure_despite_clock_skew() { + // One virtual time, two hosts whose wall clocks disagree: the target's + // reads 9 000 s *behind* the source's. + let time = ManualClock::new(); + let source_clock = time.clone().with_unix_origin_ms(10_000_000); + let target_clock = time.clone().with_unix_origin_ms(1_000_000); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let (source_queue, source_feed) = fact_channel(16, RetryPolicy::default()); + let mut source = ZoneHost::::new( + "realm.east", + HostConfig { + boot_epoch: 3, + config_version: "combat@12".into(), + ..HostConfig::default() + }, + Arc::new(source_clock), + RealmConfig { + neighbour: Some("realm.west".into()), + move_facts: true, + ..RealmConfig::new() + }, + ) + .with_facts(source_queue) + .with_transfer_issuer(move |r: &HandoffRequest| { + seen.lock().unwrap().push(r.clone()); + Some("signed".into()) + }); + let (target_queue, target_feed) = fact_channel(16, RetryPolicy::default()); + let mut target = ZoneHost::::new( + "realm.west", + HostConfig { + boot_epoch: 8, + ..HostConfig::default() + }, + Arc::new(target_clock), + RealmConfig { + move_facts: true, + ..RealmConfig::new() + }, + ) + .with_facts(target_queue); + + let ana = source.join(&claims("ana", "realm.east"), 0, "").unwrap(); + source.push_input(ana.client, 1, 0, &[CMD_HANDOFF]); + time.advance(Duration::from_millis(40)); + source.advance(); + let request = requests.lock().unwrap().pop().expect("handoff issued"); + // The ManualClock decides the stamp time: origin + virtual time. + assert_eq!(request.departed_at_ms, 10_000_040); + + // The source's leave runs later on its clock, still stamped with the + // departure time the ticket carries. + time.advance(Duration::from_millis(5)); + let transfer = Transfer { + claims: claims("ana", "realm.west"), + source_zone: request.source_zone.clone(), + issuer: "h1".into(), + generation: 3, + carried: request.carried.clone(), + departed_at_ms: request.departed_at_ms, + }; + target.join_transfer(&transfer, 0, "").unwrap(); + drop(source); + drop(target); + + // The target commits first. + let sink = MemoryFactSink::new(); + futures_executor(target_feed.run(sink.clone(), |_| async {})); + futures_executor(source_feed.run(sink.clone(), |_| async {})); + let committed = sink.committed(); + let commands: Vec<_> = committed.iter().map(Fact::command).collect(); + assert_eq!(commands, ["realm.arrived", "realm.left"]); + let (arrived, left) = (committed[0].stamp(), committed[1].stamp()); + + assert_eq!(left.emitted_at_ms, 10_000_040); + assert_eq!( + (left.activation, left.config_version.as_str()), + (3, "combat@12") + ); + // The target's own clock says 1 000 045; the stamp starts after the + // departure instead. + assert_eq!(arrived.emitted_at_ms, 10_000_041); + assert_eq!(arrived.activation, 8); + assert!(arrived.emitted_after(left)); + let mut ordered = committed.clone(); + ordered.sort_by(|a, b| a.stamp().emission_order(b.stamp())); + let ordered: Vec<_> = ordered.iter().map(Fact::command).collect(); + assert_eq!(ordered, ["realm.left", "realm.arrived"]); +} + +#[test] +fn stamps_follow_the_manual_clock_and_config_version() { + let clock = ManualClock::new().with_unix_origin_ms(500); + let h = host( + "z", + HostConfig { + boot_epoch: 42, + ..HostConfig::default() + }, + &clock, + RealmConfig::new(), + ); + clock.advance(Duration::from_millis(250)); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = h.with_facts(queue); + let joined = h.join(&claims("bo", "z"), 0, "").unwrap(); + h.push_input(joined.client, 1, 0, &[CMD_COUNT]); + h.set_config_version("combat@13"); + clock.advance(Duration::from_millis(40)); + h.advance(); + drop(h); + let sink = MemoryFactSink::new(); + futures_executor(feed.run(sink.clone(), |_| async {})); + let stamp = sink.committed()[0].stamp().clone(); + assert_eq!(stamp.emitted_at_ms, 790); + assert_eq!(stamp.activation, 42); + assert_eq!(stamp.config_version, "combat@13"); + assert_eq!(stamp.zone_id, "z"); +} From 7a32cc87078db38932ead1cec57aa9ad731fbd6a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 00:03:10 -0500 Subject: [PATCH 03/40] feat(game): typed ticket extension claims Tickets carry game-declared claims (match id, team, party) under `ext`. `#[derive(TicketExtension)]` declares them with the FactCommand deny list (shared module, JSON fixture checked by Rust and JS tests) and refuses rename_all/flatten. ServerConfig::with_ticket_extensions admits only the declared schema for join (any verifier) and transfer tickets; any ext without a schema is refused. Claims reach JoinRequest, PlayerInfo and ObserverInfo, are replaced on rejoin and carried by handoff transfer tickets. JS: mintGameTicket takes `extensions`, createGameTicketHandler gains a server-side `claims()` callback; the browser body still cannot add claims, and denied names or oversize objects answer 400. [[tasks/gr-p23-ticket-extension-claims]] --- distributed_game/README.md | 18 +- distributed_game/src/bots.rs | 1 + distributed_game/src/claims.rs | 247 ++++++++++++++++++ distributed_game/src/handoff.rs | 3 + distributed_game/src/host.rs | 14 +- distributed_game/src/lib.rs | 4 +- distributed_game/src/metrics.rs | 1 + distributed_game/src/net/mod.rs | 20 ++ distributed_game/src/net/ticket.rs | 21 +- distributed_game/src/replay.rs | 6 + distributed_game/src/zone.rs | 17 ++ .../tests/fixtures/sveltekit_tickets.json | 8 + .../fixtures/trusted_field_denylist.json | 6 + distributed_game/tests/net_e2e/main.rs | 140 +++++++++- distributed_game/tests/netcode/main.rs | 1 + distributed_game/tests/world/main.rs | 62 +++++ distributed_game/tests/world_host/main.rs | 1 + distributed_game/tests/zone_runtime/arena.rs | 4 + distributed_game/tests/zone_runtime/main.rs | 1 + distributed_game_macros/src/claims.rs | 69 +++++ distributed_game_macros/src/lib.rs | 12 + distributed_game_macros/src/trusted.rs | 24 +- .../compile_fail/ticket_extension_flatten.rs | 15 ++ .../ticket_extension_flatten.stderr | 5 + .../compile_fail/ticket_extension_identity.rs | 12 + .../ticket_extension_identity.stderr | 11 + .../ticket_extension_rename_all.rs | 9 + .../ticket_extension_rename_all.stderr | 5 + examples/game/server/src/common/dev.rs | 1 + examples/game/server/tests/gallery_smoke.rs | 1 + examples/game/server/tests/support/mod.rs | 1 + js/README.md | 10 +- js/src/sveltekit/game/server.ts | 39 ++- js/src/sveltekit/game/ticket.ts | 83 +++++- js/tests/sveltekit-game.test.mjs | 62 ++++- 35 files changed, 920 insertions(+), 14 deletions(-) create mode 100644 distributed_game/src/claims.rs create mode 100644 distributed_game/tests/fixtures/trusted_field_denylist.json create mode 100644 distributed_game_macros/src/claims.rs create mode 100644 distributed_game_macros/tests/compile_fail/ticket_extension_flatten.rs create mode 100644 distributed_game_macros/tests/compile_fail/ticket_extension_flatten.stderr create mode 100644 distributed_game_macros/tests/compile_fail/ticket_extension_identity.rs create mode 100644 distributed_game_macros/tests/compile_fail/ticket_extension_identity.stderr create mode 100644 distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.rs create mode 100644 distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.stderr diff --git a/distributed_game/README.md b/distributed_game/README.md index f03dab693..b9b201447 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -451,7 +451,9 @@ runnable: [`examples/game/sveltekit`](../examples/game/sveltekit) role }`; any other body field is refused, so identity, ownership and rewards never come from the request. Same-origin, a per-principal rate limit and `cache-control: no-store` are built in; the zone server still - enforces signature, zone, expiry and single use. `js/tests` reproduce + enforces signature, zone, expiry and single use. Extension claims (match + id, team…) come only from the server-side `claims(context)` callback and + are signed under `ext`. `js/tests` reproduce `tests/fixtures/sveltekit_tickets.json` byte for byte and `net_e2e` admits those tickets on a real `GameServer` (and refuses tampered, expired, wrong-zone and replayed ones). @@ -913,6 +915,20 @@ principals commit concurrently; settlements arrive in completion order. enforces route zone, expiry and single use; `TicketClaims::name` becomes `PlayerInfo::name` / `ObserverInfo::name` and travels with handoffs. Transfer tickets always use `ticket_key`. +- **Extension claims**: game-declared claims the issuer vouches for (match + id, team, party) travel in the ticket's `ext` object. Declare them once — + `#[derive(Serialize, Deserialize, TicketExtension)] struct MatchClaims { + match_id: String, team: u8 }` — and admit them with + `ServerConfig::with_ticket_extensions::()`. The derive + refuses identity-, reward- and economy-looking names (the `FactCommand` + deny list, shared with the JS handler through + `tests/fixtures/trusted_field_denylist.json`); the server refuses tickets + whose `ext` has undeclared names or wrong types, and any `ext` when no + schema is declared (join tickets from any verifier and transfer tickets + alike). Zones read them typed in `on_auth` + (`req.claims.extensions.typed::()`) and afterwards from + `PlayerInfo::extensions`; a rejoin replaces them with the fresh ticket's + and handoffs carry them to the next zone. - **Input pacing**: `InputCredits::new(max)` gives each player one input credit per tick (banking up to `max` for jitter bursts); `credits.take(ctx, player)` consumes queued inputs while credits last, so diff --git a/distributed_game/src/bots.rs b/distributed_game/src/bots.rs index 3cf03bce3..8f76e9a8e 100644 --- a/distributed_game/src/bots.rs +++ b/distributed_game/src/bots.rs @@ -362,6 +362,7 @@ async fn drive( nonce: format!("{principal}-{}-{}", config.seed, unix), role: Role::Player, name: None, + extensions: Default::default(), }); let url = format!("{}/zones/{}", config.url.trim_end_matches('/'), config.zone); let (mut ws, _) = tokio_tungstenite::connect_async(url.as_str()) diff --git a/distributed_game/src/claims.rs b/distributed_game/src/claims.rs new file mode 100644 index 000000000..197b66d16 --- /dev/null +++ b/distributed_game/src/claims.rs @@ -0,0 +1,247 @@ +//! Ticket extension claims: game-declared facts the ticket issuer vouches +//! for (match id, team, party, mode), carried next to the fixed +//! [`TicketClaims`](crate::TicketClaims) fields. +//! +//! The issuer (the web backend, a matchmaker) signs them under the ticket's +//! `ext` object; the zone reads them from +//! [`TicketClaims::extensions`](crate::TicketClaims::extensions) in +//! [`Zone::on_auth`](crate::Zone::on_auth) and from +//! [`PlayerInfo::extensions`](crate::PlayerInfo::extensions) afterwards, typed +//! with [`TicketExtensions::typed`]. Handoff transfer tickets carry them to +//! the next zone unchanged. +//! +//! A game declares the schema once with `#[derive(TicketExtension)]`: +//! +//! ```rust,ignore +//! #[derive(Serialize, Deserialize, TicketExtension)] +//! pub struct MatchClaims { pub match_id: String, pub team: u8 } +//! // net: ServerConfig { ticket_extensions: Some(ExtensionSchema::of::()), .. } +//! ``` +//! +//! The derive refuses identity-, reward- and economy-looking names (the +//! same deny list as `FactCommand`): identity is only ever the verified +//! `sub`, and amounts are decided by command handlers. The `net` server +//! refuses a ticket whose extensions do not match the declared schema +//! (unknown names, wrong types), and any ticket with extensions when no +//! schema is declared. + +use std::collections::BTreeMap; +use std::fmt; + +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::{Map, Value}; + +/// Largest accepted extension object, as compact JSON (bytes). +pub const MAX_EXTENSION_BYTES: usize = 2048; + +/// A game-declared extension claims type (`#[derive(TicketExtension)]`). +pub trait TicketExtension: Serialize + DeserializeOwned { + /// Claim names as serialized (serde renames applied). + const CLAIMS: &'static [&'static str]; +} + +/// Why extension claims were refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExtensionError { + /// `ext` is not a JSON object. + NotAnObject, + /// Larger than [`MAX_EXTENSION_BYTES`]. + TooLarge, + /// A name the declared schema does not know. + Unknown(String), + /// Extensions present but the server declares no schema. + Undeclared, + /// The values do not decode as the declared type. + Invalid(String), +} + +impl fmt::Display for ExtensionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotAnObject => f.write_str("extension claims must be an object"), + Self::TooLarge => write!(f, "extension claims exceed {MAX_EXTENSION_BYTES} bytes"), + Self::Unknown(name) => write!(f, "undeclared extension claim `{name}`"), + Self::Undeclared => f.write_str("this server declares no ticket extension claims"), + Self::Invalid(why) => write!(f, "invalid extension claims: {why}"), + } + } +} + +impl std::error::Error for ExtensionError {} + +/// Verified extension claims (name → JSON value), empty for a ticket +/// without any. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct TicketExtensions(BTreeMap); + +impl TicketExtensions { + /// No extensions. + pub fn new() -> Self { + Self::default() + } + + /// From a typed declaration. + pub fn from_typed(claims: &E) -> Result { + let value = + serde_json::to_value(claims).map_err(|e| ExtensionError::Invalid(e.to_string()))?; + Self::from_json(value) + } + + /// From a JSON object (the ticket's `ext`), size-checked. + pub fn from_json(value: Value) -> Result { + let Value::Object(map) = value else { + return Err(ExtensionError::NotAnObject); + }; + let extensions = Self(map.into_iter().collect()); + if extensions.encoded_len() > MAX_EXTENSION_BYTES { + return Err(ExtensionError::TooLarge); + } + Ok(extensions) + } + + /// As a JSON object (what the ticket signs under `ext`). + pub fn to_json(&self) -> Value { + Value::Object(self.0.clone().into_iter().collect::>()) + } + + fn encoded_len(&self) -> usize { + self.to_json().to_string().len() + } + + /// True when the ticket carried none. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// One raw claim. + pub fn get(&self, name: &str) -> Option<&Value> { + self.0.get(name) + } + + /// All claims, by name. + pub fn iter(&self) -> impl Iterator { + self.0.iter().map(|(k, v)| (k.as_str(), v)) + } + + /// Decode as the declared type. Names `E` does not declare are refused. + pub fn typed(&self) -> Result { + if let Some(unknown) = self + .0 + .keys() + .find(|name| !E::CLAIMS.contains(&name.as_str())) + { + return Err(ExtensionError::Unknown(unknown.clone())); + } + serde_json::from_value(self.to_json()).map_err(|e| ExtensionError::Invalid(e.to_string())) + } +} + +/// The extension schema a server admits ([`ExtensionSchema::of`]). +#[derive(Clone, Copy)] +pub struct ExtensionSchema { + claims: &'static [&'static str], + check: fn(&TicketExtensions) -> Result<(), ExtensionError>, +} + +impl fmt::Debug for ExtensionSchema { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExtensionSchema") + .field("claims", &self.claims) + .finish() + } +} + +impl ExtensionSchema { + /// Admit extensions that decode as `E` (missing optional fields are + /// fine; unknown names are not). + pub fn of() -> Self { + Self { + claims: E::CLAIMS, + check: |ext| ext.typed::().map(drop), + } + } + + /// Declared claim names. + pub fn claims(&self) -> &'static [&'static str] { + self.claims + } + + /// Check verified extensions against the schema. + pub fn validate(&self, extensions: &TicketExtensions) -> Result<(), ExtensionError> { + if extensions.is_empty() { + return Ok(()); + } + (self.check)(extensions) + } +} + +/// Check extensions against an optional server schema: none declared means +/// none admitted. +pub fn validate_extensions( + schema: Option<&ExtensionSchema>, + extensions: &TicketExtensions, +) -> Result<(), ExtensionError> { + match schema { + Some(schema) => schema.validate(extensions), + None if extensions.is_empty() => Ok(()), + None => Err(ExtensionError::Undeclared), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + use serde_json::json; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct MatchClaims { + match_id: String, + #[serde(default)] + team: Option, + } + + impl TicketExtension for MatchClaims { + const CLAIMS: &'static [&'static str] = &["match_id", "team"]; + } + + #[test] + fn typed_round_trip_and_schema_checks() { + let claims = MatchClaims { + match_id: "m-1".into(), + team: Some(2), + }; + let ext = TicketExtensions::from_typed(&claims).unwrap(); + assert_eq!(ext.get("team"), Some(&json!(2))); + assert_eq!(ext.typed::().unwrap(), claims); + let schema = ExtensionSchema::of::(); + schema.validate(&ext).unwrap(); + schema.validate(&TicketExtensions::new()).unwrap(); + + let unknown = TicketExtensions::from_json(json!({"match_id": "m", "gold": 9})).unwrap(); + assert_eq!( + schema.validate(&unknown), + Err(ExtensionError::Unknown("gold".into())) + ); + let mistyped = TicketExtensions::from_json(json!({"match_id": 7})).unwrap(); + assert!(matches!( + schema.validate(&mistyped), + Err(ExtensionError::Invalid(_)) + )); + assert_eq!( + TicketExtensions::from_json(json!(["x"])), + Err(ExtensionError::NotAnObject) + ); + let big = "x".repeat(MAX_EXTENSION_BYTES); + assert_eq!( + TicketExtensions::from_json(json!({ "match_id": big })), + Err(ExtensionError::TooLarge) + ); + assert_eq!( + validate_extensions(None, &ext), + Err(ExtensionError::Undeclared) + ); + validate_extensions(None, &TicketExtensions::new()).unwrap(); + } +} diff --git a/distributed_game/src/handoff.rs b/distributed_game/src/handoff.rs index d50bcaf8a..72ae4dfc9 100644 --- a/distributed_game/src/handoff.rs +++ b/distributed_game/src/handoff.rs @@ -9,6 +9,7 @@ //! calls [`Zone::on_handoff_join`](crate::Zone::on_handoff_join) with the //! carried bytes. +use crate::claims::TicketExtensions; use crate::zone::TicketClaims; /// Largest carried state accepted in a handoff (it travels inside the @@ -22,6 +23,8 @@ pub struct HandoffRequest { pub principal: String, /// The player's display name (from its admission ticket), carried over. pub name: Option, + /// The player's ticket extension claims, carried over. + pub extensions: TicketExtensions, /// Zone the player leaves. pub source_zone: String, /// Zone the player goes to. diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index 7ccb1f6fd..66f44177f 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -645,11 +645,17 @@ impl Ctx<'_, Z> { if carried.len() > MAX_CARRIED_BYTES { return Err(HandoffError::TooLarge); } - let (principal, name) = self + let (principal, name, extensions) = self .core .players .get(&player) - .map(|slot| (slot.info.principal.clone(), slot.info.name.clone())) + .map(|slot| { + ( + slot.info.principal.clone(), + slot.info.name.clone(), + slot.info.extensions.clone(), + ) + }) .ok_or(HandoffError::UnknownPlayer)?; if self .core @@ -663,6 +669,7 @@ impl Ctx<'_, Z> { let request = HandoffRequest { principal, name, + extensions, source_zone: self.core.zone_id.clone(), target_zone: target_zone.to_string(), carried, @@ -1056,6 +1063,7 @@ impl ZoneHost { if claims.name.is_some() { slot.info.name = claims.name.clone(); } + slot.info.extensions = claims.extensions.clone(); } self.bind_player(player, client, capabilities); if let Some((from, carried)) = handoff.take() { @@ -1079,6 +1087,7 @@ impl ZoneHost { id: player, principal: claims.principal.clone(), name: claims.name.clone(), + extensions: claims.extensions.clone(), capabilities, connected: true, }, @@ -1111,6 +1120,7 @@ impl ZoneHost { id: observer, principal: claims.principal.clone(), name: claims.name.clone(), + extensions: claims.extensions.clone(), capabilities, }, client, diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index a5670bdbe..c9f18ba93 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -54,6 +54,7 @@ pub mod authoring; pub mod bots; #[cfg(feature = "cell")] pub mod cell; +pub mod claims; pub mod clock; pub mod codec; pub mod facts; @@ -81,11 +82,12 @@ pub mod world; pub mod zone; pub use authoring::{GameEvent, GameInput, Pushed, Replicated}; +pub use claims::{ExtensionError, ExtensionSchema, TicketExtension, TicketExtensions}; pub use clock::{Clock, ManualClock, SystemClock}; pub use codec::{Wire, WireType}; /// Authoring derives (see the crate docs and README). pub use distributed_game_macros::{ - game_pure, predict, FactCommand, GameEvent, GameInput, GameRecord, Replicated, + game_pure, predict, FactCommand, GameEvent, GameInput, GameRecord, Replicated, TicketExtension, }; #[cfg(feature = "runtime")] pub use facts::spawn_fact_worker; diff --git a/distributed_game/src/metrics.rs b/distributed_game/src/metrics.rs index 00595189d..f3e09e828 100644 --- a/distributed_game/src/metrics.rs +++ b/distributed_game/src/metrics.rs @@ -641,6 +641,7 @@ mod tests { nonce: "n".into(), role: Role::Player, name: None, + extensions: Default::default(), }; let joined = h.join(&claims, 0, "").unwrap(); metrics.record_input("z1", h.push_input(joined.client, 1, 0, &[1])); diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index f65a0f080..8271e95cb 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -174,6 +174,12 @@ pub struct ServerConfig { /// `None` = HS256 tickets signed with `ticket_key`. Expiry, route zone /// and single use are enforced by the server either way. pub ticket_verifier: Option, + /// Extension claims tickets may carry + /// ([`ExtensionSchema::of::()`](crate::ExtensionSchema::of) for a + /// `#[derive(TicketExtension)]` type). `None` = a ticket with any `ext` + /// claims is refused. Applies to join (any verifier) and transfer + /// tickets. + pub ticket_extensions: Option, /// Deadline for the first (Hello) frame. pub hello_timeout: Duration, /// Sustained inputs per second per connection. @@ -251,12 +257,19 @@ impl ServerConfig { self } + /// Admit ticket extension claims of type `E`. + pub fn with_ticket_extensions(mut self) -> Self { + self.ticket_extensions = Some(crate::claims::ExtensionSchema::of::()); + self + } + /// Defaults with the given ticket key. pub fn new(ticket_key: TicketKey) -> Self { Self { host: HostConfig::default(), ticket_key, ticket_verifier: None, + ticket_extensions: None, hello_timeout: Duration::from_secs(5), input_rate: 120.0, input_burst: 240.0, @@ -1076,6 +1089,7 @@ async fn run_zone( nonce, role: Role::Player, name: request.name.clone(), + extensions: request.extensions.clone(), }, source_zone: request.source_zone.clone(), issuer: inner.host_id.clone(), @@ -1836,6 +1850,12 @@ fn verify_hello_ticket( return Err(TicketError::Expired); } } + let extensions = match &verified { + VerifiedTicket::Join(claims) => &claims.extensions, + VerifiedTicket::Transfer(transfer) => &transfer.claims.extensions, + }; + crate::claims::validate_extensions(cfg.ticket_extensions.as_ref(), extensions) + .map_err(|e| TicketError::Invalid(e.to_string()))?; Ok(verified) } diff --git a/distributed_game/src/net/ticket.rs b/distributed_game/src/net/ticket.rs index c2ef15e75..33978b72b 100644 --- a/distributed_game/src/net/ticket.rs +++ b/distributed_game/src/net/ticket.rs @@ -14,6 +14,11 @@ //! [`TicketKey::verify`] rejects them ([`TicketError::WrongKind`]); use //! [`TicketKey::verify_ticket`] to accept either kind. //! +//! **Extension claims.** An optional `ext` object carries game-declared +//! claims ([`TicketExtensions`]: match id, team…). Join and transfer +//! tickets sign them alike; the server admits them only when they match +//! `ServerConfig::ticket_extensions` (see [`crate::claims`]). +//! //! **Custom admission tickets.** Join tickets go through a //! [`TicketVerifier`] (`ServerConfig::ticket_verifier`); the default is the //! `TicketKey` itself. A game whose web backend already mints its own tickets @@ -33,6 +38,7 @@ use hmac::{Hmac, Mac}; use serde_json::{json, Value}; use sha2::Sha256; +use crate::claims::TicketExtensions; use crate::handoff::Transfer; use crate::protocol::Role; use crate::zone::TicketClaims; @@ -323,6 +329,11 @@ impl TicketKey { .and_then(Value::as_str) .filter(|v| !v.is_empty()) .map(str::to_string), + extensions: match claims.get("ext") { + None => TicketExtensions::new(), + Some(ext) => TicketExtensions::from_json(ext.clone()) + .map_err(|e| TicketError::Invalid(e.to_string()))?, + }, }, claims, )) @@ -330,8 +341,13 @@ impl TicketKey { } fn with_name(mut body: Value, claims: &TicketClaims) -> Value { - if let (Some(name), Some(object)) = (&claims.name, body.as_object_mut()) { - object.insert("name".into(), Value::String(name.clone())); + if let Some(object) = body.as_object_mut() { + if let Some(name) = &claims.name { + object.insert("name".into(), Value::String(name.clone())); + } + if !claims.extensions.is_empty() { + object.insert("ext".into(), claims.extensions.to_json()); + } } body } @@ -382,6 +398,7 @@ mod tests { nonce: "n1".into(), role: Role::Player, name: None, + extensions: Default::default(), } } diff --git a/distributed_game/src/replay.rs b/distributed_game/src/replay.rs index b9cd61db4..6b6166894 100644 --- a/distributed_game/src/replay.rs +++ b/distributed_game/src/replay.rs @@ -579,6 +579,7 @@ fn put_claims(w: &mut Writer, claims: &TicketClaims) { Some(name) => w.u8(1).string(name), None => w.u8(0), }; + w.string(&claims.extensions.to_json().to_string()); } fn get_claims(r: &mut Reader<'_>) -> Result { @@ -594,6 +595,10 @@ fn get_claims(r: &mut Reader<'_>) -> Result { 0 => None, _ => Some(r.string()?), }; + let extensions = serde_json::from_str(&r.string()?) + .ok() + .and_then(|value| crate::claims::TicketExtensions::from_json(value).ok()) + .ok_or_else(|| bad("extension claims"))?; Ok(TicketClaims { principal, zone, @@ -601,6 +606,7 @@ fn get_claims(r: &mut Reader<'_>) -> Result { nonce, role, name, + extensions, }) } diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 166a06ebc..7a52139b2 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -10,6 +10,7 @@ //! `on_tick`/`on_timer`/`on_fact_settled` interleaved → `on_leave` / //! `on_unobserve` → `on_dispose`. +use crate::claims::TicketExtensions; use crate::facts::FactSettlement; use crate::host::Ctx; use crate::protocol::Role; @@ -74,6 +75,10 @@ pub struct TicketClaims { /// identity: use `principal` for that). Exposed as /// [`PlayerInfo::name`] / [`ObserverInfo::name`]. pub name: Option, + /// Game-declared claims the issuer vouches for (match id, team…; the + /// ticket's `ext`), checked by the `net` server against + /// `ServerConfig::ticket_extensions`. Never identity or rewards. + pub extensions: TicketExtensions, } impl TicketClaims { @@ -92,9 +97,16 @@ impl TicketClaims { nonce: nonce.into(), role, name: None, + extensions: TicketExtensions::new(), } } + /// Attach extension claims. + pub fn with_extensions(mut self, extensions: TicketExtensions) -> Self { + self.extensions = extensions; + self + } + /// Attach a display name. pub fn with_name(mut self, name: impl Into) -> Self { self.name = Some(name.into()); @@ -154,6 +166,9 @@ pub struct PlayerInfo { pub principal: String, /// Display name from the ticket ([`TicketClaims::name`]), if any. pub name: Option, + /// Extension claims from the ticket ([`TicketClaims::extensions`]); + /// replaced by a fresh ticket's on rejoin, carried across handoffs. + pub extensions: TicketExtensions, /// Hello capabilities. pub capabilities: u32, /// False while inside the reconnect grace window. @@ -169,6 +184,8 @@ pub struct ObserverInfo { pub principal: String, /// Display name from the ticket, if any. pub name: Option, + /// Extension claims from the ticket. + pub extensions: TicketExtensions, /// Hello capabilities. pub capabilities: u32, } diff --git a/distributed_game/tests/fixtures/sveltekit_tickets.json b/distributed_game/tests/fixtures/sveltekit_tickets.json index ca6fbe151..3c231db41 100644 --- a/distributed_game/tests/fixtures/sveltekit_tickets.json +++ b/distributed_game/tests/fixtures/sveltekit_tickets.json @@ -12,6 +12,14 @@ "ticket": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMiIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLW9ic2VydmVyIiwicm9sZSI6Im9ic2VydmVyIn0.1j2Ql0XePnslZ1kJf64VhjUeDYkdl_nAt6Vep2zQ1bs", "principal": "session-user-2" }, + "extended": { + "ticket": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMyIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLWV4dGVuZGVkIiwicm9sZSI6InBsYXllciIsImV4dCI6eyJtYXRjaF9pZCI6Im0tNyIsInRlYW0iOjJ9fQ.sQwdQFAEEsYZcCy2qB3szKm8bIBRWqmngIdE-FGwfwo", + "principal": "session-user-3", + "ext": { + "match_id": "m-7", + "team": 2 + } + }, "expired": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMSIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6MzAwLCJub25jZSI6ImZpeHR1cmUtZXhwaXJlZCIsInJvbGUiOiJwbGF5ZXIifQ.pJED6dt-O-JNtFi-fqzUFNPtEWYljUs0pKR9gJrX380", "wrongZone": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMSIsInpvbmUiOiJ6LWVsc2V3aGVyZSIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLXdyb25nLXpvbmUiLCJyb2xlIjoicGxheWVyIn0.s4mdpfnnVke8Ntt_LjHAgonARO5RRt_KVgKH5v96k7A", "tampered": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhZG1pbiIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLXRhbXBlcmVkIiwicm9sZSI6InBsYXllciJ9.k0UNks88hcYz9M18yZdUA2OPH-Xwgf6jBwi6snvkoJE", diff --git a/distributed_game/tests/fixtures/trusted_field_denylist.json b/distributed_game/tests/fixtures/trusted_field_denylist.json new file mode 100644 index 000000000..36b210253 --- /dev/null +++ b/distributed_game/tests/fixtures/trusted_field_denylist.json @@ -0,0 +1,6 @@ +{ + "comment": "Names a zone fact (FactCommand) or a ticket extension claim (TicketExtension, createGameTicketHandler claims) must never use. Matching ignores '_', '-' and case. Checked against distributed_game_macros/src/trusted.rs and js/src/sveltekit/game/ticket.ts by their tests.", + "identity": ["principal", "user", "user_id", "owner", "owner_id", "account", "account_id", "subject", "sub"], + "reward": ["xp", "experience", "reward", "rewards"], + "economy": ["amount", "coins", "gold", "currency", "price", "cost", "balance", "grant", "payout", "credit", "debit"] +} diff --git a/distributed_game/tests/net_e2e/main.rs b/distributed_game/tests/net_e2e/main.rs index f4f1c13b6..48ee34e5a 100644 --- a/distributed_game/tests/net_e2e/main.rs +++ b/distributed_game/tests/net_e2e/main.rs @@ -11,9 +11,10 @@ use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; use distributed_game::protocol::{error, kick, Hello}; use distributed_game::{ ClientMessage, ClientMirror, FieldValue, MemoryFactSink, RetryPolicy, Role, ServerMessage, - TicketClaims, PROTOCOL_VERSION, + TicketClaims, TicketExtension, TicketExtensions, PROTOCOL_VERSION, }; use futures_util::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; @@ -30,6 +31,7 @@ fn ticket(principal: &str, zone: &str, nonce: &str, role: Role) -> String { nonce: nonce.into(), role, name: None, + extensions: Default::default(), }) } @@ -228,6 +230,7 @@ async fn client_joins_sends_inputs_receives_deltas_and_reconnects() { nonce: "n4".into(), role: Role::Player, name: None, + extensions: Default::default(), }); expect_error(&url, "z1", forged, error::UNAUTHORIZED).await; @@ -290,6 +293,21 @@ async fn admits_tickets_minted_by_the_sveltekit_handler() { assert_eq!(claims.nonce, text("/valid/nonce")); assert_eq!(claims.role, Role::Player); assert_eq!(claims.name.as_deref(), Some("Ana")); + assert!(claims.extensions.is_empty()); + + // Extension claims from the handler's server-side `claims()` callback. + let extended = TicketKey::new(secret.as_str()) + .verify(&text("/extended/ticket"), 1_700_000_000) + .unwrap(); + assert_eq!(extended.principal, text("/extended/principal")); + assert_eq!( + extended.extensions.typed::().unwrap(), + MatchClaims { + match_id: "m-7".into(), + side: 2, + party: None, + } + ); let (url, server, handle) = start_keyed(&secret, None).await; @@ -639,3 +657,123 @@ async fn host_config_hook_sets_per_zone_rates() { assert_eq!(hot, Some(30)); server.shutdown(); } + +/// A game's declared extension claims. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TicketExtension)] +struct MatchClaims { + match_id: String, + #[serde(rename = "team")] + side: u8, + #[serde(default)] + party: Option, +} + +fn extended_ticket(nonce: &str, ext: serde_json::Value) -> String { + TicketKey::new(SECRET).sign(&TicketClaims { + principal: "ana".into(), + zone: "z-match".into(), + expires_at: 4_000_000_000, + nonce: nonce.into(), + role: Role::Player, + name: None, + extensions: TicketExtensions::from_json(ext).unwrap(), + }) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn extension_claims_reach_on_auth_and_player_info() { + assert_eq!(MatchClaims::CLAIMS, ["match_id", "team", "party"]); + let config = ServerConfig::new(TicketKey::new(SECRET)).with_ticket_extensions::(); + let server = GameServer::::new(config, |_| Some(ArenaConfig::default())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}", listener.local_addr().unwrap()); + let serving = server.clone(); + let handle = tokio::spawn(async move { + serving + .serve(listener, std::future::pending()) + .await + .unwrap(); + }); + + // Undeclared names and mistyped values are refused at admission. + for (nonce, ext) in [ + ( + "x-unknown", + serde_json::json!({"match_id": "m-7", "team": 2, "gold": 50}), + ), + ( + "x-typed", + serde_json::json!({"match_id": "m-7", "team": "red"}), + ), + ("x-object", serde_json::json!(["m-7"])), + ] { + let ticket = if nonce == "x-object" { + TicketKey::new(SECRET).sign_json(&serde_json::json!({ + "sub": "ana", "zone": "z-match", "exp": 4_000_000_000u64, + "nonce": nonce, "ext": ext, + })) + } else { + extended_ticket(nonce, ext) + }; + expect_error(&url, "z-match", ticket, error::UNAUTHORIZED).await; + } + + let expected = MatchClaims { + match_id: "m-7".into(), + side: 2, + party: None, + }; + let mut ws = connect(&url, "z-match").await; + hello( + &mut ws, + extended_ticket("ok", serde_json::json!({"match_id": "m-7", "team": 2})), + String::new(), + ) + .await; + let Some(ServerMessage::Welcome(_)) = next(&mut ws, &mut ClientMirror::new()).await else { + panic!("expected Welcome") + }; + let (auth, players) = server + .inspect("z-match", |host| { + ( + host.zone().auth_extensions.clone(), + host.players().cloned().collect::>(), + ) + }) + .await + .expect("zone active"); + assert_eq!( + auth.last().unwrap().typed::().unwrap(), + expected + ); + assert_eq!( + players[0].extensions.typed::().unwrap(), + expected + ); + + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("server stops") + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn extension_claims_are_refused_without_a_declared_schema() { + let (url, server, handle) = start(None).await; + let ticket = TicketKey::new(SECRET).sign(&TicketClaims { + principal: "ana".into(), + zone: "z1".into(), + expires_at: 4_000_000_000, + nonce: "undeclared".into(), + role: Role::Player, + name: None, + extensions: TicketExtensions::from_json(serde_json::json!({"match_id": "m"})).unwrap(), + }); + expect_error(&url, "z1", ticket, error::UNAUTHORIZED).await; + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("server stops") + .unwrap(); +} diff --git a/distributed_game/tests/netcode/main.rs b/distributed_game/tests/netcode/main.rs index 5d3711f68..7a4a985c8 100644 --- a/distributed_game/tests/netcode/main.rs +++ b/distributed_game/tests/netcode/main.rs @@ -24,6 +24,7 @@ fn claims(principal: &str, zone: &str) -> TicketClaims { nonce: format!("n-{principal}"), role: Role::Player, name: None, + extensions: Default::default(), } } diff --git a/distributed_game/tests/world/main.rs b/distributed_game/tests/world/main.rs index b5b0ff1cf..6c4c2cb73 100644 --- a/distributed_game/tests/world/main.rs +++ b/distributed_game/tests/world/main.rs @@ -36,6 +36,7 @@ fn ticket(principal: &str, zone: &str, nonce: &str) -> String { nonce: nonce.into(), role: Role::Player, name: None, + extensions: Default::default(), }) } @@ -327,6 +328,7 @@ async fn cross_host_handoff_follows_redirect_and_ticket_is_single_use() { nonce: "x".into(), role: Role::Player, name: None, + extensions: Default::default(), }); assert!(join_url(&b.addr.zone_url("realm.west"), &forged, "") .await @@ -1055,3 +1057,63 @@ fn hibernation_outcomes(server: &GameServer) -> u64 { let m = server.metrics(); m.hibernation_saves("saved") + m.hibernation_saves("stale") + m.hibernation_saves("error") } + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize, TicketExtension)] +struct PartyClaims { + party: String, + team: u8, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn transfer_tickets_carry_extension_claims_across_hosts() { + let (a, b, _dir) = two_hosts( + || base_config().with_ticket_extensions::(), + &[("realm.east", "A"), ("realm.west", "B")], + realm_factory(&[("realm.east", "realm.west")], &[]), + Arc::new(LocalOwnership::new()), + ) + .await; + let party = PartyClaims { + party: "p-9".into(), + team: 1, + }; + let t = TicketKey::new(SECRET).sign( + &TicketClaims::new("ana", "realm.east", 4_000_000_000, "ext-1", Role::Player) + .with_extensions(TicketExtensions::from_typed(&party).unwrap()), + ); + let mut ana = join_url(&a.addr.zone_url("realm.east"), &t, "") + .await + .unwrap(); + input(&mut ana, 1, CMD_HANDOFF).await; + let transfer = loop { + match control(&mut ana).await { + Some(ServerMessage::Handoff { + transfer_ticket, .. + }) => break transfer_ticket, + Some(_) => continue, + None => panic!("closed before Handoff"), + } + }; + // The server-minted transfer ticket signs the claims... + let distributed_game::net::VerifiedTicket::Transfer(verified) = TicketKey::new(SECRET) + .verify_ticket(&transfer, 1_700_000_000) + .unwrap() + else { + panic!("transfer ticket") + }; + assert_eq!( + verified.claims.extensions.typed::().unwrap(), + party + ); + // ...and the target zone on the other host sees them. + let _west = join_url(&a.addr.zone_url("realm.west"), &transfer, "") + .await + .unwrap(); + let players = b + .server + .inspect("realm.west", |h| h.players().cloned().collect::>()) + .await + .unwrap(); + assert_eq!(players[0].principal, "ana"); + assert_eq!(players[0].extensions.typed::().unwrap(), party); +} diff --git a/distributed_game/tests/world_host/main.rs b/distributed_game/tests/world_host/main.rs index 92650bbd3..d85e3b950 100644 --- a/distributed_game/tests/world_host/main.rs +++ b/distributed_game/tests/world_host/main.rs @@ -20,6 +20,7 @@ fn claims(principal: &str, zone: &str) -> TicketClaims { nonce: format!("n-{principal}"), role: Role::Player, name: None, + extensions: Default::default(), } } diff --git a/distributed_game/tests/zone_runtime/arena.rs b/distributed_game/tests/zone_runtime/arena.rs index d3805d52e..9fdc6454d 100644 --- a/distributed_game/tests/zone_runtime/arena.rs +++ b/distributed_game/tests/zone_runtime/arena.rs @@ -47,6 +47,8 @@ pub struct Arena { pub ticks: u64, pub timer_fires: u64, pub tiers: Vec<(Tier, Tier)>, + /// Extension claims seen by `on_auth`. + pub auth_extensions: Vec, } impl Arena { @@ -114,10 +116,12 @@ impl Zone for Arena { ticks: 0, timer_fires: 0, tiers: Vec::new(), + auth_extensions: Vec::new(), } } fn on_auth(&mut self, _ctx: &mut Ctx<'_, Self>, req: &JoinRequest<'_>) -> Admission { + self.auth_extensions.push(req.claims.extensions.clone()); if self.cfg.reject_principal.as_deref() == Some(req.claims.principal.as_str()) { return Admission::Reject("banned".into()); } diff --git a/distributed_game/tests/zone_runtime/main.rs b/distributed_game/tests/zone_runtime/main.rs index 6bd1bd5c4..7d4d6f5df 100644 --- a/distributed_game/tests/zone_runtime/main.rs +++ b/distributed_game/tests/zone_runtime/main.rs @@ -25,6 +25,7 @@ fn claims(principal: &str, zone: &str) -> TicketClaims { nonce: format!("n-{principal}"), role: Role::Player, name: None, + extensions: Default::default(), } } diff --git a/distributed_game_macros/src/claims.rs b/distributed_game_macros/src/claims.rs new file mode 100644 index 000000000..fd489190e --- /dev/null +++ b/distributed_game_macros/src/claims.rs @@ -0,0 +1,69 @@ +//! `#[derive(TicketExtension)]`: declare the extension claims a ticket +//! issuer may vouch for (match id, team, party…). + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{DeriveInput, LitStr}; + +use crate::shared::{field_name, named_fields, no_generics, runtime}; +use crate::trusted::reject_forbidden; + +pub(crate) fn expand(input: DeriveInput) -> syn::Result { + no_generics(&input, "TicketExtension")?; + let rt = runtime()?; + // Container-wide renames would change the names the schema admits + // behind the deny list's back. + for attr in input.attrs.iter().filter(|a| a.path().is_ident("serde")) { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("rename_all") || meta.path.is_ident("rename_all_fields") { + return Err(meta.error( + "#[derive(TicketExtension)] does not support `rename_all`; rename fields individually", + )); + } + if meta.input.peek(syn::Token![=]) { + let _ = meta.value()?.parse::()?; + } else if meta.input.peek(syn::token::Paren) { + meta.parse_nested_meta(|_| Ok(()))?; + } + Ok(()) + })?; + } + let fields = named_fields(&input, "TicketExtension")?; + reject_forbidden( + &fields, + "ticket claim", + "identity is only ever the verified `sub`, and rewards and economy values are decided by command handlers, never vouched for in a ticket", + )?; + let mut names = Vec::new(); + for field in &fields { + let mut name = field_name(field.ident.as_ref().expect("named")); + for attr in field.attrs.iter().filter(|a| a.path().is_ident("serde")) { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("flatten") { + return Err(meta.error( + "#[derive(TicketExtension)] does not support `flatten`; declare every claim as a field", + )); + } + if meta.path.is_ident("rename") { + if meta.input.peek(syn::Token![=]) { + name = meta.value()?.parse::()?.value(); + } else { + return Err(meta.error("use `rename = \"..\"` (one name for both directions)")); + } + } else if meta.input.peek(syn::Token![=]) { + let _ = meta.value()?.parse::()?; + } else if meta.input.peek(syn::token::Paren) { + meta.parse_nested_meta(|_| Ok(()))?; + } + Ok(()) + })?; + } + names.push(name); + } + let ident = &input.ident; + Ok(quote! { + impl #rt::TicketExtension for #ident { + const CLAIMS: &'static [&'static str] = &[#(#names),*]; + } + }) +} diff --git a/distributed_game_macros/src/lib.rs b/distributed_game_macros/src/lib.rs index 9b3487054..55c41b630 100644 --- a/distributed_game_macros/src/lib.rs +++ b/distributed_game_macros/src/lib.rs @@ -9,6 +9,7 @@ //! | [`GameEvent`] | struct / enum | `GameEvent`: stable ids, typed payload codec | //! | [`GameRecord`] | struct | `Wire`: a nested payload record | //! | [`FactCommand`] | struct | `FactCommand`: the cell command name of a fact input | +//! | [`TicketExtension`] | struct | `TicketExtension`: the extension claims a ticket may carry | //! | [`macro@predict`] (attribute) | fn | `PredictStep` for a client-predicted step + its WebAssembly export | //! | [`macro@game_pure`] (attribute) | fn | `GamePure` for a pure function compiled to WebAssembly + its export | //! @@ -24,6 +25,7 @@ //! - GameInput only: `#[range(min = A, max = B)]` (clamp), `#[buttons(a, b, ..)]` //! (bit flags, unknown bits cleared; also generates `Type::A`, `Type::B` consts). +mod claims; mod event; mod fact; mod input; @@ -91,6 +93,16 @@ pub fn derive_fact_command(input: TokenStream) -> TokenStream { finish(fact::expand(parse_macro_input!(input as DeriveInput))) } +/// Declare a game's ticket extension claims (match id, team, party…): +/// a struct with named fields that also derives serde `Serialize` and +/// `Deserialize`. Generates `TicketExtension::CLAIMS` (serialized names). +/// Identity-, reward- and economy-looking names are rejected with the same +/// deny list as [`FactCommand`]; `rename_all` and `flatten` are refused. +#[proc_macro_derive(TicketExtension)] +pub fn derive_ticket_extension(input: TokenStream) -> TokenStream { + finish(claims::expand(parse_macro_input!(input as DeriveInput))) +} + /// Declare a zone's client-predicted step: /// `#[predict(ArenaStep, kind = "player")]` on /// `fn(state: &State, input: &Input, context: &Context, dt: f32) -> State`. diff --git a/distributed_game_macros/src/trusted.rs b/distributed_game_macros/src/trusted.rs index e4930c23c..a7a23df83 100644 --- a/distributed_game_macros/src/trusted.rs +++ b/distributed_game_macros/src/trusted.rs @@ -103,7 +103,29 @@ pub(crate) fn reject_forbidden(fields: &[&Field], what: &str, rule: &str) -> syn #[cfg(test)] mod tests { - use super::forbidden; + use super::{forbidden, FORBIDDEN}; + + /// The JS ticket handler enforces the same list from this fixture. + #[test] + fn deny_list_matches_the_shared_fixture() { + let fixture = + include_str!("../../distributed_game/tests/fixtures/trusted_field_denylist.json"); + let mut expected = Vec::new(); + for (group, why) in [ + ("identity", "identity"), + ("reward", "a reward"), + ("economy", "an economy value"), + ] { + let start = fixture.find(&format!("\"{group}\": [")).expect(group); + let list = &fixture[start..]; + let list = &list[list.find('[').unwrap() + 1..list.find(']').unwrap()]; + for name in list.split(',') { + expected.push((name.trim().trim_matches('"').to_string(), why)); + } + } + let actual: Vec<_> = FORBIDDEN.iter().map(|(n, w)| (n.to_string(), *w)).collect(); + assert_eq!(actual, expected); + } #[test] fn matches_ignore_separators_and_case() { diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.rs b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.rs new file mode 100644 index 000000000..ecfdfcf84 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.rs @@ -0,0 +1,15 @@ +use distributed_game::TicketExtension; + +#[derive(serde::Serialize, serde::Deserialize)] +struct Extra { + team: u8, +} + +#[derive(serde::Serialize, serde::Deserialize, TicketExtension)] +struct MatchClaims { + match_id: String, + #[serde(flatten)] + extra: Extra, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.stderr b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.stderr new file mode 100644 index 000000000..aa6a428c9 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.stderr @@ -0,0 +1,5 @@ +error: #[derive(TicketExtension)] does not support `flatten`; declare every claim as a field + --> tests/compile_fail/ticket_extension_flatten.rs:11:13 + | +11 | #[serde(flatten)] + | ^^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_identity.rs b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.rs new file mode 100644 index 000000000..f0390da9e --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.rs @@ -0,0 +1,12 @@ +use distributed_game::TicketExtension; + +// Identity only ever comes from the verified `sub`; amounts from handlers. +#[derive(serde::Serialize, serde::Deserialize, TicketExtension)] +struct MatchClaims { + match_id: String, + userId: String, + #[serde(rename = "Gold")] + purse: u32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_identity.stderr b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.stderr new file mode 100644 index 000000000..dfcc205e9 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.stderr @@ -0,0 +1,11 @@ +error: ticket claim field `userId` looks like identity: identity is only ever the verified `sub`, and rewards and economy values are decided by command handlers, never vouched for in a ticket + --> tests/compile_fail/ticket_extension_identity.rs:7:5 + | +7 | userId: String, + | ^^^^^^ + +error: ticket claim field `Gold` looks like an economy value: identity is only ever the verified `sub`, and rewards and economy values are decided by command handlers, never vouched for in a ticket + --> tests/compile_fail/ticket_extension_identity.rs:8:22 + | +8 | #[serde(rename = "Gold")] + | ^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.rs b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.rs new file mode 100644 index 000000000..65f6992ab --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.rs @@ -0,0 +1,9 @@ +use distributed_game::TicketExtension; + +#[derive(serde::Serialize, serde::Deserialize, TicketExtension)] +#[serde(rename_all = "camelCase")] +struct MatchClaims { + match_id: String, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.stderr b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.stderr new file mode 100644 index 000000000..842049fc0 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.stderr @@ -0,0 +1,5 @@ +error: #[derive(TicketExtension)] does not support `rename_all`; rename fields individually + --> tests/compile_fail/ticket_extension_rename_all.rs:4:9 + | +4 | #[serde(rename_all = "camelCase")] + | ^^^^^^^^^^ diff --git a/examples/game/server/src/common/dev.rs b/examples/game/server/src/common/dev.rs index 23c65cb07..443d99355 100644 --- a/examples/game/server/src/common/dev.rs +++ b/examples/game/server/src/common/dev.rs @@ -171,6 +171,7 @@ async fn ticket( nonce: guests::random_hex(24), role, name: None, + extensions: Default::default(), }); Ok(Json(TicketResponse { ticket, diff --git a/examples/game/server/tests/gallery_smoke.rs b/examples/game/server/tests/gallery_smoke.rs index ad3d06932..5cec901be 100644 --- a/examples/game/server/tests/gallery_smoke.rs +++ b/examples/game/server/tests/gallery_smoke.rs @@ -41,6 +41,7 @@ impl Rig { nonce: "n1".into(), role: Role::Player, name: None, + extensions: Default::default(), }, 0, "", diff --git a/examples/game/server/tests/support/mod.rs b/examples/game/server/tests/support/mod.rs index 09b17fea0..6835a4757 100644 --- a/examples/game/server/tests/support/mod.rs +++ b/examples/game/server/tests/support/mod.rs @@ -111,6 +111,7 @@ pub fn ticket(principal: &str, zone: &str, role: Role) -> String { nonce: format!("n-{principal}-{n}"), role, name: None, + extensions: Default::default(), }) } diff --git a/js/README.md b/js/README.md index 7f88e4af6..96938d282 100644 --- a/js/README.md +++ b/js/README.md @@ -493,7 +493,15 @@ boolean or a promise of one, or `false`); 400 for any body field other than returns nothing; 503 without a valid secret. Every response is `cache-control: no-store`. Tickets live 60 s by default (`ttlSeconds` ≤ 300) and are single use on the zone server. -`mintGameTicket({ secret, principal, zone, role, name })` signs one directly. +`mintGameTicket({ secret, principal, zone, role, name, extensions })` signs one directly. + +Extension claims (match id, team, party) come from the server-side +`claims({ principal, zone, role, session, event })` callback — never from the +body — and are signed under the ticket's `ext`; the zone server admits them +only against its declared `TicketExtension` schema +(`ServerConfig::with_ticket_extensions`). Names that look like identity, +rewards or economy values (`TRUSTED_FIELD_DENYLIST`, the same list as the +Rust derives) and objects over 2 KiB are refused with 400 `invalid_claims`. The default limiter counts per process, so N instances behind a load balancer allow N× the budget. Share it through Redis (or any store that runs diff --git a/js/src/sveltekit/game/server.ts b/js/src/sveltekit/game/server.ts index 5076090c6..7fed6abad 100644 --- a/js/src/sveltekit/game/server.ts +++ b/js/src/sveltekit/game/server.ts @@ -9,6 +9,7 @@ import { MAX_GAME_TICKET_TTL_SECONDS, MIN_GAME_TICKET_SECRET_BYTES, mintGameTicket, + type GameTicketExtensions, type GameTicketRole, type GameTicketSecret } from './ticket.js'; @@ -26,9 +27,14 @@ export { } from './rate-limit.js'; export { + MAX_GAME_TICKET_EXTENSION_BYTES, MAX_GAME_TICKET_TTL_SECONDS, MIN_GAME_TICKET_SECRET_BYTES, + TRUSTED_FIELD_DENYLIST, + checkGameTicketExtensions, mintGameTicket, + type GameTicketClaimValue, + type GameTicketExtensions, type GameTicketRole, type GameTicketSecret, type MintGameTicketOptions, @@ -64,6 +70,12 @@ export type GameTicketZoneContext = Readonly<{ role: GameTicketRole; }>; +export type GameTicketClaimsContext = GameTicketZoneContext & + Readonly<{ + /** The zone the server chose (the ticket's `zone`). */ + zone: string; + }>; + export type GameTicketRateLimit = | false | Readonly<{ max: number; windowMs: number }> @@ -94,6 +106,16 @@ export type CreateGameTicketHandlerOptions ): string | null | undefined | Promise; + /** + * Extension claims for this ticket, decided on the server (match id, + * team, party from the matchmaker or the session): signed under `ext` + * and checked by the zone server against its declared + * `TicketExtension` schema. The browser body can never add claims. + * Identity-, reward- and economy-looking names are refused. + */ + claims?( + context: GameTicketClaimsContext + ): GameTicketExtensions | null | undefined | Promise; /** Roles a browser may ask for. Default both (`observer` owns no player). */ roles?: readonly GameTicketRole[]; /** Ticket lifetime in seconds (1..=300). Default 60. */ @@ -259,8 +281,9 @@ async function readBody(request: Request): Promise<{ zone?: string; role?: GameT * A SvelteKit `POST` handler that mints a `distributed_game` admission * ticket for the **verified session's** principal and the zone **the server * chooses**. The request body may only carry `{ zone?, role? }` hints - * (unknown fields → 400); identity, zone ownership and rewards are never - * accepted from it. Every response is `cache-control: no-store`. + * (unknown fields → 400); identity, zone ownership, rewards and extension + * claims are never accepted from it (extension claims come only from the + * server-side `claims()` callback). Every response is `cache-control: no-store`. * * ```ts * // src/lib/server/game.ts @@ -378,12 +401,24 @@ export function createGameTicketHandler name.replaceAll(/[_-]/g, '').toLowerCase(); +const DENIED = new Set(TRUSTED_FIELD_DENYLIST.map(normalizeName)); + +/** JSON value of an extension claim. */ +export type GameTicketClaimValue = + | string + | number + | boolean + | null + | readonly GameTicketClaimValue[] + | { readonly [key: string]: GameTicketClaimValue }; + +/** + * Game-declared extension claims (match id, team, party…) signed under the + * ticket's `ext`. The zone server admits them only when they match its + * declared `#[derive(TicketExtension)]` schema. + */ +export type GameTicketExtensions = Readonly>; const CONTROL = /[\u0000-\u001f\u007f]/; const CONTROLS = /[\u0000-\u001f\u007f]/g; const encoder = new TextEncoder(); @@ -33,6 +66,11 @@ export type MintGameTicketOptions = Readonly<{ role?: GameTicketRole; /** Display name vouched for by this issuer (never identity). */ name?: string; + /** + * Extension claims decided **on the server** (never from the browser). + * Identity-, reward- and economy-looking names are refused. + */ + extensions?: GameTicketExtensions; /** Lifetime in seconds (1..=300, default 60). */ ttlSeconds?: number; /** Clock in Unix milliseconds (tests). Default `Date.now()`. */ @@ -73,6 +111,45 @@ function boundedText(value: unknown, label: string, maxBytes: number): string { return value; } +function plainJson(value: unknown, depth: number): boolean { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return true; + if (typeof value === 'number') return Number.isFinite(value); + if (depth > 8 || typeof value !== 'object') return false; + if (Array.isArray(value)) return value.every((item) => plainJson(item, depth + 1)); + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return false; + return Object.values(value).every((item) => plainJson(item, depth + 1)); +} + +/** + * Validate extension claims: a plain object of identifier names (not on + * the deny list) to JSON values, at most `MAX_GAME_TICKET_EXTENSION_BYTES`. + * Returns undefined for none. Throws `TypeError`. + */ +export function checkGameTicketExtensions(value: unknown): GameTicketExtensions | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value !== 'object' || Array.isArray(value) || !plainJson(value, 0)) { + throw new TypeError('game ticket extensions must be a plain JSON object'); + } + const names = Object.keys(value); + if (names.length === 0) return undefined; + for (const name of names) { + if (DENIED.has(normalizeName(name))) { + throw new TypeError( + `game ticket extension \`${name}\` looks like identity, a reward or an economy value; the zone binds identity from \`sub\` and handlers decide amounts` + ); + } + if (!EXTENSION_NAME.test(name)) { + throw new TypeError(`game ticket extension name \`${name}\` must be an identifier`); + } + } + const json = JSON.stringify(value); + if (encoder.encode(json).byteLength > MAX_GAME_TICKET_EXTENSION_BYTES) { + throw new TypeError(`game ticket extensions exceed ${MAX_GAME_TICKET_EXTENSION_BYTES} bytes`); + } + return JSON.parse(json) as GameTicketExtensions; +} + /** Unpadded base64url (RFC 7515 §2). */ export function base64url(bytes: Uint8Array): string { let binary = ''; @@ -143,6 +220,7 @@ export async function mintGameTicket(options: MintGameTicketOptions): Promise JSON.parse(Buffer.from(ticket.split('.')[1], 'base64url').toString('utf8')); @@ -60,6 +65,11 @@ async function fixtureTickets() { const forgedClaims = Buffer.from(JSON.stringify({ sub: 'admin', zone: 'z-sveltekit', exp: 4_000_000_000, nonce: 'fixture-tampered', role: 'player' })).toString('base64url'); + const extended = await mintGameTicket({ + secret: SECRET, principal: 'session-user-3', zone: 'z-sveltekit', + ttlSeconds: 300, now: future, nonce: 'fixture-extended', + extensions: { match_id: 'm-7', team: 2 } + }); const otherKey = await mintGameTicket({ secret: 'a-different-secret-0123456789abcdef!!', principal: 'session-user-1', zone: 'z-sveltekit', ttlSeconds: 300, now: future, nonce: 'fixture-other-key' @@ -69,6 +79,7 @@ async function fixtureTickets() { zone: 'z-sveltekit', valid: { ticket: valid.ticket, principal: 'session-user-1', name: 'Ana', exp: 4_000_000_000, nonce: 'fixture-valid' }, observer: { ticket: observer.ticket, principal: 'session-user-2' }, + extended: { ticket: extended.ticket, principal: 'session-user-3', ext: { match_id: 'm-7', team: 2 } }, expired: expired.ticket, wrongZone: wrongZone.ticket, tampered: `${h}.${forgedClaims}.${s}`, @@ -92,6 +103,55 @@ test('mintGameTicket is the PROTOCOL.md §8 format the Rust GameServer admits (f assert.equal(decodeClaims(tickets.observer.ticket).role, 'observer'); }); +test('extension claims: server-only, deny list shared with the Rust derives', async () => { + const fixture = JSON.parse(readFileSync(DENYLIST, 'utf8')); + assert.deepEqual([...TRUSTED_FIELD_DENYLIST], [...fixture.identity, ...fixture.reward, ...fixture.economy]); + const secret = SECRET; + const minted = await mintGameTicket({ secret, principal: 'p', zone: 'z', extensions: { match_id: 'm', team: 1 } }); + assert.deepEqual(decodeClaims(minted.ticket).ext, { match_id: 'm', team: 1 }); + const none = await mintGameTicket({ secret, principal: 'p', zone: 'z', extensions: {} }); + assert.equal('ext' in decodeClaims(none.ticket), false); + for (const name of ['userId', 'Gold', 'pay_out', 'XP', 'sub', 'owner-id']) { + await assert.rejects( + mintGameTicket({ secret, principal: 'p', zone: 'z', extensions: { [name]: 1 } }), + /identity, a reward or an economy value/, + name + ); + } + await assert.rejects(mintGameTicket({ secret, principal: 'p', zone: 'z', extensions: ['m'] }), /plain JSON object/); + await assert.rejects(mintGameTicket({ secret, principal: 'p', zone: 'z', extensions: { at: new Date() } }), /plain JSON object/); + await assert.rejects(mintGameTicket({ secret, principal: 'p', zone: 'z', extensions: { 'match id': 1 } }), /identifier/); + await assert.rejects( + mintGameTicket({ secret, principal: 'p', zone: 'z', extensions: { blob: 'x'.repeat(2100) } }), + /exceed 2048 bytes/ + ); +}); + +test('ticket handler: claims() adds server-side extensions; the browser body cannot', async () => { + const contexts = []; + const { call } = harness({ + claims: (context) => { + contexts.push(context); + return { match_id: `match-of-${context.principal}`, team: 2 }; + } + }); + const body = await expectStatus(call({ body: { zone: 'pit-u1' } }), 200); + assert.deepEqual(decodeClaims(body.ticket).ext, { match_id: 'match-of-u1', team: 2 }); + assert.equal(contexts[0].zone, 'pit-u1'); + assert.equal(contexts[0].principal, 'u1'); + + // Claims are never request fields, whatever they are called. + for (const field of ['ext', 'claims', 'extensions', 'match_id', 'team']) { + const fresh = harness({ claims: () => ({ team: 1 }) }); + await expectStatus(fresh.call({ body: { zone: 'pit-u1', [field]: { team: 9 } } }), 400, 'unexpected_field'); + } + // No callback: no ext claim at all. + const plain = await expectStatus(harness().call({}), 200); + assert.equal('ext' in decodeClaims(plain.ticket), false); + // A callback returning a denied name is refused, not signed. + await expectStatus(harness({ claims: () => ({ gold: 100 }) }).call({}), 400, 'invalid_claims'); +}); + test('mintGameTicket rejects short secrets and malformed claims', async () => { await assert.rejects(mintGameTicket({ secret: 'short', principal: 'p', zone: 'z' }), /at least 32 bytes/); const secret = SECRET; From 326cfb4baf462c032f4cde46faecf4ef6a3e2384 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 00:10:06 -0500 Subject: [PATCH 04/40] feat(game): async zone factory with allocation records GameServer takes a ZoneFactory: ZoneFactory::new(|activation| async {..}) may await storage, ZoneFactory::sync covers factories without I/O. The factory now runs on the activation task after the ownership claim and snapshot load and receives ZoneActivation { zone_id, activation (fencing token), allocation }. Allocation records come from a ZoneAllocationStore (with_allocation_store; memory store included). Joins queued during creation are admitted afterwards; Ok(None) answers NOT_FOUND, a factory or store error and a factory outliving activation_timeout answer the new ZONE_UNAVAILABLE (9) code. Refusals release the claim and are counted in activations_refused_total. activate() now waits for the zone; begin_activation() only starts it. Demos migrated. [[tasks/gr-p22-zone-allocation-payload]] --- distributed_game/PROTOCOL.md | 4 +- distributed_game/README.md | 29 ++- distributed_game/examples/echo_arena.rs | 8 +- distributed_game/src/metrics.rs | 39 ++++ distributed_game/src/net/mod.rs | 136 ++++++++++--- distributed_game/src/protocol/messages.rs | 3 + distributed_game/src/world/allocation.rs | 204 +++++++++++++++++++ distributed_game/src/world/mod.rs | 8 + distributed_game/tests/bots/main.rs | 9 +- distributed_game/tests/net_e2e/main.rs | 198 ++++++++++++++++-- distributed_game/tests/world/main.rs | 10 +- distributed_game/tests/zone_runtime/arena.rs | 2 + examples/game/server/src/arena/mod.rs | 16 +- examples/game/server/src/freeze/world.rs | 24 ++- examples/game/server/src/gallery/mod.rs | 16 +- examples/game/server/src/grid/mod.rs | 22 +- examples/game/server/src/mobpit/world.rs | 18 +- examples/game/server/src/ripple/mod.rs | 20 +- js/src/game/messages.ts | 4 +- 19 files changed, 667 insertions(+), 103 deletions(-) create mode 100644 distributed_game/src/world/allocation.rs diff --git a/distributed_game/PROTOCOL.md b/distributed_game/PROTOCOL.md index 52a12eb94..568c8a6c7 100644 --- a/distributed_game/PROTOCOL.md +++ b/distributed_game/PROTOCOL.md @@ -308,7 +308,8 @@ Redirect := 0x89, string zone_id, string url Hello. The server closes the socket. - **Error**: codes `1` BAD_MESSAGE, `2` UNAUTHORIZED, `3` REJECTED, `4` ZONE_FULL, `5` VERSION_MISMATCH, `6` RATE_LIMITED, `7` NOT_FOUND, - `8` RECONNECT_FAILED. Errors during the handshake are followed by close; + `8` RECONNECT_FAILED, `9` ZONE_UNAVAILABLE (the zone could not be created + right now; retry later). Errors during the handshake are followed by close; `BAD_MESSAGE` and `RATE_LIMITED` after joining are not. Worked example — Welcome (session "s1", zone "z1", player, 30 Hz ticks, @@ -560,6 +561,7 @@ error codes and treat them as "session over" / "request failed"): - `0x89 Redirect` (server → client). - Kick code `7` RELOAD. +- Error code `9` ZONE_UNAVAILABLE. - Transfer tickets (§8.1) and content-version policies (§8.2); `Handoff` and `Freeze`/`Resume` encodings are unchanged. diff --git a/distributed_game/README.md b/distributed_game/README.md index b9b201447..38d912867 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -218,13 +218,14 @@ for out in host.drain_outbound() { /* Outbound::Frame bytes go to the client */ assert_eq!(game_surface().kinds[0].name, "dot"); ``` -Serve it over WebSockets (`net` feature). The closure decides which zone -ids this server hosts and with what config: +Serve it over WebSockets (`net` feature). The zone factory decides which +zone ids this server hosts and with what config: ```rust,ignore -let server = GameServer::::new(ServerConfig::new(TicketKey::new(secret)), |zone_id| { - zone_id.starts_with("room-").then_some(()) -}); +let server = GameServer::::new( + ServerConfig::new(TicketKey::new(secret)), + ZoneFactory::sync(|activation| activation.zone_id.starts_with("room-").then_some(())), +); server.serve(tokio::net::TcpListener::bind("0.0.0.0:2567").await?, shutdown_signal).await?; ``` @@ -994,6 +995,24 @@ principals commit concurrently; settlements arrive in completion order. 2 Hz while its neighborhoods run at 30 Hz); the world content version is applied afterwards. +**Zone factory and allocation.** `ZoneFactory::new(|activation| async move +{ … })` may await storage: it runs on the activation task after the +ownership claim and snapshot load, before the zone exists. It receives a +`ZoneActivation { zone_id, activation, allocation }` — `activation` is the +fencing token (= `boot_epoch` = the facts' stamp activation), `allocation` +the `ZoneAllocation` record (roster, mode, config version) that the service +which allocated the zone wrote into the store passed to +`GameServer::with_allocation_store` (`MemoryZoneAllocationStore`, or your +own `ZoneAllocationStore`). Return `Ok(Some(config))` to create the zone +(`on_create` gets it), `Ok(None)` for "no such zone" (`Error(NOT_FOUND)`), +`Err(ZoneCreateError)` when something it needs failed +(`Error(ZONE_UNAVAILABLE)`). Store failures and a factory that outlives +`ServerConfig::activation_timeout` also answer `ZONE_UNAVAILABLE`, never a +hang. Joins that arrive during creation wait and are admitted afterwards; +the tick never awaits. `ZoneFactory::sync(|activation| …)` covers factories +without I/O. `GameServer::activate(zone).await` waits for a zone to run; +`begin_activation(zone)` only starts it. + **Activate on demand / hibernate.** `GameServer` creates a zone on first join (or tier-up event). With `ServerConfig::hibernate_after = Some(d)` a zone that stays Dormant for `d` (no clients, no in-flight facts, not diff --git a/distributed_game/examples/echo_arena.rs b/distributed_game/examples/echo_arena.rs index 3da2662d9..98c2b88f5 100644 --- a/distributed_game/examples/echo_arena.rs +++ b/distributed_game/examples/echo_arena.rs @@ -16,7 +16,7 @@ use std::collections::BTreeMap; use std::io::{BufRead, Write}; -use distributed_game::net::{GameServer, ServerConfig, TicketKey}; +use distributed_game::net::{GameServer, ServerConfig, TicketKey, ZoneFactory}; use distributed_game::{ ConsumePolicy, Ctx, EntitySchema, EntityState, FieldType, FieldValue, KindDef, LeaveReason, PlayerId, View, WorldView, Zone, @@ -105,8 +105,10 @@ async fn main() -> std::io::Result<()> { let mut config = ServerConfig::new(TicketKey::new(secret)); config.host.tick_rate = 30; config.host.patch_rate = 20; - let server = - GameServer::::new(config, |zone| zone.starts_with("arena-").then_some(())); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with("arena-").then_some(())), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; println!("LISTENING ws://{addr}"); diff --git a/distributed_game/src/metrics.rs b/distributed_game/src/metrics.rs index f3e09e828..19c66e27e 100644 --- a/distributed_game/src/metrics.rs +++ b/distributed_game/src/metrics.rs @@ -37,6 +37,7 @@ //! | `distributed_game_zone_facts_pending` | gauge | zone | //! | `distributed_game_zone_facts_settled_total` | counter | zone, outcome | //! | `distributed_game_hibernation_saves_total` | counter | outcome (`saved`, `stale`, `error`) | +//! | `distributed_game_activations_refused_total` | counter | reason (`not_found`, `unavailable`) | use crate::facts::FactOutcome; use crate::host::{AdvanceReport, ZoneHost}; @@ -163,11 +164,16 @@ struct State { other: Counters, /// Server-wide (zones are forgotten when they hibernate). hibernation_saves: BTreeMap<&'static str, u64>, + /// Server-wide activations refused before the zone ran. + activations_refused: BTreeMap<&'static str, u64>, } /// Hibernation snapshot save outcome label values (fixed set). pub const HIBERNATION_OUTCOMES: [&str; 3] = ["saved", "stale", "error"]; +/// Refused activation reason label values (fixed set). +pub const ACTIVATION_REFUSALS: [&str; 2] = ["not_found", "unavailable"]; + impl State { fn zone(&mut self, zone: &str, max: usize) -> &mut ZoneSeries { if !self.zones.contains_key(zone) { @@ -294,6 +300,26 @@ impl GameMetrics { } } + /// A zone activation refused before the zone ran, by protocol error + /// code (`NOT_FOUND` → `not_found`, anything else → `unavailable`). + pub fn record_activation_refused(&self, code: u32) { + let label = if code == crate::protocol::error::NOT_FOUND { + "not_found" + } else { + "unavailable" + }; + *self.lock().activations_refused.entry(label).or_insert(0) += 1; + } + + /// Refused activations recorded with `reason`. + pub fn activations_refused(&self, reason: &str) -> u64 { + self.lock() + .activations_refused + .get(reason) + .copied() + .unwrap_or(0) + } + /// Hibernation saves recorded with `outcome`. pub fn hibernation_saves(&self, outcome: &str) -> u64 { self.lock() @@ -502,6 +528,19 @@ impl GameMetrics { "distributed_game_hibernation_saves_total{{outcome=\"{label}\"}} {v}" ); } + header( + &mut out, + "activations_refused_total", + "counter", + "Zone activations refused before the zone ran (unknown zone, factory or allocation failure).", + ); + for label in ACTIVATION_REFUSALS { + let v = state.activations_refused.get(label).copied().unwrap_or(0); + let _ = writeln!( + out, + "distributed_game_activations_refused_total{{reason=\"{label}\"}} {v}" + ); + } out } } diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index 8271e95cb..e0f72e89f 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -1,8 +1,13 @@ //! WebSocket transport (`net` feature): one axum server hosting many zones. //! //! - Route: `GET /zones/{zone_id}` upgrades to a WebSocket. Zones are created -//! on first use through the game's factory and each runs on its own Tokio -//! task driving a [`ZoneHost`]. +//! on first use through the game's async [`ZoneFactory`] and each runs on +//! its own Tokio task driving a [`ZoneHost`]. Creation (ownership claim, +//! snapshot, [`ZoneAllocation`](crate::world::ZoneAllocation) record, +//! factory) happens before the first tick; joins that arrive meanwhile +//! wait and are admitted afterwards, or answered `Error(NOT_FOUND)` (the +//! factory knows no such zone) / `Error(ZONE_UNAVAILABLE)` (it failed or +//! timed out). //! - Binary frames only (one protocol message per frame); text frames get //! `Error(BAD_MESSAGE)` and are closed. //! - Admission: the first frame must be Hello within `hello_timeout`. A fresh @@ -80,8 +85,8 @@ use crate::protocol::{error, kick, ClientMessage, Notice, ServerMessage, PROTOCO use crate::rng::ZoneRng; use crate::world::{ BarrierMode, ChildEvent, DottedTopology, FreezeState, HostAddr, LocalOwnership, OwnershipError, - StalePolicy, StateError, StoredZoneState, ZoneClaim, ZoneDirectory, ZoneOwnership, - ZoneStateStore, ZoneTopology, + StalePolicy, StateError, StoredZoneState, ZoneActivation, ZoneAllocationStore, ZoneClaim, + ZoneCreateError, ZoneDirectory, ZoneOwnership, ZoneStateStore, ZoneTopology, }; use crate::zone::{ClientId, TicketClaims, Tier, Zone}; @@ -341,7 +346,41 @@ enum ConnOut { Close, } -type ConfigFactory = dyn Fn(&str) -> Option<::Config> + Send + Sync; +type ZoneFactoryFn = + dyn Fn(ZoneActivation) -> BoxFuture<'static, Result, ZoneCreateError>> + Send + Sync; + +/// Creates the configuration of a zone being activated, or says it does +/// not exist (`Ok(None)` → `Error(NOT_FOUND)`) or cannot be created now +/// (`Err` → `Error(ZONE_UNAVAILABLE)`). Runs on the activation task before +/// the zone exists: it may await storage, and joins wait for it (bounded by +/// `ServerConfig::activation_timeout`). The tick never waits for it. +pub struct ZoneFactory(Box>); + +impl std::fmt::Debug for ZoneFactory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ZoneFactory(..)") + } +} + +impl ZoneFactory { + /// Async factory: `ZoneFactory::new(|activation| async move { … })`. + pub fn new(factory: F) -> Self + where + F: Fn(ZoneActivation) -> Fut + Send + Sync + 'static, + Fut: Future, ZoneCreateError>> + Send + 'static, + { + Self(Box::new(move |activation| Box::pin(factory(activation)))) + } + + /// Factory that needs no I/O (the zone id and the allocation record + /// decide): `ZoneFactory::sync(|a| a.zone_id.starts_with("arena-").then(Config::default))`. + pub fn sync(factory: impl Fn(&ZoneActivation) -> Option + Send + Sync + 'static) -> Self { + Self(Box::new(move |activation| { + let config = factory(&activation); + Box::pin(async move { Ok(config) }) + })) + } +} type FactsFactory = dyn Fn(&str) -> Option<(FactQueue, JoinHandle<()>)> + Send + Sync; type HostConfigFn = dyn Fn(&str, &mut HostConfig) + Send + Sync; @@ -361,7 +400,8 @@ struct WorldState { struct Inner { config: ServerConfig, clock: Arc, - factory: Box>, + factory: ZoneFactory, + allocations: Option>, facts: Option>, /// Per-zone adjustments of `config.host` (tick rates, capacity …). host_config: Option>, @@ -402,11 +442,9 @@ where Z: Zone, Z::Config: Send, { - /// Server whose zones are created by `factory(zone_id)`; `None` = 404. - pub fn new( - config: ServerConfig, - factory: impl Fn(&str) -> Option + Send + Sync + 'static, - ) -> Self { + /// Server whose zones are configured by `factory` when they activate + /// (see [`ZoneFactory`]). + pub fn new(config: ServerConfig, factory: ZoneFactory) -> Self { Self::with_clock(config, Arc::new(SystemClock::new()), factory) } @@ -414,7 +452,7 @@ where pub fn with_clock( config: ServerConfig, clock: Arc, - factory: impl Fn(&str) -> Option + Send + Sync + 'static, + factory: ZoneFactory, ) -> Self { let (shutdown, _) = watch::channel(false); let (net_sim, _) = watch::channel(config.net_sim); @@ -422,7 +460,8 @@ where Self { inner: Arc::new(Inner { clock, - factory: Box::new(factory), + factory, + allocations: None, facts: None, host_config: None, nonces: Mutex::new(NonceCache::default()), @@ -470,6 +509,13 @@ where self.configure(|inner| inner.ownership = ownership) } + /// Where zone allocation records are read at activation + /// ([`ZoneActivation::allocation`]); a store error refuses the + /// activation with `ZONE_UNAVAILABLE`. + pub fn with_allocation_store(self, store: Arc) -> Self { + self.configure(|inner| inner.allocations = Some(store)) + } + /// Where hibernated zone snapshots go. pub fn with_state_store(self, store: Arc) -> Self { self.configure(|inner| inner.state_store = Some(store)) @@ -501,7 +547,7 @@ where self.inner.world.lock().expect("world").version.clone() } - /// True when `zone_id` is active on this server. + /// True when `zone_id` is active (or being created) on this server. pub fn is_active(&self, zone_id: &str) -> bool { self.inner .zones @@ -532,11 +578,24 @@ where rx.await.ok() } - /// Activate `zone_id` locally (as if a client asked for it). - pub fn activate(&self, zone_id: &str) -> bool { + /// Start activating `zone_id` locally without waiting (it is registered + /// at once, so barriers and joins already reach it). False only while + /// shutting down; an unknown zone unregisters itself once its factory + /// answers. + pub fn begin_activation(&self, zone_id: &str) -> bool { activate_local(&self.inner, zone_id).is_some() } + /// Activate `zone_id` locally (as if a client asked for it) and wait + /// until it runs. False when the factory knows no such zone, creation + /// failed, or the server is shutting down. + pub async fn activate(&self, zone_id: &str) -> bool { + if activate_local(&self.inner, zone_id).is_none() { + return false; + } + self.inspect(zone_id, |_| ()).await.is_some() + } + /// Barrier phase 1 on this server: freeze every active zone (and every /// zone activated until commit/abort) with `notice`, then wait up to /// `timeout` for all to acknowledge. `Err` lists zones that did not. @@ -789,7 +848,6 @@ where if let Some(entry) = zones.get(zone_id).filter(|e| !e.tx.is_closed()) { return Some(entry.tx.clone()); } - let zone_config = (inner.factory)(zone_id)?; let (tx, rx) = mpsc::channel(1024); let activation = inner.activations.fetch_add(1, Ordering::Relaxed) + 1; let world = inner.world.lock().expect("world").clone(); @@ -804,7 +862,6 @@ where let task = tokio::spawn(run_zone::( inner.clone(), zone_id.to_string(), - zone_config, facts, rx, activation, @@ -958,12 +1015,13 @@ async fn acquire_claim( } } -/// Activation failed: answer everything queued and unregister. +/// Activation failed: answer everything queued with `code` and unregister. fn refuse_activation( inner: &Arc>, zone_id: &str, activation: u64, mut rx: mpsc::Receiver, + code: u32, message: &str, ) { { @@ -980,7 +1038,7 @@ fn refuse_activation( match cmd { ZoneCmd::Join { reply, .. } | ZoneCmd::Reconnect { reply, .. } => { let _ = reply.send(Err(JoinError { - code: error::NOT_FOUND, + code, message: message.to_string(), })); } @@ -1018,7 +1076,6 @@ async fn renew_claim( async fn run_zone( inner: Arc>, zone_id: String, - zone_config: Z::Config, facts: Option<(FactQueue, JoinHandle<()>)>, mut rx: mpsc::Receiver, activation: u64, @@ -1031,7 +1088,9 @@ async fn run_zone( let holder = format!("{}#{activation}", inner.host_id); let claim = match acquire_claim(&inner, &zone_id, &holder).await { Ok(claim) => claim, - Err(message) => return refuse_activation(&inner, &zone_id, activation, rx, &message), + Err(message) => { + return refuse_activation(&inner, &zone_id, activation, rx, error::NOT_FOUND, &message) + } }; // 2. Restore the hibernated snapshot, if any. let saved = match &inner.state_store { @@ -1047,6 +1106,7 @@ async fn run_zone( &zone_id, activation, rx, + error::ZONE_UNAVAILABLE, "zone state unavailable", ); } @@ -1054,6 +1114,36 @@ async fn run_zone( } None => None, }; + // 3. The allocation record and the game's factory, bounded together. + let created = tokio::time::timeout(inner.config.activation_timeout, async { + let allocation = match &inner.allocations { + Some(store) => store.load(&zone_id).await?, + None => None, + }; + (inner.factory.0)(ZoneActivation { + zone_id: zone_id.clone(), + activation: claim.token, + allocation, + }) + .await + }) + .await; + let zone_config = match created { + Ok(Ok(Some(config))) => config, + refused => { + inner.ownership.release(&claim).await; + let (code, message) = match refused { + Ok(Ok(None)) => (error::NOT_FOUND, "unknown zone".to_string()), + Ok(Err(e)) => (error::ZONE_UNAVAILABLE, e.to_string()), + _ => ( + error::ZONE_UNAVAILABLE, + "zone creation timed out".to_string(), + ), + }; + inner.metrics.record_activation_refused(code); + return refuse_activation(&inner, &zone_id, activation, rx, code, &message); + } + }; let mut host_config = inner.config.host.clone(); if let Some(adjust) = &inner.host_config { adjust(&zone_id, &mut host_config); @@ -1112,7 +1202,7 @@ async fn run_zone( host.request_freeze(notice); } - // 3. Keep the claim alive. + // 4. Keep the claim alive. let current = Arc::new(Mutex::new(claim)); let (lost_tx, mut lost_rx) = oneshot::channel(); let renewer = tokio::spawn(renew_claim( diff --git a/distributed_game/src/protocol/messages.rs b/distributed_game/src/protocol/messages.rs index 61561f82d..c0a010abe 100644 --- a/distributed_game/src/protocol/messages.rs +++ b/distributed_game/src/protocol/messages.rs @@ -77,6 +77,9 @@ pub mod error { pub const NOT_FOUND: u32 = 7; /// Reconnect token unknown or grace window elapsed. pub const RECONNECT_FAILED: u32 = 8; + /// The zone could not be created right now (its factory or allocation + /// store failed or timed out); retry later. + pub const ZONE_UNAVAILABLE: u32 = 9; } /// Client handshake. diff --git a/distributed_game/src/world/allocation.rs b/distributed_game/src/world/allocation.rs new file mode 100644 index 000000000..156b185aa --- /dev/null +++ b/distributed_game/src/world/allocation.rs @@ -0,0 +1,204 @@ +//! Zone allocation: what a zone is for, decided before it starts. +//! +//! A service that allocates a zone (game-services `Match`, a party +//! finder, an instance spawner) writes a [`ZoneAllocation`] record for the +//! zone id: roster, mode, config version, seed. When the zone activates, +//! the server loads the record and hands it to the async zone factory in a +//! [`ZoneActivation`] together with the activation's fencing token, so the +//! zone's `on_create` sees it before the first player joins. Joins that +//! arrive while the zone is being created wait; the tick never awaits. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Mutex; + +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::Value; + +use crate::facts::BoxFuture; + +/// The record an allocating service writes for a zone. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZoneAllocation { + /// Who allocated it (service name, e.g. `"game-services.match"`). + pub allocated_by: String, + /// Monotonic revision per zone id (a re-allocation bumps it). + pub revision: u64, + /// Game-defined payload (roster, mode, config version…). + pub payload: Value, +} + +impl ZoneAllocation { + /// Record with a typed payload. + pub fn new( + allocated_by: impl Into, + revision: u64, + payload: &impl Serialize, + ) -> Result { + Ok(Self { + allocated_by: allocated_by.into(), + revision, + payload: serde_json::to_value(payload)?, + }) + } + + /// Decode the payload. + pub fn payload_as(&self) -> Result { + serde_json::from_value(self.payload.clone()) + } +} + +/// Why an allocation store failed (the activation is refused with +/// `ZONE_UNAVAILABLE`, never admitted half-configured). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AllocationError(pub String); + +impl fmt::Display for AllocationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "zone allocation store: {}", self.0) + } +} + +impl std::error::Error for AllocationError {} + +/// Where allocation records live. Async: typically a service's database or +/// read model, read once per activation. +pub trait ZoneAllocationStore: Send + Sync + 'static { + /// The record for `zone_id`, if any. + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, AllocationError>>; + + /// Write a record. Refused when a record with a higher revision exists. + fn put<'a>( + &'a self, + zone_id: &'a str, + allocation: ZoneAllocation, + ) -> BoxFuture<'a, Result<(), AllocationError>>; + + /// Drop the record (the zone was released). + fn remove<'a>(&'a self, zone_id: &'a str) -> BoxFuture<'a, Result<(), AllocationError>>; +} + +/// In-memory allocation store (tests, single process). +#[derive(Debug, Default)] +pub struct MemoryZoneAllocationStore { + records: Mutex>, +} + +impl MemoryZoneAllocationStore { + /// Empty. + pub fn new() -> Self { + Self::default() + } +} + +impl ZoneAllocationStore for MemoryZoneAllocationStore { + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, AllocationError>> { + let found = self.records.lock().expect("records").get(zone_id).cloned(); + Box::pin(async move { Ok(found) }) + } + + fn put<'a>( + &'a self, + zone_id: &'a str, + allocation: ZoneAllocation, + ) -> BoxFuture<'a, Result<(), AllocationError>> { + let mut records = self.records.lock().expect("records"); + let result = match records.get(zone_id) { + Some(current) if current.revision > allocation.revision => { + Err(AllocationError(format!( + "revision {} is older than the stored {}", + allocation.revision, current.revision + ))) + } + _ => { + records.insert(zone_id.to_string(), allocation); + Ok(()) + } + }; + Box::pin(async move { result }) + } + + fn remove<'a>(&'a self, zone_id: &'a str) -> BoxFuture<'a, Result<(), AllocationError>> { + self.records.lock().expect("records").remove(zone_id); + Box::pin(async { Ok(()) }) + } +} + +/// What the zone factory is told about the activation it configures. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZoneActivation { + /// Zone id. + pub zone_id: String, + /// Activation epoch: the ownership fencing token, also the host's + /// `boot_epoch` and every fact stamp's `activation`. + pub activation: u64, + /// The allocation record, when an allocation store is configured and + /// holds one for this zone. + pub allocation: Option, +} + +/// The factory refused to create the zone because something it needs +/// failed (storage down, invalid allocation). Clients waiting to join get +/// `Error(ZONE_UNAVAILABLE)` with the message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZoneCreateError(pub String); + +impl ZoneCreateError { + /// Error with a message for the joining clients. + pub fn new(message: impl Into) -> Self { + Self(message.into()) + } +} + +impl fmt::Display for ZoneCreateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ZoneCreateError {} + +impl From for ZoneCreateError { + fn from(error: AllocationError) -> Self { + Self(error.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn block(f: impl std::future::Future) -> T { + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + let mut f = std::pin::pin!(f); + loop { + if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { + return v; + } + } + } + + #[test] + fn memory_store_keeps_the_newest_revision() { + let store = MemoryZoneAllocationStore::new(); + let record = |revision| { + ZoneAllocation::new("match", revision, &serde_json::json!({"mode": "duel"})).unwrap() + }; + block(store.put("m1", record(2))).unwrap(); + assert!(block(store.put("m1", record(1))).is_err()); + let loaded = block(store.load("m1")).unwrap().unwrap(); + assert_eq!(loaded.revision, 2); + let payload: serde_json::Value = loaded.payload_as().unwrap(); + assert_eq!(payload["mode"], "duel"); + block(store.remove("m1")).unwrap(); + assert_eq!(block(store.load("m1")).unwrap(), None); + } +} diff --git a/distributed_game/src/world/mod.rs b/distributed_game/src/world/mod.rs index d325cccaa..29add916b 100644 --- a/distributed_game/src/world/mod.rs +++ b/distributed_game/src/world/mod.rs @@ -10,9 +10,13 @@ //! ([`MemoryZoneStateStore`]; `SqliteZoneStateStore` with `sqlite`). //! - [`topology`]: [`ZoneTopology`] parent links (province > city > borough //! > neighborhood) and [`ChildEvent`] tier-up propagation. +//! - [`allocation`]: [`ZoneAllocation`] records written by the service that +//! allocates a zone, read before the zone is created +//! ([`ZoneActivation`], [`ZoneAllocationStore`]). //! - [`barrier`]: content-version policy types shared by the host and the //! world barrier ([`StalePolicy`], [`BarrierMode`]). +pub mod allocation; pub mod barrier; pub mod directory; pub mod ownership; @@ -20,6 +24,10 @@ pub mod state; pub mod topology; pub use crate::facts::BoxFuture; +pub use allocation::{ + AllocationError, MemoryZoneAllocationStore, ZoneActivation, ZoneAllocation, + ZoneAllocationStore, ZoneCreateError, +}; pub use barrier::{BarrierMode, FreezeState, StalePolicy}; pub use directory::{HashRing, HostAddr, StaticDirectory, ZoneDirectory}; #[cfg(feature = "lease")] diff --git a/distributed_game/tests/bots/main.rs b/distributed_game/tests/bots/main.rs index e8543d42c..6a673838d 100644 --- a/distributed_game/tests/bots/main.rs +++ b/distributed_game/tests/bots/main.rs @@ -9,7 +9,7 @@ use std::time::Duration; use arena::{Arena, ArenaConfig}; use distributed_game::bots::{run_bots, BehaviourKind, BotConfig}; -use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; +use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey, ZoneFactory}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; const SECRET: &str = "bots-secret"; @@ -34,9 +34,10 @@ async fn fifty_bots_join_play_and_receive_deltas() { Duration::from_millis(5), Duration::from_millis(5), )); - let server = GameServer::::new(config, |zone| { - zone.starts_with("load").then(ArenaConfig::default) - }); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with("load").then(ArenaConfig::default)), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let serving = server.clone(); diff --git a/distributed_game/tests/net_e2e/main.rs b/distributed_game/tests/net_e2e/main.rs index 48ee34e5a..68a376b48 100644 --- a/distributed_game/tests/net_e2e/main.rs +++ b/distributed_game/tests/net_e2e/main.rs @@ -7,7 +7,7 @@ mod arena; use std::time::Duration; use arena::{Arena, ArenaConfig, AVATAR}; -use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; +use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey, ZoneFactory}; use distributed_game::protocol::{error, kick, Hello}; use distributed_game::{ ClientMessage, ClientMirror, FieldValue, MemoryFactSink, RetryPolicy, Role, ServerMessage, @@ -49,9 +49,10 @@ async fn start_keyed( config.host.tick_rate = 30; config.host.patch_rate = 20; config.net_sim = net_sim; - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }) + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ) .with_fact_sink(MemoryFactSink::new(), 64, RetryPolicy::default()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -419,9 +420,10 @@ async fn http_get(addr: std::net::SocketAddr, path: &str) -> String { async fn serve_with_merges_extra_routes_and_shuts_down_gracefully() { let mut config = ServerConfig::new(TicketKey::new(SECRET)); config.host.tick_rate = 30; - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let extra = axum::Router::new().route("/hello", axum::routing::get(|| async { "hi" })); @@ -548,9 +550,10 @@ fn bff_ticket(jti: &str, exp_in: u64, zone_route: Option<&str>) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn custom_verifier_admits_web_bff_tickets_with_replay_protection() { let config = ServerConfig::new(TicketKey::new(SECRET)).with_ticket_verifier(bff_verifier); - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("ws://{}", listener.local_addr().unwrap()); let serving = server.clone(); @@ -631,9 +634,10 @@ async fn host_config_hook_sets_per_zone_rates() { let mut config = ServerConfig::new(TicketKey::new(SECRET)); config.host.tick_rate = 30; config.host.content_version = "c1".into(); - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }) + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ) .with_host_config(|zone_id, host| { if zone_id == "z-slow" { host.tick_rate = 5; @@ -641,8 +645,8 @@ async fn host_config_hook_sets_per_zone_rates() { host.content_version = "ignored".into(); } }); - assert!(server.activate("z-slow")); - assert!(server.activate("z-hot")); + assert!(server.activate("z-slow").await); + assert!(server.activate("z-hot").await); let slow = server .inspect("z-slow", |h| { ( @@ -684,7 +688,8 @@ fn extended_ticket(nonce: &str, ext: serde_json::Value) -> String { async fn extension_claims_reach_on_auth_and_player_info() { assert_eq!(MatchClaims::CLAIMS, ["match_id", "team", "party"]); let config = ServerConfig::new(TicketKey::new(SECRET)).with_ticket_extensions::(); - let server = GameServer::::new(config, |_| Some(ArenaConfig::default())); + let server = + GameServer::::new(config, ZoneFactory::sync(|_| Some(ArenaConfig::default()))); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("ws://{}", listener.local_addr().unwrap()); let serving = server.clone(); @@ -777,3 +782,164 @@ async fn extension_claims_are_refused_without_a_declared_schema() { .expect("server stops") .unwrap(); } + +async fn serve_arena(server: &GameServer) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}", listener.local_addr().unwrap()); + let serving = server.clone(); + let handle = tokio::spawn(async move { + serving + .serve(listener, std::future::pending()) + .await + .unwrap(); + }); + (url, handle) +} + +async fn stop(server: GameServer, handle: tokio::task::JoinHandle<()>) { + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("server stops") + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_factory_reads_the_allocation_before_admitting_queued_joins() { + use distributed_game::world::{MemoryZoneAllocationStore, ZoneAllocation, ZoneAllocationStore}; + use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + use std::sync::Arc; + + let store = Arc::new(MemoryZoneAllocationStore::new()); + let roster = serde_json::json!({ "mode": "duel", "roster": ["ana", "bo"] }); + store + .put( + "z-match", + ZoneAllocation::new("game-services.match", 1, &roster).unwrap(), + ) + .await + .unwrap(); + let created = Arc::new(AtomicBool::new(false)); + let calls = Arc::new(AtomicU32::new(0)); + let (done, calls_in) = (created.clone(), calls.clone()); + let server = GameServer::::new( + ServerConfig::new(TicketKey::new(SECRET)), + ZoneFactory::new(move |activation| { + let (done, calls) = (done.clone(), calls_in.clone()); + async move { + calls.fetch_add(1, Ordering::SeqCst); + assert!(activation.activation > 0, "the fencing token"); + // Storage is slow: both joins arrive while this awaits. + tokio::time::sleep(Duration::from_millis(200)).await; + done.store(true, Ordering::SeqCst); + Ok(Some(ArenaConfig { + allocation: activation.allocation.map(|a| a.payload), + ..ArenaConfig::default() + })) + } + }), + ) + .with_allocation_store(store); + let (url, handle) = serve_arena(&server).await; + + let join = |principal: &'static str, nonce: &'static str| { + let url = url.clone(); + let created = created.clone(); + async move { + let mut ws = connect(&url, "z-match").await; + hello( + &mut ws, + ticket(principal, "z-match", nonce, Role::Player), + String::new(), + ) + .await; + let msg = next(&mut ws, &mut ClientMirror::new()).await; + assert!( + created.load(Ordering::SeqCst), + "admitted only after the factory finished" + ); + matches!(msg, Some(ServerMessage::Welcome(_))) + } + }; + let (ana, bo) = tokio::join!(join("ana", "q1"), join("bo", "q2")); + assert!(ana && bo, "both queued joins admitted"); + assert_eq!(calls.load(Ordering::SeqCst), 1, "one activation"); + let allocation = server + .inspect("z-match", |host| host.zone().cfg.allocation.clone()) + .await + .unwrap(); + assert_eq!(allocation, Some(roster), "the allocation reached on_create"); + stop(server, handle).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn factory_failures_answer_joins_with_typed_errors() { + use distributed_game::world::{ + AllocationError, BoxFuture, ZoneAllocation, ZoneAllocationStore, ZoneCreateError, + }; + use std::sync::Arc; + + struct DownStore; + impl ZoneAllocationStore for DownStore { + fn load<'a>( + &'a self, + _: &'a str, + ) -> BoxFuture<'a, Result, AllocationError>> { + Box::pin(async { Err(AllocationError("connection refused".into())) }) + } + fn put<'a>( + &'a self, + _: &'a str, + _: ZoneAllocation, + ) -> BoxFuture<'a, Result<(), AllocationError>> { + Box::pin(async { Ok(()) }) + } + fn remove<'a>(&'a self, _: &'a str) -> BoxFuture<'a, Result<(), AllocationError>> { + Box::pin(async { Ok(()) }) + } + } + + let mut config = ServerConfig::new(TicketKey::new(SECRET)); + config.activation_timeout = Duration::from_millis(300); + let server = GameServer::::new( + config, + ZoneFactory::new(|activation| async move { + match activation.zone_id.as_str() { + "z-broken" => Err(ZoneCreateError::new("roster store unavailable")), + "z-hang" => std::future::pending().await, + "z-ok" => Ok(Some(ArenaConfig::default())), + _ => Ok(None), + } + }), + ); + let (url, handle) = serve_arena(&server).await; + let started = std::time::Instant::now(); + for (zone, code, nonce) in [ + ("z-broken", error::ZONE_UNAVAILABLE, "f1"), + ("z-hang", error::ZONE_UNAVAILABLE, "f2"), + ("z-unknown", error::NOT_FOUND, "f3"), + ] { + expect_error(&url, zone, ticket("ana", zone, nonce, Role::Player), code).await; + assert!(!server.is_active(zone), "{zone} unregistered"); + } + assert!(started.elapsed() < Duration::from_secs(3), "no hang"); + assert!(!server.activate("z-broken").await); + assert!(server.activate("z-ok").await); + stop(server, handle).await; + + // An allocation store failure refuses too, without calling the factory. + let server = GameServer::::new( + ServerConfig::new(TicketKey::new(SECRET)), + ZoneFactory::new(|_| async { panic!("factory must not run without its allocation") }), + ) + .with_allocation_store(Arc::new(DownStore)); + let (url, handle) = serve_arena(&server).await; + expect_error( + &url, + "z-match", + ticket("ana", "z-match", "f4", Role::Player), + error::ZONE_UNAVAILABLE, + ) + .await; + stop(server, handle).await; +} diff --git a/distributed_game/tests/world/main.rs b/distributed_game/tests/world/main.rs index 6c4c2cb73..78cf2a7e7 100644 --- a/distributed_game/tests/world/main.rs +++ b/distributed_game/tests/world/main.rs @@ -10,6 +10,7 @@ use std::time::Duration; use distributed::lease::InMemoryLeaseStore; use distributed_game::net::{ AbortNotes, GameServer, RemoteHost, RolloutError, ServerConfig, TicketKey, WorldCoordinator, + ZoneFactory, }; use distributed_game::protocol::{error, kick, Hello, Notice}; use distributed_game::world::{ @@ -93,7 +94,7 @@ async fn two_hosts( for (host, listener) in [(a, la), (b, lb)] { let sink = MemoryFactSink::new(); let f = factory.clone(); - let server = GameServer::::new(config(), move |zone| f(zone)) + let server = GameServer::::new(config(), ZoneFactory::sync(move |a| f(&a.zone_id))) .with_fact_sink(sink.clone(), 64, RetryPolicy::default()) .with_host_id(host.id.clone()) .with_directory(dir.clone()) @@ -519,7 +520,7 @@ async fn dormant_zone_hibernates_to_sqlite_and_restores() { let sink = MemoryFactSink::new(); let (listener, addr) = listen().await; let factory = realm_factory(&[], &[]); - let server = GameServer::::new(config, move |z| factory(z)) + let server = GameServer::::new(config, ZoneFactory::sync(move |a| factory(&a.zone_id))) .with_fact_sink(sink.clone(), 64, RetryPolicy::default()) .with_state_store(store.clone()); serve(&server, listener); @@ -606,7 +607,8 @@ async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { let (listener, addr) = listen().await; let factory = realm_factory(&[], &[]); let server = - GameServer::::new(config, move |z| factory(z)).with_state_store(store.clone()); + GameServer::::new(config, ZoneFactory::sync(move |a| factory(&a.zone_id))) + .with_state_store(store.clone()); serve(&server, listener); // Two activations per run, so the first run's last token exceeds // the restarted run's first token unless tokens survive restarts. @@ -679,7 +681,7 @@ async fn fenced_lease_keeps_zone_on_one_host_and_loss_stops_it() { for id in ["A", "B"] { let (listener, addr) = listen().await; let f = realm_factory(&[], &[]); - let server = GameServer::::new(config(), move |z| f(z)) + let server = GameServer::::new(config(), ZoneFactory::sync(move |a| f(&a.zone_id))) .with_host_id(id) .with_ownership(ownership.clone()); serve(&server, listener); diff --git a/distributed_game/tests/zone_runtime/arena.rs b/distributed_game/tests/zone_runtime/arena.rs index 9fdc6454d..1c1225536 100644 --- a/distributed_game/tests/zone_runtime/arena.rs +++ b/distributed_game/tests/zone_runtime/arena.rs @@ -21,6 +21,8 @@ pub struct ArenaConfig { pub reject_principal: Option, pub background: bool, pub spawn_every: Option, + /// Allocation payload handed over by the zone factory. + pub allocation: Option, } #[derive(Debug, Clone)] diff --git a/examples/game/server/src/arena/mod.rs b/examples/game/server/src/arena/mod.rs index 3dcafd4bd..2c6e5719d 100644 --- a/examples/game/server/src/arena/mod.rs +++ b/examples/game/server/src/arena/mod.rs @@ -30,7 +30,7 @@ pub use game_sim::arena as movement; use std::collections::BTreeMap; -use distributed_game::net::{GameServer, ServerConfig}; +use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; use distributed_game::{ schema, Ctx, EntitySchema, GameEvent, GameInput, GameSurface, InputCredits, LeaveReason, PlayerId, Replicated, View, WorldView, Zone, @@ -352,10 +352,14 @@ impl Demo for ArenaDemo { } fn server(config: ServerConfig, guests: Guests) -> GameServer { - GameServer::new(config, move |zone_id| { - Self::hosts_zone(zone_id).then(|| ArenaConfig { - guests: guests.clone(), - }) - }) + GameServer::new( + config, + ZoneFactory::sync(move |activation| { + let zone_id = activation.zone_id.as_str(); + Self::hosts_zone(zone_id).then(|| ArenaConfig { + guests: guests.clone(), + }) + }), + ) } } diff --git a/examples/game/server/src/freeze/world.rs b/examples/game/server/src/freeze/world.rs index 2a0376422..7b61dae06 100644 --- a/examples/game/server/src/freeze/world.rs +++ b/examples/game/server/src/freeze/world.rs @@ -31,7 +31,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use distributed_game::net::peer::{post_json, ADMIN_SECRET_HEADER}; use distributed_game::net::{ - GameServer, RemoteHost, RolloutError, ServerConfig, TicketKey, WorldCoordinator, + GameServer, RemoteHost, RolloutError, ServerConfig, TicketKey, WorldCoordinator, ZoneFactory, }; use distributed_game::world::{HostAddr, StaticDirectory}; use distributed_game::{BarrierMode, FreezeState, MemoryFactSink, RetryPolicy}; @@ -120,14 +120,18 @@ fn zone_server_with_sink( sink: MemoryFactSink, ) -> GameServer { let host = host.to_string(); - GameServer::new(config, move |zone_id| { - ZoneKind::of(zone_id).map(|kind| FreezeConfig { - kind, - guests: guests.clone(), - rituals: rituals.clone(), - host: host.clone(), - }) - }) + GameServer::new( + config, + ZoneFactory::sync(move |activation| { + let zone_id = activation.zone_id.as_str(); + ZoneKind::of(zone_id).map(|kind| FreezeConfig { + kind, + guests: guests.clone(), + rituals: rituals.clone(), + host: host.clone(), + }) + }), + ) .with_fact_sink(sink, 256, RetryPolicy::default()) } @@ -202,7 +206,7 @@ pub fn start_host( }; let task = tokio::spawn(server.clone().serve_with(listener, extra, signal)); for zone in &local { - server.activate(zone); + server.begin_activation(zone); } Ok(WorldHost { addr, diff --git a/examples/game/server/src/gallery/mod.rs b/examples/game/server/src/gallery/mod.rs index 005c6a90d..a1f6898dc 100644 --- a/examples/game/server/src/gallery/mod.rs +++ b/examples/game/server/src/gallery/mod.rs @@ -26,7 +26,7 @@ use std::collections::BTreeMap; use std::time::Duration; -use distributed_game::net::{GameServer, ServerConfig}; +use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; use distributed_game::rewind::{Historic, HitQuery, RewindFrame, ShotTiming, Vec3}; use distributed_game::{ schema, ConsumePolicy, Ctx, EntitySchema, FieldType, GameEvent, GameInput, GameRecord, @@ -622,10 +622,14 @@ impl Demo for GalleryDemo { fn server(mut config: ServerConfig, guests: Guests) -> GameServer { config.host.rewind.max_rewind = Duration::from_millis(MAX_REWIND_MS); - GameServer::new(config, move |zone_id| { - Self::hosts_zone(zone_id).then(|| GalleryConfig { - guests: guests.clone(), - }) - }) + GameServer::new( + config, + ZoneFactory::sync(move |activation| { + let zone_id = activation.zone_id.as_str(); + Self::hosts_zone(zone_id).then(|| GalleryConfig { + guests: guests.clone(), + }) + }), + ) } } diff --git a/examples/game/server/src/grid/mod.rs b/examples/game/server/src/grid/mod.rs index ae462487b..3b2002f2f 100644 --- a/examples/game/server/src/grid/mod.rs +++ b/examples/game/server/src/grid/mod.rs @@ -47,7 +47,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use distributed_game::net::{GameServer, ServerConfig}; +use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; use distributed_game::world::{FlatTopology, HashRing, HostAddr, ZoneDirectory}; use distributed_game::{ schema, ConsumePolicy, Ctx, EntitySchema, GameEvent, GameInput, GameSurface, LeaveReason, @@ -643,14 +643,18 @@ impl ClusterDemo for GridDemo { config.host.max_observers = 64; let guests = world.guests.clone(); let host_id = world.host.id.clone(); - GameServer::new(config, move |zone_id| { - world::parse_zone(zone_id).map(|cell| GridConfig { - guests: guests.clone(), - host_id: host_id.clone(), - seed, - cell, - }) - }) + GameServer::new( + config, + ZoneFactory::sync(move |activation| { + let zone_id = activation.zone_id.as_str(); + world::parse_zone(zone_id).map(|cell| GridConfig { + guests: guests.clone(), + host_id: host_id.clone(), + seed, + cell, + }) + }), + ) .with_topology(Arc::new(FlatTopology)) } diff --git a/examples/game/server/src/mobpit/world.rs b/examples/game/server/src/mobpit/world.rs index 278e3287b..351f77c4c 100644 --- a/examples/game/server/src/mobpit/world.rs +++ b/examples/game/server/src/mobpit/world.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use axum::Router; -use distributed_game::net::{GameServer, ServerConfig}; +use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; use distributed_game::RetryPolicy; use serde_json::{json, Value}; @@ -205,12 +205,16 @@ impl MobPitWorld { guests: Guests, ) -> (GameServer, Router) { let world = self.clone(); - let server = GameServer::new(config, move |zone_id| { - MobPitDemo::hosts_zone(zone_id).then(|| MobPitConfig { - guests: guests.clone(), - world: world.clone(), - }) - }) + let server = GameServer::new( + config, + ZoneFactory::sync(move |activation| { + let zone_id = activation.zone_id.as_str(); + MobPitDemo::hosts_zone(zone_id).then(|| MobPitConfig { + guests: guests.clone(), + world: world.clone(), + }) + }), + ) .with_fact_sink(self.fact_sink(), FACT_CAPACITY, RetryPolicy::default()); (server, routes::routes(self.clone())) } diff --git a/examples/game/server/src/ripple/mod.rs b/examples/game/server/src/ripple/mod.rs index 7d1110762..8e86be046 100644 --- a/examples/game/server/src/ripple/mod.rs +++ b/examples/game/server/src/ripple/mod.rs @@ -44,7 +44,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use std::time::Duration; -use distributed_game::net::{GameServer, ServerConfig}; +use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; use distributed_game::world::{HostAddr, ZoneDirectory}; use distributed_game::{ schema, ChildEvent, ConsumePolicy, Ctx, EntitySchema, GameEvent, GameInput, GameRecord, @@ -840,13 +840,17 @@ impl ClusterDemo for RippleDemo { config.host.max_observers = 64; let guests = world.guests.clone(); let host_id = world.host.id.clone(); - GameServer::new(config, move |zone_id| { - tier_of(zone_id).map(|tier| RippleConfig { - guests: guests.clone(), - host_id: host_id.clone(), - tier, - }) - }) + GameServer::new( + config, + ZoneFactory::sync(move |activation| { + let zone_id = activation.zone_id.as_str(); + tier_of(zone_id).map(|tier| RippleConfig { + guests: guests.clone(), + host_id: host_id.clone(), + tier, + }) + }), + ) .with_host_config(|zone_id, host| { if let Some(tier) = tier_of(zone_id) { let (tick, patch, idle) = tier.rates(); diff --git a/js/src/game/messages.ts b/js/src/game/messages.ts index 032c8b58c..685afb641 100644 --- a/js/src/game/messages.ts +++ b/js/src/game/messages.ts @@ -54,7 +54,9 @@ export const GameErrorCode = Object.freeze({ VERSION_MISMATCH: 5, RATE_LIMITED: 6, NOT_FOUND: 7, - RECONNECT_FAILED: 8 + RECONNECT_FAILED: 8, + /** The zone could not be created right now (factory or allocation failure); retry later. */ + ZONE_UNAVAILABLE: 9 }); export type HelloMessage = Readonly<{ From c20225cb0043a79bbd69f3c7e95f836c9576225b Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 00:23:00 -0500 Subject: [PATCH 05/40] feat(game): host-emitted player lifecycle facts With HostConfig::lifecycle_facts the host emits PlayerJoined (ticket, rejoin, handoff), ConnectionLost (grace started), PlayerReconnected, PlayerDisconnected (grace expired) and PlayerLeft { reason } as typed FactCommand inputs bound to the verified principal. Keys use boot epoch, player slot and a per-slot transition counter; a host backlog keeps them in order instead of dropping them on backpressure, and they share the principal's commit lane. New hooks Zone::on_connection_lost and Zone::on_reconnected run at grace start and reconnect. Off by default. Mob Pit enables them and projects presence (/dev/mobpit/presence), ordered by fact stamp so a late handoff leave cannot mark a player offline. [[tasks/gr-p19-zone-lifecycle-facts]] --- distributed_game/Cargo.toml | 5 + distributed_game/README.md | 15 +- distributed_game/src/host.rs | 145 +++++++- distributed_game/src/lib.rs | 1 + distributed_game/src/lifecycle.rs | 209 +++++++++++ distributed_game/src/replay.rs | 2 + distributed_game/src/zone.rs | 12 +- distributed_game/tests/lifecycle/main.rs | 362 +++++++++++++++++++ distributed_game/tests/zone_runtime/arena.rs | 11 + examples/game/README.md | 2 +- examples/game/server/src/mobpit/mod.rs | 3 + examples/game/server/src/mobpit/presence.rs | 140 +++++++ examples/game/server/src/mobpit/routes.rs | 7 + examples/game/server/src/mobpit/store.rs | 23 ++ examples/game/server/src/mobpit/world.rs | 13 + examples/game/server/src/mobpit/zone.rs | 4 + examples/game/server/tests/mobpit_smoke.rs | 95 +++++ 17 files changed, 1042 insertions(+), 7 deletions(-) create mode 100644 distributed_game/src/lifecycle.rs create mode 100644 distributed_game/tests/lifecycle/main.rs create mode 100644 examples/game/server/src/mobpit/presence.rs diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index e9665bb55..c96d43464 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -68,6 +68,11 @@ name = "net_e2e" path = "tests/net_e2e/main.rs" required-features = ["net"] +[[test]] +name = "lifecycle" +path = "tests/lifecycle/main.rs" +required-features = ["runtime"] + [[test]] name = "cell_sink" path = "tests/cell_sink/main.rs" diff --git a/distributed_game/README.md b/distributed_game/README.md index 38d912867..81855fa7b 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -261,7 +261,8 @@ requestAnimationFrame(function frame(now) { ``` The rest of `Zone` is optional hooks: `on_auth`, `on_join`, `on_leave`, -`on_rejoin`, `on_observe`, `on_timer`, `on_fact_settled`, +`on_rejoin`, `on_connection_lost` (grace started), `on_reconnected`, +`on_observe`, `on_timer`, `on_fact_settled`, `on_tier_changed`, `on_dispose`, `hibernate` / `on_restore`, `on_handoff_join`, `on_child_event`, `is_gate`, `freeze_ready`, `on_content_version_changed`, `record_rewind`, `state_hash`. Each is @@ -685,6 +686,18 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), activation's boot epoch for you (see [Durable facts](#durable-facts)). - If the process crashes, each in-flight fact is either fully saved or not saved at all. Mob Pit shows this with `kill -9`. +- **Player lifecycle facts.** With `HostConfig::lifecycle_facts = true` + the host itself emits `lifecycle::PlayerJoined` (ticket, rejoin or + handoff), `ConnectionLost` (grace started; dropped or told to reload), + `PlayerReconnected`, `PlayerDisconnected` (grace expired) and + `PlayerLeft { reason }` as facts bound to the verified principal, so + presence, match admission and abandon need no hand-written facts. Keys + are `{command}:{zone}:{boot_epoch}:p{slot}:{n}` (stable across retries, + new after a restart); they share the principal's commit lane (order kept + with `max_concurrent_commits > 1`) and wait in a host backlog rather + than being dropped when the queue is full. `lifecycle::Lifecycle::of(&fact)` + decodes them in a sink. Mob Pit projects them into a presence read model + (`/dev/mobpit/presence`), ordered by stamp. - Every fact carries a `FactStamp`: zone, activation (boot epoch), tick, host wall time `emitted_at_ms` (`Clock::unix_ms`; `ManualClock` in tests), content version and config version (`HostConfig::config_version`, diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index 66f44177f..3c95be79c 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -26,7 +26,7 @@ //! reported in [`AdvanceReport::transitions`] and via //! [`Zone::on_tier_changed`]. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use std::sync::Arc; use std::time::Duration; @@ -36,6 +36,10 @@ use crate::facts::{EmitError, Fact, FactCommand, FactQueue, FactSettlement}; use crate::handoff::{HandoffError, HandoffRequest, Transfer, TransferIssuer, MAX_CARRIED_BYTES}; use crate::input::{ConsumePolicy, InputBuffer, PushOutcome, SequencedInput}; use crate::interest::{InterestConfig, View}; +use crate::lifecycle::{ + ConnectionLost, JoinVia, LeftReason, LostCause, PlayerDisconnected, PlayerJoined, PlayerLeft, + PlayerReconnected, +}; use crate::protocol::{error, kick, Notice, Role, ServerMessage, Welcome}; use crate::replication::{ClientReplica, DeltaStats, EncodedWorld, WorldView}; use crate::rewind::{Rewind, RewindConfig, RewindFrame}; @@ -97,6 +101,10 @@ pub struct HostConfig { /// runtime with [`ZoneHost::set_config_version`] / /// [`Ctx::set_config_version`]. pub config_version: String, + /// Emit the player lifecycle facts ([`crate::lifecycle`]: joined, + /// connection lost, reconnected, disconnected, left) on the zone's fact + /// queue, bound to each player's verified principal. Off by default. + pub lifecycle_facts: bool, } impl Default for HostConfig { @@ -120,6 +128,7 @@ impl Default for HostConfig { stale_policy: StalePolicy::Strict, boot_epoch: 0, config_version: String::new(), + lifecycle_facts: false, } } } @@ -218,6 +227,8 @@ struct PlayerSlot { reconnect_token: String, session_id: String, content_version: String, + /// Lifecycle facts emitted for this slot (part of their keys). + transitions: u32, } struct ObserverSlot { @@ -267,6 +278,8 @@ pub(crate) struct Core { /// Stamp time forced while a handoff departure runs `on_leave`, so the /// source's departure facts carry exactly the time in the ticket. pinned_stamp_ms: Option, + /// Lifecycle facts waiting for queue capacity, in emission order. + lifecycle_backlog: VecDeque, } impl Core { @@ -296,7 +309,60 @@ impl Core { } fn in_flight(&self) -> usize { - self.facts.as_ref().map_or(0, FactQueue::in_flight) + self.facts.as_ref().map_or(0, FactQueue::in_flight) + self.lifecycle_backlog.len() + } + + /// Queue a lifecycle fact for `principal`'s slot `player` (transition + /// `n`), keyed restart-safely and stamped now. + fn lifecycle(&mut self, player: PlayerId, principal: &str, n: u32, input: C) { + if !self.config.lifecycle_facts || self.facts.is_none() { + return; + } + let key = crate::facts::fact_key( + C::COMMAND, + &self.zone_id, + self.config.boot_epoch, + &[&format_args!("p{}", player.0), &n], + ); + let Ok(fact) = Fact::typed(key, &input) else { + return; + }; + let fact = fact.bind(Some(principal.to_string()), self.stamp()); + self.lifecycle_backlog.push_back(fact); + self.flush_lifecycle(); + } + + /// Send backlogged lifecycle facts while the queue has room. + fn flush_lifecycle(&mut self) { + let Some(queue) = self.facts.as_mut() else { + self.lifecycle_backlog.clear(); + return; + }; + while let Some(fact) = self.lifecycle_backlog.front() { + match queue.try_send(fact.clone()) { + Err(EmitError::Backpressure) => return, + // Sent, or a duplicate already in flight: done either way. + _ => { + self.lifecycle_backlog.pop_front(); + } + } + } + } + + /// Next transition number of a slot. + fn transition(&mut self, player: PlayerId) -> Option<(String, u32)> { + let slot = self.players.get_mut(&player)?; + slot.transitions += 1; + Some((slot.info.principal.clone(), slot.transitions)) + } + + fn emit_lifecycle(&mut self, player: PlayerId, input: C) { + if !self.config.lifecycle_facts { + return; + } + if let Some((principal, n)) = self.transition(player) { + self.lifecycle(player, &principal, n, input); + } } /// Wall time stamped on anything produced now (Unix ms). @@ -775,6 +841,7 @@ impl ZoneHost { fact_seq: (0, 0), stamp_floor_ms: 0, pinned_stamp_ms: None, + lifecycle_backlog: VecDeque::new(), config, }; let zone = Z::on_create(&mut Ctx { core: &mut core }, zone_config); @@ -1066,9 +1133,26 @@ impl ZoneHost { slot.info.extensions = claims.extensions.clone(); } self.bind_player(player, client, capabilities); + let zone = self.core.zone_id.clone(); if let Some((from, carried)) = handoff.take() { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Handoff, + from_zone: Some(from.to_string()), + }, + ); hook!(self.on_handoff_join(player, from, carried)); } else { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Rejoin, + from_zone: None, + }, + ); hook!(self.on_rejoin(player)); } Joined { @@ -1098,12 +1182,34 @@ impl ZoneHost { reconnect_token: String::new(), session_id: (self.token_source)(), content_version: content_version.to_string(), + transitions: 0, }, ); self.bind_player(player, client, capabilities); + let zone = self.core.zone_id.clone(); match handoff.take() { - Some((from, carried)) => hook!(self.on_handoff_join(player, from, carried)), - None => hook!(self.on_join(player)), + Some((from, carried)) => { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Handoff, + from_zone: Some(from.to_string()), + }, + ); + hook!(self.on_handoff_join(player, from, carried)); + } + None => { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Ticket, + from_zone: None, + }, + ); + hook!(self.on_join(player)); + } } Joined { client, @@ -1204,9 +1310,13 @@ impl ZoneHost { slot.content_version = version.to_string(); } } + let zone = self.core.zone_id.clone(); + self.core.emit_lifecycle(player, PlayerReconnected { zone }); + hook!(self.on_reconnected(player)); if reload { self.reload_player(player); } + self.process_kicks(); Ok(Joined { client, viewer: Viewer::Player(player), @@ -1234,9 +1344,21 @@ impl ZoneHost { self.core.clients.remove(&client); self.core.server_message(client, &message); self.core.outbound.push(Outbound::Close { client }); + self.connection_lost(player, LostCause::Reload); } } + /// Grace started: lifecycle fact, then [`Zone::on_connection_lost`]. + fn connection_lost(&mut self, player: PlayerId, cause: LostCause) { + let lost = ConnectionLost { + zone: self.core.zone_id.clone(), + grace_ms: self.core.config.reconnect_grace.as_millis() as u64, + cause, + }; + self.core.emit_lifecycle(player, lost); + hook!(self.on_connection_lost(player)); + } + /// [`reload_player`](Self::reload_player) every connected player whose /// content version differs from the zone's (hard rollout). Returns the /// number reloaded. @@ -1371,6 +1493,7 @@ impl ZoneHost { p.info.connected = false; p.inputs.clear(); p.grace_until = Some(self.core.clock.now() + self.core.config.reconnect_grace); + self.connection_lost(player, LostCause::Dropped); } } Viewer::Observer(observer) => self.remove_observer(observer, None), @@ -1549,6 +1672,19 @@ impl ZoneHost { self.core.outbound.push(Outbound::Close { client }); } hook!(self.on_leave(player, reason)); + if self.core.config.lifecycle_facts { + let zone = self.core.zone_id.clone(); + let (principal, n) = (slot.info.principal, slot.transitions + 1); + match LeftReason::from_leave(reason) { + Some(reason) => { + self.core + .lifecycle(player, &principal, n, PlayerLeft { zone, reason }) + } + None => self + .core + .lifecycle(player, &principal, n, PlayerDisconnected { zone }), + } + } } fn remove_observer(&mut self, observer: ObserverId, last: Option<&ServerMessage>) { @@ -1735,6 +1871,7 @@ impl ZoneHost { let now = self.core.clock.now(); let elapsed = now.saturating_sub(self.last_advance); self.last_advance = now; + self.core.flush_lifecycle(); let expired: Vec<_> = self .core diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index c9f18ba93..fed2fa28a 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -62,6 +62,7 @@ pub mod handoff; pub mod host; pub mod input; pub mod interest; +pub mod lifecycle; #[cfg(feature = "metrics")] pub mod metrics; #[cfg(feature = "net")] diff --git a/distributed_game/src/lifecycle.rs b/distributed_game/src/lifecycle.rs new file mode 100644 index 000000000..f51c2f055 --- /dev/null +++ b/distributed_game/src/lifecycle.rs @@ -0,0 +1,209 @@ +//! Player lifecycle facts emitted by the host itself. +//! +//! With [`HostConfig::lifecycle_facts`](crate::HostConfig::lifecycle_facts) +//! on, every player transition becomes a durable fact on the zone's +//! [`FactQueue`](crate::FactQueue), bound to the player's **verified +//! principal** (never to anything a zone or client wrote): +//! +//! | Fact | Command | When | +//! |---|---|---| +//! | [`PlayerJoined`] | [`PLAYER_JOINED`] | admitted with a ticket, a fresh ticket for a held slot (rejoin) or a transfer ticket (handoff) | +//! | [`ConnectionLost`] | [`CONNECTION_LOST`] | the connection dropped (or was told to reload); the reconnect grace starts | +//! | [`PlayerReconnected`] | [`PLAYER_RECONNECTED`] | a reconnect token resumed the slot within the grace | +//! | [`PlayerDisconnected`] | [`PLAYER_DISCONNECTED`] | the grace expired; the slot is gone | +//! | [`PlayerLeft`] | [`PLAYER_LEFT`] | the slot was removed for any other [`LeftReason`] | +//! +//! Every slot ends with exactly one of [`PlayerLeft`] / [`PlayerDisconnected`]. +//! +//! **Keys** are `"{command}:{zone}:{boot_epoch}:p{slot}:{n}"` where `n` +//! counts the slot's transitions: computed once at emission (a retry keeps +//! it), never repeated within an activation, and distinct after a restart +//! (new boot epoch). **Order** per principal follows emission: the facts +//! travel in the principal's lane of the fact worker (also with +//! `max_concurrent_commits > 1`), and when the queue is full the host keeps +//! them in a backlog and sends them, in order, as capacity frees up — a +//! lifecycle fact is never dropped for backpressure. A `PlayerJoined` is +//! queued before anything the zone emits in `on_join`, a `PlayerLeft` after +//! `on_leave`. A handoff departure is stamped with the time in the transfer +//! ticket, so the target's `PlayerJoined` sorts after the source's +//! `PlayerLeft` ([`FactStamp::emission_order`](crate::FactStamp::emission_order)). + +use serde::{Deserialize, Serialize}; + +use crate::zone::LeaveReason; + +/// Command of [`PlayerJoined`]. +pub const PLAYER_JOINED: &str = "zone.player_joined"; +/// Command of [`PlayerLeft`]. +pub const PLAYER_LEFT: &str = "zone.player_left"; +/// Command of [`ConnectionLost`]. +pub const CONNECTION_LOST: &str = "zone.connection_lost"; +/// Command of [`PlayerReconnected`]. +pub const PLAYER_RECONNECTED: &str = "zone.player_reconnected"; +/// Command of [`PlayerDisconnected`]. +pub const PLAYER_DISCONNECTED: &str = "zone.player_disconnected"; + +/// Every lifecycle command, in the table's order. +pub const LIFECYCLE_COMMANDS: [&str; 5] = [ + PLAYER_JOINED, + CONNECTION_LOST, + PLAYER_RECONNECTED, + PLAYER_DISCONNECTED, + PLAYER_LEFT, +]; + +/// How a player entered the zone. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JoinVia { + /// An admission ticket for a new slot. + Ticket, + /// A fresh ticket for a slot the principal already held (page reload, + /// second tab). + Rejoin, + /// A handoff transfer ticket. + Handoff, +} + +/// Player admitted. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_JOINED)] +pub struct PlayerJoined { + /// Zone id. + pub zone: String, + /// Ticket, rejoin or handoff. + pub via: JoinVia, + /// Source zone of a handoff. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_zone: Option, +} + +/// Why a slot was removed (a grace expiry is [`PlayerDisconnected`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LeftReason { + /// The client left gracefully. + Left, + /// Kicked by the zone or the server (incl. the zone inbox). + Kicked, + /// Moved to another zone. + Handoff, + /// The zone was disposed (shutdown, hibernation with players is + /// impossible, ownership loss). + Disposed, +} + +impl LeftReason { + /// The fact reason for a host leave reason (`None` for `Disconnected`, + /// which is its own fact, and `Rejoined`, which keeps the slot). + pub fn from_leave(reason: LeaveReason) -> Option { + match reason { + LeaveReason::Left => Some(Self::Left), + LeaveReason::Kicked => Some(Self::Kicked), + LeaveReason::Handoff => Some(Self::Handoff), + LeaveReason::Disposed => Some(Self::Disposed), + LeaveReason::Disconnected | LeaveReason::Rejoined => None, + } + } +} + +/// Slot removed (terminal). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_LEFT)] +pub struct PlayerLeft { + /// Zone id. + pub zone: String, + /// Why. + pub reason: LeftReason, +} + +/// What ended the connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LostCause { + /// The transport dropped. + Dropped, + /// The server told the client to reload its content (`Kick(RELOAD)`). + Reload, +} + +/// Connection lost; the slot is kept for `grace_ms`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = CONNECTION_LOST)] +pub struct ConnectionLost { + /// Zone id. + pub zone: String, + /// Reconnect grace (ms). + pub grace_ms: u64, + /// Dropped or told to reload. + pub cause: LostCause, +} + +/// Slot resumed with a reconnect token. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_RECONNECTED)] +pub struct PlayerReconnected { + /// Zone id. + pub zone: String, +} + +/// The reconnect grace expired; the slot is gone (terminal). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_DISCONNECTED)] +pub struct PlayerDisconnected { + /// Zone id. + pub zone: String, +} + +/// Any lifecycle fact input, decoded from a [`Fact`](crate::Fact) or +/// [`FactSettlement`](crate::FactSettlement) by command name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Lifecycle { + /// [`PlayerJoined`]. + Joined(PlayerJoined), + /// [`ConnectionLost`]. + ConnectionLost(ConnectionLost), + /// [`PlayerReconnected`]. + Reconnected(PlayerReconnected), + /// [`PlayerDisconnected`]. + Disconnected(PlayerDisconnected), + /// [`PlayerLeft`]. + Left(PlayerLeft), +} + +impl Lifecycle { + /// Decode `input` of `command`; `None` for other commands or a + /// malformed input. + pub fn decode(command: &str, input: &serde_json::Value) -> Option { + let input = input.clone(); + Some(match command { + PLAYER_JOINED => Self::Joined(serde_json::from_value(input).ok()?), + CONNECTION_LOST => Self::ConnectionLost(serde_json::from_value(input).ok()?), + PLAYER_RECONNECTED => Self::Reconnected(serde_json::from_value(input).ok()?), + PLAYER_DISCONNECTED => Self::Disconnected(serde_json::from_value(input).ok()?), + PLAYER_LEFT => Self::Left(serde_json::from_value(input).ok()?), + _ => return None, + }) + } + + /// Decode a fact. + pub fn of(fact: &crate::Fact) -> Option { + Self::decode(fact.command(), fact.input()) + } + + /// Zone id. + pub fn zone(&self) -> &str { + match self { + Self::Joined(f) => &f.zone, + Self::ConnectionLost(f) => &f.zone, + Self::Reconnected(f) => &f.zone, + Self::Disconnected(f) => &f.zone, + Self::Left(f) => &f.zone, + } + } + + /// True for the terminal facts (the slot is gone). + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Disconnected(_) | Self::Left(_)) + } +} diff --git a/distributed_game/src/replay.rs b/distributed_game/src/replay.rs index 6b6166894..364ebd20e 100644 --- a/distributed_game/src/replay.rs +++ b/distributed_game/src/replay.rs @@ -564,6 +564,7 @@ fn put_config(w: &mut Writer, c: &HostConfig) { }); put_u64(w, c.boot_epoch); w.string(&c.config_version); + w.bool(c.lifecycle_facts); } fn put_claims(w: &mut Writer, claims: &TicketClaims) { @@ -643,6 +644,7 @@ fn get_config(r: &mut Reader<'_>) -> Result { }, boot_epoch: get_u64(r)?, config_version: r.string()?, + lifecycle_facts: r.bool()?, }) } diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 7a52139b2..3431125df 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -224,9 +224,19 @@ pub trait Zone: Sized + Send + 'static { /// A player joined (not called on reconnect). fn on_join(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId) {} - /// A player left for `reason` (not called on a disconnect still inside the grace window). + /// A player left for `reason` (not called on a disconnect still inside + /// the grace window: see [`on_connection_lost`](Self::on_connection_lost)). fn on_leave(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId, _reason: LeaveReason) {} + /// A player's connection dropped (or it was told to reload) and its + /// reconnect grace window started; the slot and its entity stay. + /// `on_leave(Disconnected)` follows if it does not come back in time. + fn on_connection_lost(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId) {} + + /// A player resumed its slot with a reconnect token within the grace + /// window. + fn on_reconnected(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId) {} + /// A fresh ticket (not a reconnect token) arrived for a principal that /// already holds a player slot here, e.g. the page was reloaded or opened /// in another tab with the same identity. The old connection was diff --git a/distributed_game/tests/lifecycle/main.rs b/distributed_game/tests/lifecycle/main.rs new file mode 100644 index 000000000..cbf7f5dd7 --- /dev/null +++ b/distributed_game/tests/lifecycle/main.rs @@ -0,0 +1,362 @@ +//! Host-emitted player lifecycle facts (`HostConfig::lifecycle_facts`): +//! the five transitions, restart-safe keys, per-principal order under +//! concurrent commits, backpressure backlog and the connection hooks. + +#[path = "../zone_runtime/arena.rs"] +mod arena; + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use arena::{Arena, ArenaConfig}; +use distributed_game::lifecycle::{ + JoinVia, LeftReason, Lifecycle, LostCause, CONNECTION_LOST, PLAYER_DISCONNECTED, PLAYER_JOINED, + PLAYER_RECONNECTED, +}; +use distributed_game::protocol::ServerMessage; +use distributed_game::{ + fact_channel, spawn_fact_worker, BoxFuture, ClientId, Fact, FactError, FactFeed, FactReceipt, + FactSink, HostConfig, ManualClock, MemoryFactSink, Outbound, RetryPolicy, Role, TicketClaims, + ZoneHost, +}; + +const MS: fn(u64) -> Duration = Duration::from_millis; + +fn claims(principal: &str, nonce: &str) -> TicketClaims { + TicketClaims::new(principal, "z1", u64::MAX, nonce, Role::Player) +} + +fn config(boot_epoch: u64) -> HostConfig { + HostConfig { + lifecycle_facts: true, + boot_epoch, + reconnect_grace: MS(500), + ..HostConfig::default() + } +} + +fn token(out: &[Outbound], client: ClientId) -> String { + out.iter() + .find_map(|o| match o { + Outbound::Frame { client: c, frame } if *c == client => { + match ServerMessage::decode(frame, &()).ok()? { + ServerMessage::Welcome(w) => Some(w.reconnect_token), + _ => None, + } + } + _ => None, + }) + .expect("welcome") +} + +/// Drive a whole scripted session and return the committed facts. +fn run_session(boot_epoch: u64) -> (Vec, Vec) { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(64, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "z1", + config(boot_epoch), + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + + // ana: join, drop, reconnect, reload the page (fresh ticket), drop, + // grace expires. + let ana = h.join(&claims("ana", "a1"), 0, "").unwrap(); + let ana_token = token(&h.drain_outbound(), ana.client); + h.disconnect(ana.client, false); + let back = h.reconnect(&ana_token, 0).unwrap(); + h.drain_outbound(); + let again = h.join(&claims("ana", "a2"), 0, "").unwrap(); + assert_ne!(back.client, again.client); + h.disconnect(again.client, false); + clock.advance(MS(600)); + h.advance(); + // bo: kicked. cy: leaves gracefully. dee: still here at dispose. + let bo = h.join(&claims("bo", "b1"), 0, "").unwrap(); + let Some(distributed_game::Viewer::Player(bo_id)) = h.viewer_of(bo.client) else { + panic!("bo is a player") + }; + h.kick(bo_id, "cheating"); + let cy = h.join(&claims("cy", "c1"), 0, "").unwrap(); + h.disconnect(cy.client, true); + h.join(&claims("dee", "d1"), 0, "").unwrap(); + h.dispose(); + let hooks = h.zone().connection_events.clone(); + drop(h); + let sink = MemoryFactSink::new(); + block_on(feed.run(sink.clone(), |_| async {})); + (sink.committed(), hooks) +} + +fn block_on(f: impl std::future::Future) { + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + let mut f = std::pin::pin!(f); + while f.as_mut().poll(&mut cx).is_pending() {} +} + +fn of(facts: &[Fact], principal: &str) -> Vec { + facts + .iter() + .filter(|f| f.principal() == Some(principal)) + .map(|f| Lifecycle::of(f).expect("only lifecycle facts")) + .collect() +} + +#[test] +fn the_host_emits_every_transition_bound_to_the_verified_principal() { + let (facts, hooks) = run_session(7); + assert!(facts.iter().all(|f| f.stamp().zone_id == "z1")); + let ana = of(&facts, "ana"); + let commands: Vec<_> = facts + .iter() + .filter(|f| f.principal() == Some("ana")) + .map(Fact::command) + .collect(); + assert_eq!( + commands, + [ + PLAYER_JOINED, + CONNECTION_LOST, + PLAYER_RECONNECTED, + PLAYER_JOINED, + CONNECTION_LOST, + PLAYER_DISCONNECTED + ] + ); + let Lifecycle::Joined(first) = &ana[0] else { + panic!() + }; + assert_eq!(first.via, JoinVia::Ticket); + let Lifecycle::ConnectionLost(lost) = &ana[1] else { + panic!() + }; + assert_eq!((lost.grace_ms, lost.cause), (500, LostCause::Dropped)); + let Lifecycle::Joined(rejoin) = &ana[3] else { + panic!() + }; + assert_eq!(rejoin.via, JoinVia::Rejoin); + assert!(ana[5].is_terminal()); + + let reasons: Vec<_> = ["bo", "cy", "dee"] + .iter() + .map(|p| match of(&facts, p).as_slice() { + [Lifecycle::Joined(_), Lifecycle::Left(left)] => left.reason, + other => panic!("{p}: {other:?}"), + }) + .collect(); + assert_eq!( + reasons, + [LeftReason::Kicked, LeftReason::Left, LeftReason::Disposed] + ); + + // Keys: command, zone, boot epoch, slot, transition counter (ana's + // slot is PlayerId(2): ids are shared with connection ids). + let keys: Vec<_> = facts + .iter() + .filter(|f| f.principal() == Some("ana")) + .map(|f| f.key().to_string()) + .collect(); + assert_eq!(keys[0], format!("{PLAYER_JOINED}:z1:7:p2:1")); + assert_eq!(keys[5], format!("{PLAYER_DISCONNECTED}:z1:7:p2:6")); + // The payload names no identity; the principal is the binding. + assert!(facts.iter().all(|f| f.input().get("principal").is_none())); + + // Connection hooks: grace start and reconnect. + assert_eq!(hooks, ["lost:2", "back:2", "lost:2"]); +} + +#[test] +fn keys_are_stable_per_run_and_distinct_after_a_restart() { + let (first, _) = run_session(100); + let (same, _) = run_session(100); + let (restarted, _) = run_session(101); + let keys = |facts: &[Fact]| { + facts + .iter() + .map(|f| f.key().to_string()) + .collect::>() + }; + assert_eq!(keys(&first), keys(&same), "deterministic per activation"); + let before: std::collections::HashSet<_> = keys(&first).into_iter().collect(); + assert_eq!(before.len(), first.len(), "unique within a run"); + assert!( + keys(&restarted).iter().all(|k| !before.contains(k)), + "a restart never reuses a key" + ); +} + +#[test] +fn off_by_default() { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "z1", + HostConfig::default(), + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + let ana = h.join(&claims("ana", "a1"), 0, "").unwrap(); + h.disconnect(ana.client, false); + let bo = h.join(&claims("bo", "b1"), 0, "").unwrap(); + h.disconnect(bo.client, true); + assert_eq!(h.zone().connection_events, ["lost:2"], "hooks still run"); + drop(h); + let sink = MemoryFactSink::new(); + block_on(feed.run(sink.clone(), |_| async {})); + assert!(sink.committed().is_empty()); +} + +/// Commits slowly for "joined" (so a later fact of the same principal would +/// overtake it if lanes were not respected) and fails the first attempt of +/// every reconnect (retries keep the key). +#[derive(Clone, Default)] +struct SlowSink { + order: Arc>>, + attempts: Arc>>, +} + +impl FactSink for SlowSink { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + let attempt = { + let mut attempts = self.attempts.lock().unwrap(); + let n = attempts.entry(fact.key().to_string()).or_default(); + *n += 1; + *n + }; + let delay = match fact.command() { + PLAYER_JOINED => 40, + CONNECTION_LOST => 5, + _ => 1, + }; + tokio::time::sleep(MS(delay)).await; + if fact.command() == PLAYER_RECONNECTED && attempt == 1 { + return Err(FactError::Retryable("flaky".into())); + } + self.order.lock().unwrap().push(( + fact.principal().unwrap_or_default().to_string(), + fact.command().to_string(), + fact.key().to_string(), + )); + Ok(FactReceipt { + payload: serde_json::Value::Null, + replayed: false, + }) + }) + } +} + +fn concurrent(feed: FactFeed) -> FactFeed { + feed.with_max_concurrent_commits(4) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn per_principal_order_holds_with_concurrent_commits_and_backpressure() { + let clock = ManualClock::new(); + // Capacity 2: most lifecycle facts wait in the host's backlog. + let policy = RetryPolicy { + base_backoff: MS(2), + ..RetryPolicy::default() + }; + let (queue, feed) = fact_channel(2, policy); + let sink = SlowSink::default(); + let worker = spawn_fact_worker(concurrent(feed), sink.clone()); + let mut h = ZoneHost::::new( + "z1", + config(9), + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + + let mut emitted: BTreeMap> = BTreeMap::new(); + for principal in ["ana", "bo", "cy"] { + let joined = h.join(&claims(principal, principal), 0, "").unwrap(); + let token = token(&h.drain_outbound(), joined.client); + h.disconnect(joined.client, false); + h.reconnect(&token, 0).unwrap(); + h.drain_outbound(); + emitted.insert( + principal.into(), + vec![PLAYER_JOINED, CONNECTION_LOST, PLAYER_RECONNECTED], + ); + } + assert!(h.facts_in_flight() > 2, "backlog beyond the queue capacity"); + // The host keeps flushing its backlog as the queue drains. + for _ in 0..400 { + if h.facts_in_flight() == 0 { + break; + } + tokio::time::sleep(MS(5)).await; + clock.advance(MS(5)); + h.advance(); + } + assert_eq!(h.facts_in_flight(), 0, "every lifecycle fact settled"); + drop(h); + worker.await.unwrap(); + + let order = sink.order.lock().unwrap().clone(); + assert_eq!(order.len(), 9, "nothing dropped: {order:?}"); + for (principal, expected) in &emitted { + let committed: Vec<_> = order + .iter() + .filter(|(p, _, _)| p == principal) + .map(|(_, command, _)| command.as_str()) + .collect(); + assert_eq!(&committed, expected, "{principal}"); + } + // The flaky reconnects were retried under the same key. + let attempts = sink.attempts.lock().unwrap(); + for (_, command, key) in order.iter().filter(|(_, c, _)| c == PLAYER_RECONNECTED) { + assert_eq!(attempts[key], 2, "{command} retried once with key {key}"); + } +} + +#[test] +fn handoff_and_reload_are_lifecycle_transitions_too() { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "z1", + HostConfig { + content_version: "v1".into(), + ..config(3) + }, + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + let transfer = distributed_game::Transfer { + claims: claims("ana", "t1"), + source_zone: "z0".into(), + issuer: "h0".into(), + generation: 1, + carried: Vec::new(), + departed_at_ms: 0, + }; + let joined = h.join_transfer(&transfer, 0, "v1").unwrap(); + let Some(distributed_game::Viewer::Player(ana)) = h.viewer_of(joined.client) else { + panic!() + }; + h.reload_player(ana); + h.handoff(ana, "z2", "ticket"); + drop(h); + let sink = MemoryFactSink::new(); + block_on(feed.run(sink.clone(), |_| async {})); + match of(&sink.committed(), "ana").as_slice() { + [Lifecycle::Joined(j), Lifecycle::ConnectionLost(l), Lifecycle::Left(left)] => { + assert_eq!( + (j.via, j.from_zone.as_deref()), + (JoinVia::Handoff, Some("z0")) + ); + assert_eq!(l.cause, LostCause::Reload); + assert_eq!(left.reason, LeftReason::Handoff); + } + other => panic!("{other:?}"), + } +} diff --git a/distributed_game/tests/zone_runtime/arena.rs b/distributed_game/tests/zone_runtime/arena.rs index 1c1225536..7a72b2620 100644 --- a/distributed_game/tests/zone_runtime/arena.rs +++ b/distributed_game/tests/zone_runtime/arena.rs @@ -51,6 +51,8 @@ pub struct Arena { pub tiers: Vec<(Tier, Tier)>, /// Extension claims seen by `on_auth`. pub auth_extensions: Vec, + /// `lost:{player}` / `back:{player}` from the connection hooks. + pub connection_events: Vec, } impl Arena { @@ -119,6 +121,7 @@ impl Zone for Arena { timer_fires: 0, tiers: Vec::new(), auth_extensions: Vec::new(), + connection_events: Vec::new(), } } @@ -152,6 +155,14 @@ impl Zone for Arena { self.log.push(format!("leave:{reason:?}")); } + fn on_connection_lost(&mut self, _ctx: &mut Ctx<'_, Self>, player: PlayerId) { + self.connection_events.push(format!("lost:{}", player.0)); + } + + fn on_reconnected(&mut self, _ctx: &mut Ctx<'_, Self>, player: PlayerId) { + self.connection_events.push(format!("back:{}", player.0)); + } + fn on_observe(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId) { ctx.set_view(observer, View::new(50.0, 50.0, 1000.0)); let info = ctx diff --git a/examples/game/README.md b/examples/game/README.md index 8454a8223..ce7ce5f47 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -9,7 +9,7 @@ guests and lets any page change the network simulation. | Demo | Run | Mechanic it demonstrates | |---|---|---| | **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | -| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player native SQLite cells (`cell-sqlite`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery | +| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player native SQLite cells (`cell-sqlite`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence (`/dev/mobpit/presence`) projected from the runtime's player lifecycle facts | | **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | | **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | | **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | diff --git a/examples/game/server/src/mobpit/mod.rs b/examples/game/server/src/mobpit/mod.rs index 850d89706..e3301f85e 100644 --- a/examples/game/server/src/mobpit/mod.rs +++ b/examples/game/server/src/mobpit/mod.rs @@ -11,6 +11,8 @@ //! `distributed_game::cell::CellFactSink`. //! - [`board`]: kills/XP leaderboards (in-memory or Redis), fire-and-forget //! live updates, idempotent apply keys, rebuild from the cells' events. +//! - [`presence`]: who is online where, projected from the runtime's player +//! lifecycle facts (`HostConfig::lifecycle_facts`), ordered by fact stamp. //! - [`routes`]: dev routes (`/dev/mobpit/*`). //! //! `game-demos mobpit` and `game-demos leaderboard` start the same server @@ -21,6 +23,7 @@ pub mod board; pub mod crawler; +pub mod presence; pub mod routes; pub mod store; pub mod world; diff --git a/examples/game/server/src/mobpit/presence.rs b/examples/game/server/src/mobpit/presence.rs new file mode 100644 index 000000000..eaabe1e40 --- /dev/null +++ b/examples/game/server/src/mobpit/presence.rs @@ -0,0 +1,140 @@ +//! Presence read model projected from the runtime's player lifecycle facts +//! (`HostConfig::lifecycle_facts`, `distributed_game::lifecycle`). +//! +//! The fact sink hands every lifecycle fact to [`Presence::apply`] instead +//! of a Crawler cell. Facts commit in per-principal order within one zone, +//! but across zones (a handoff, a reconnect elsewhere) commit order says +//! nothing: each entry keeps the [`FactStamp`] of the fact that set it and a +//! fact that was *emitted* earlier (`FactStamp::emission_order`) is ignored, +//! so "left pit-1" committed after "joined pit-2" cannot mark a player +//! offline. A retried fact (same key) applies idempotently. + +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::sync::Mutex; + +use distributed_game::lifecycle::Lifecycle; +use distributed_game::FactStamp; +use serde::Serialize; + +/// Where a player is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PresenceStatus { + /// Connected to `zone`. + Online, + /// Connection lost; `zone` keeps the slot for the reconnect grace. + Away, + /// Left `zone` (or its grace expired). + Offline, +} + +/// One player's presence. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PresenceEntry { + /// Verified principal (the fact's binding, never its payload). + pub principal: String, + /// Zone of the latest transition. + pub zone: String, + /// Status after it. + pub status: PresenceStatus, + /// Host wall time of that transition (Unix ms). + pub since_ms: u64, + #[serde(skip)] + stamp: FactStamp, +} + +/// The projection. +#[derive(Debug, Default)] +pub struct Presence { + entries: Mutex>, +} + +impl Presence { + /// Apply one lifecycle fact of `principal`. False when an entry set by + /// a later-emitted fact already exists (stale, ignored). + pub fn apply(&self, principal: &str, stamp: &FactStamp, event: &Lifecycle) -> bool { + let status = match event { + Lifecycle::Joined(_) | Lifecycle::Reconnected(_) => PresenceStatus::Online, + Lifecycle::ConnectionLost(_) => PresenceStatus::Away, + Lifecycle::Left(_) | Lifecycle::Disconnected(_) => PresenceStatus::Offline, + }; + let mut entries = self.entries.lock().expect("presence"); + if let Some(current) = entries.get(principal) { + if current.stamp.emission_order(stamp) != Ordering::Less { + return false; + } + } + entries.insert( + principal.to_string(), + PresenceEntry { + principal: principal.to_string(), + zone: event.zone().to_string(), + status, + since_ms: stamp.emitted_at_ms, + stamp: stamp.clone(), + }, + ); + true + } + + /// One player. + pub fn get(&self, principal: &str) -> Option { + self.entries + .lock() + .expect("presence") + .get(principal) + .cloned() + } + + /// Everyone seen, by principal. + pub fn snapshot(&self) -> Vec { + self.entries + .lock() + .expect("presence") + .values() + .cloned() + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use distributed_game::lifecycle::{JoinVia, LeftReason, PlayerJoined, PlayerLeft}; + + fn stamp(zone: &str, activation: u64, tick: u64, at: u64) -> FactStamp { + FactStamp { + zone_id: zone.into(), + tick, + emitted_at_ms: at, + activation, + ..FactStamp::default() + } + } + + #[test] + fn a_handoff_leave_committed_late_does_not_mark_the_player_offline() { + let presence = Presence::default(); + let joined_b = Lifecycle::Joined(PlayerJoined { + zone: "pit-2".into(), + via: JoinVia::Handoff, + from_zone: Some("pit-1".into()), + }); + let left_a = Lifecycle::Left(PlayerLeft { + zone: "pit-1".into(), + reason: LeftReason::Handoff, + }); + // pit-2 commits first; pit-1's departure (emitted earlier) after. + assert!(presence.apply("ana", &stamp("pit-2", 4, 2, 1_001), &joined_b)); + assert!(!presence.apply("ana", &stamp("pit-1", 3, 90, 1_000), &left_a)); + let ana = presence.get("ana").unwrap(); + assert_eq!( + (ana.zone.as_str(), ana.status), + ("pit-2", PresenceStatus::Online) + ); + // Replaying the same fact is a no-op. + assert!(!presence.apply("ana", &stamp("pit-2", 4, 2, 1_001), &joined_b)); + } +} diff --git a/examples/game/server/src/mobpit/routes.rs b/examples/game/server/src/mobpit/routes.rs index 416b74c00..156636ea2 100644 --- a/examples/game/server/src/mobpit/routes.rs +++ b/examples/game/server/src/mobpit/routes.rs @@ -10,6 +10,8 @@ //! snapshot, plus rank / neighbours of `member` on both boards. //! - `POST /dev/mobpit/leaderboard/rebuild` — rebuild both boards from the //! cells' event history; reports whether the result matches the cells. +//! - `GET /dev/mobpit/presence` — who is online / away / offline where, +//! projected from the runtime's player lifecycle facts. use std::collections::HashMap; use std::sync::Arc; @@ -41,6 +43,7 @@ pub fn routes(world: Arc) -> Router { .route("/dev/mobpit/crawlers", get(crawlers)) .route("/dev/mobpit/leaderboard", get(leaderboard)) .route("/dev/mobpit/leaderboard/rebuild", post(rebuild)) + .route("/dev/mobpit/presence", get(presence)) .with_state(world) } @@ -99,6 +102,10 @@ async fn leaderboard( Ok(Json(json!({ "snapshot": snapshot, "me": me }))) } +async fn presence(State(world): State>) -> Json { + Json(json!({ "presence": world.presence().snapshot() })) +} + async fn rebuild(State(world): State>) -> ApiResult { let report = world.rebuild_board().await.map_err(internal)?; Ok(Json(serde_json::to_value(report).map_err(internal)?)) diff --git a/examples/game/server/src/mobpit/store.rs b/examples/game/server/src/mobpit/store.rs index aebfd0e47..e4c6dec17 100644 --- a/examples/game/server/src/mobpit/store.rs +++ b/examples/game/server/src/mobpit/store.rs @@ -30,11 +30,13 @@ use distributed::entity::EventRecord; use distributed::BusPublisher; use distributed::{RepositoryError, StreamIdentity}; use distributed_game::cell::CellFactSink; +use distributed_game::lifecycle::Lifecycle; use distributed_game::{BoxFuture, Fact, FactError, FactReceipt, FactSink}; use serde::Serialize; use super::board::Leaderboards; use super::crawler::{Crawler, KillReceipt, RecordKill, AGGREGATE_TYPE, RECORD_KILL}; +use super::presence::Presence; /// Command-ledger service id for game facts. pub const SERVICE_ID: &str = "mobpit"; @@ -262,6 +264,8 @@ pub struct SinkStats { pub errors: u64, /// Facts currently inside `commit` (incl. the artificial latency). pub in_sink: u64, + /// Lifecycle facts projected into presence (not counted above). + pub presence: u64, } /// Shared sink state (lives in the world). @@ -312,6 +316,7 @@ pub struct CrawlerFactSink { state: Arc, board: Arc, projection: Arc>, + presence: Arc, } impl CrawlerFactSink { @@ -327,9 +332,16 @@ impl CrawlerFactSink { state, board, projection, + presence: Arc::default(), } } + /// Project lifecycle facts into `presence` (instead of a private one). + pub fn with_presence(mut self, presence: Arc) -> Self { + self.presence = presence; + self + } + async fn commit_inner(&self, fact: &Fact) -> Result { let latency = self.state.latency(); if !latency.is_zero() { @@ -338,6 +350,7 @@ impl CrawlerFactSink { let principal = fact .principal() .ok_or_else(|| FactError::Permanent("kill facts need a player principal".into()))?; + let cell = self .cells .get(principal) @@ -376,6 +389,16 @@ impl CrawlerFactSink { impl FactSink for CrawlerFactSink { fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { Box::pin(async move { + // Runtime lifecycle facts feed the presence read model, not a + // cell (and not the kill counters). + if let (Some(event), Some(principal)) = (Lifecycle::of(fact), fact.principal()) { + let applied = self.presence.apply(principal, fact.stamp(), &event); + self.state.update(|s| s.presence += 1); + return Ok(FactReceipt { + payload: serde_json::json!({ "presence": applied }), + replayed: !applied, + }); + } self.state.update(|s| { s.attempts += 1; s.in_sink += 1; diff --git a/examples/game/server/src/mobpit/world.rs b/examples/game/server/src/mobpit/world.rs index 351f77c4c..5656bc7be 100644 --- a/examples/game/server/src/mobpit/world.rs +++ b/examples/game/server/src/mobpit/world.rs @@ -13,6 +13,7 @@ use distributed_game::RetryPolicy; use serde_json::{json, Value}; use super::board::Leaderboards; +use super::presence::Presence; use super::store::{CrawlerCells, CrawlerFactSink, ProjectionBus, SinkState}; use super::zone::{MobPit, MobPitConfig}; use super::{routes, MobPitDemo}; @@ -60,6 +61,7 @@ pub struct MobPitWorld { zone_pending: AtomicUsize, profiles: Arc>>, projection: Arc>, + presence: Arc, } impl MobPitWorld { @@ -77,6 +79,7 @@ impl MobPitWorld { zone_pending: AtomicUsize::new(0), profiles: Arc::default(), projection: Arc::default(), + presence: Arc::default(), })) } @@ -106,6 +109,11 @@ impl MobPitWorld { &self.board } + /// Presence projected from the zones' lifecycle facts. + pub fn presence(&self) -> &Arc { + &self.presence + } + /// The game's fact sink. pub fn fact_sink(&self) -> CrawlerFactSink { CrawlerFactSink::new( @@ -114,6 +122,7 @@ impl MobPitWorld { self.board.clone(), self.projection.clone(), ) + .with_presence(self.presence.clone()) } /// Publish the cells' Crawler domain events to the SQLite bus at `url` @@ -205,6 +214,10 @@ impl MobPitWorld { guests: Guests, ) -> (GameServer, Router) { let world = self.clone(); + let mut config = config; + // Joined / connection lost / reconnected / disconnected / left facts + // for the presence projection. + config.host.lifecycle_facts = true; let server = GameServer::new( config, ZoneFactory::sync(move |activation| { diff --git a/examples/game/server/src/mobpit/zone.rs b/examples/game/server/src/mobpit/zone.rs index 22375c768..d615146bc 100644 --- a/examples/game/server/src/mobpit/zone.rs +++ b/examples/game/server/src/mobpit/zone.rs @@ -960,6 +960,10 @@ impl Zone for MobPit { } fn on_fact_settled(&mut self, ctx: &mut Ctx<'_, Self>, settlement: FactSettlement) { + // The host's own lifecycle facts (presence) are not kills. + if distributed_game::lifecycle::LIFECYCLE_COMMANDS.contains(&settlement.command.as_str()) { + return; + } let pending = self.pending.remove(&settlement.key); let (principal, x, y, waited_ms) = match &pending { Some(p) => ( diff --git a/examples/game/server/tests/mobpit_smoke.rs b/examples/game/server/tests/mobpit_smoke.rs index 338c6f3ba..d31089627 100644 --- a/examples/game/server/tests/mobpit_smoke.rs +++ b/examples/game/server/tests/mobpit_smoke.rs @@ -542,3 +542,98 @@ async fn redis_board_projects_and_rebuilds_when_redis_url_is_set() { } let _ = std::fs::remove_dir_all(dir); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn presence_is_projected_from_the_runtime_lifecycle_facts() { + use distributed_game::protocol::Hello; + use distributed_game::{ClientMessage, ClientMirror, Role, ServerMessage, TicketClaims}; + use futures_util::{SinkExt, StreamExt}; + use game_demos::mobpit::presence::PresenceStatus; + use tokio_tungstenite::tungstenite::Message; + + let dir = temp_dir("presence"); + let world = MobPitWorld::open(WorldOptions::new(&dir)).unwrap(); + let (demo, stop) = start(world.clone()).await; + let join = |principal: &'static str| { + let url = format!("{}/zones/pit-1", demo.ws_url()); + async move { + let (mut ws, _) = tokio_tungstenite::connect_async(url).await.unwrap(); + let ticket = TicketKey::new(SECRET).sign(&TicketClaims::new( + principal, + "pit-1", + 4_000_000_000, + format!("presence-{principal}"), + Role::Player, + )); + let hello = ClientMessage::Hello(Hello { + protocol_version: distributed_game::PROTOCOL_VERSION, + content_version: String::new(), + capabilities: 0, + ticket, + reconnect_token: String::new(), + }); + ws.send(Message::Binary(hello.encode().into())) + .await + .unwrap(); + let mut mirror = ClientMirror::new(); + loop { + match ws.next().await.unwrap().unwrap() { + Message::Binary(bytes) => { + if let ServerMessage::Welcome(_) = mirror.apply(&bytes).unwrap() { + break; + } + } + _ => continue, + } + } + ws + } + }; + let status = |principal: &'static str| { + let world = world.clone(); + move || { + let world = world.clone(); + async move { + world + .presence() + .get(principal) + .map(|e| (e.zone.clone(), e.status)) + } + } + }; + let mut ana = join("guest-ana").await; + let bo = join("guest-bo").await; + let online = Some(("pit-1".to_string(), PresenceStatus::Online)); + eventually("ana online", Duration::from_secs(5), { + let s = status("guest-ana"); + let online = online.clone(); + move || { + let (s, online) = (s.clone(), online.clone()); + async move { s().await == online } + } + }) + .await; + // Graceful leave → offline; a dropped socket → away (grace running). + ana.send(Message::Binary(ClientMessage::Leave.encode().into())) + .await + .unwrap(); + drop(bo); + eventually("ana offline, bo away", Duration::from_secs(5), { + let (a, b) = (status("guest-ana"), status("guest-bo")); + move || { + let (a, b) = (a.clone(), b.clone()); + async move { + a().await == Some(("pit-1".into(), PresenceStatus::Offline)) + && b().await == Some(("pit-1".into(), PresenceStatus::Away)) + } + } + }) + .await; + let (code, body) = http(demo.addr, "GET", "/dev/mobpit/presence", None).await; + assert_eq!(code, 200); + assert_eq!(body["presence"].as_array().unwrap().len(), 2, "{body}"); + // Lifecycle facts never touch the Crawler cells. + assert!(world.cells().principals().is_empty()); + let _ = stop.send(()); + let _ = std::fs::remove_dir_all(dir); +} From d9a3fd9903a12de10664179a5e8e47de791952ad Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 00:35:42 -0500 Subject: [PATCH 06/40] feat(game): typed zone inbox with kick and admin route Zones declare `type Message` (a GameEvent-derived type, or NoMessages) and handle Zone::on_message on their own task between ticks. ZoneHost::deliver takes ZoneInbound::{Kick, Message}; the built-in kick removes every connection of a principal with LeaveReason::Kicked (and a PlayerLeft lifecycle fact). GameServer::send delivers locally; POST /admin/zone-message (admin secret) serves peers and services, answering 404 for inactive zones (never activating them) and 421 wrong_owner with the owner's admin URL. net::ZoneInboxClient resolves the owner via ZoneDirectory and follows those answers. Deliveries are recorded and replayed. Messages appear in the game surface (`messages`) and the generated TypeScript gains MESSAGE, MESSAGES and encodeMessage. [[tasks/gr-p21-zone-inbox]] --- distributed_cli/src/game_client.rs | 154 ++++--- .../fixtures/game-client/sample/events.ts | 53 +++ .../fixtures/game-client/sample/surface.json | 34 ++ .../tests/fixtures/game-surface/src/lib.rs | 1 + distributed_game/Cargo.toml | 5 + distributed_game/README.md | 18 + distributed_game/examples/echo_arena.rs | 1 + distributed_game/src/host.rs | 43 ++ distributed_game/src/inbox.rs | 69 +++ distributed_game/src/lib.rs | 2 + distributed_game/src/metrics.rs | 1 + distributed_game/src/net/inbox.rs | 300 +++++++++++++ distributed_game/src/net/mod.rs | 62 +++ distributed_game/src/net/world.rs | 37 ++ distributed_game/src/replay.rs | 86 ++++ distributed_game/src/surface.rs | 35 +- distributed_game/src/zone.rs | 9 + distributed_game/tests/authoring/main.rs | 29 ++ .../tests/fixtures/authoring.json | 54 +++ distributed_game/tests/inbox/main.rs | 398 ++++++++++++++++++ distributed_game/tests/netcode/main.rs | 1 + distributed_game/tests/world/realm.rs | 1 + distributed_game/tests/zone_runtime/arena.rs | 1 + distributed_game/tests/zone_runtime/main.rs | 1 + examples/game/server/src/arena/mod.rs | 1 + examples/game/server/src/freeze/mod.rs | 1 + examples/game/server/src/gallery/mod.rs | 1 + examples/game/server/src/grid/mod.rs | 1 + examples/game/server/src/mobpit/zone.rs | 1 + examples/game/server/src/ripple/mod.rs | 1 + js/tests/game-generated.test.mjs | 12 + 31 files changed, 1364 insertions(+), 49 deletions(-) create mode 100644 distributed_game/src/inbox.rs create mode 100644 distributed_game/src/net/inbox.rs create mode 100644 distributed_game/tests/inbox/main.rs diff --git a/distributed_cli/src/game_client.rs b/distributed_cli/src/game_client.rs index 75eecb8b8..af0bb32c1 100644 --- a/distributed_cli/src/game_client.rs +++ b/distributed_cli/src/game_client.rs @@ -9,7 +9,7 @@ //! |---|---| //! | `schema.ts` | `KIND` ids, per-kind field interfaces + accessors over rendered entities, field wire types, `CORRECTIONS` defaults | //! | `input.ts` | the input interface, `BUTTON` bits, `encodeInput` / `decodeInput` / `sanitizeInput` | -//! | `events.ts` | record + event payload interfaces, `EVENT` ids, `EVENTS` codecs, `decodeEvent`, `onEvent`; `encode` / `decode` for the surface's standalone codecs | +//! | `events.ts` | record + event payload interfaces, `EVENT` ids, `EVENTS` codecs, `decodeEvent`, `onEvent`; `encode` / `decode` for the surface's standalone codecs; with zone inbox messages, `Message` interfaces, `MESSAGE` ids, `MESSAGES` codecs and `encodeMessage` (for services posting to `/admin/zone-message`) | //! | `client.ts` | `createClient`: a `GameClient` with the input encoder and correction defaults wired in, plus typed `on`; with a predicted step, prediction defaults to the compiled WebAssembly step | //! | `predict.ts` | only with a `#[predict]` step: its state/context layouts, the module URL (`wasm/.wasm`, built by `distributed game-wasm`) and `createPrediction` | //! | `pures.ts` | only with `#[game_pure]` functions: `createPures`, one typed wrapper per function over the same module | @@ -77,6 +77,8 @@ struct Surface { constants: Vec, #[serde(default)] codecs: Vec, + #[serde(default)] + messages: Vec, } /// A pure function compiled into the surface's module (`#[game_pure]`). @@ -1168,11 +1170,14 @@ impl<'a> Gen<'a> { "type GameEventCodecs", "type GameEventSource", ]; + let messages = &self.surface.messages; let uses_fields = events .iter() + .chain(messages) .any(|e| matches!(e.payload, Payload::Fields { .. })); let uses_value = events .iter() + .chain(messages) .any(|e| matches!(e.payload, Payload::Value { .. })); if uses_fields { sdk.extend(["decodePayload", "encodePayload", "type PayloadLayout"]); @@ -1220,51 +1225,9 @@ impl<'a> Gen<'a> { ); } - for event in events { - let type_name = format!("{}Event", pascal(&event.name)); - out.push('\n'); - match &event.payload { - Payload::Fields { fields } if fields.is_empty() => { - let _ = writeln!( - out, - "/** Event `{}` (id {}): no payload. */\nexport type {type_name} = Readonly>;", - event.name, event.id - ); - let _ = writeln!( - out, - "\nconst {}_LAYOUT: PayloadLayout = [];", - constant(&event.name) - ); - } - Payload::Fields { fields } => { - out.push_str(&self.interface( - &type_name, - &format!("Event `{}` (id {}).", event.name, event.id), - fields, - )); - let _ = writeln!( - out, - "\nconst {}_LAYOUT: PayloadLayout = {};", - constant(&event.name), - self.layout(fields) - ); - } - Payload::Value { ty } => { - let _ = writeln!( - out, - "/** Event `{}` (id {}). */\nexport type {type_name} = {};", - event.name, - event.id, - self.ts_type(ty) - ); - let _ = writeln!( - out, - "\nconst {}_PAYLOAD: WireType = {};", - constant(&event.name), - self.wire_literal(ty) - ); - } - } + self.payload_types(&mut out, events, "Event", ""); + if !messages.is_empty() { + self.payload_types(&mut out, messages, "Message", "MESSAGE_"); } out.push_str("\n/** Event type ids by name. */\nexport const EVENT = {\n"); @@ -1310,9 +1273,106 @@ impl<'a> Gen<'a> { out.push_str("/** A decoded event, discriminated by `name`. */\nexport type AnyEvent = DecodedGameEvent;\n\n"); out.push_str("/** Decode an Event message (`undefined` for ids this surface does not declare). */\nexport function decodeEvent(event: { readonly typeId: number; readonly payload: Uint8Array }): AnyEvent | undefined {\n\treturn decodeGameEvent(EVENTS, event);\n}\n\n"); out.push_str("/** Subscribe to one typed event on every zone of `client`; returns the unsubscribe. */\nexport function onEvent(\n\tclient: GameEventSource,\n\tname: K,\n\thandler: (value: Events[K], zoneId: string) => void\n): () => void {\n\treturn onGameEvent(client, EVENTS, name, handler);\n}\n"); + if !messages.is_empty() { + self.messages_ts(&mut out, messages); + } Ok(out) } + /// Zone inbox message ids, codecs and `encodeMessage` (the body of a + /// `/admin/zone-message` `message`). + fn messages_ts(&self, out: &mut String, messages: &[Event]) { + out.push_str("\n/** Zone inbox message type ids by name. */\nexport const MESSAGE = {\n"); + for m in messages { + let _ = writeln!(out, "\t{}: {},", key(&camel(&m.name)), m.id); + } + out.push_str("} as const;\n\n/** Zone inbox message payload types by name. */\nexport interface Messages {\n"); + for m in messages { + let _ = writeln!( + out, + "\t{}: {}Message;", + key(&camel(&m.name)), + pascal(&m.name) + ); + } + out.push_str("}\n\n/** Codecs of every zone inbox message. */\nexport const MESSAGES: GameEventCodecs = {\n"); + for m in messages { + let type_name = format!("{}Message", pascal(&m.name)); + let c = format!("MESSAGE_{}", constant(&m.name)); + let (encode, decode) = match &m.payload { + Payload::Fields { .. } => ( + format!("(value) => encodePayload({c}_LAYOUT, value)"), + format!( + "(payload) => decodePayload({c}_LAYOUT, payload) as unknown as {type_name}" + ), + ), + Payload::Value { .. } => ( + format!("(value) => encodeValue({c}_PAYLOAD, value)"), + format!("(payload) => decodeValue({c}_PAYLOAD, payload) as {type_name}"), + ), + }; + let _ = writeln!( + out, + "\t{}: {{\n\t\tid: {},\n\t\tname: {},\n\t\tencode: {encode},\n\t\tdecode: {decode}\n\t}},", + key(&camel(&m.name)), + m.id, + string(&m.name) + ); + } + out.push_str("};\n\n/** Encode a zone inbox message: `typeId` and `payload` of a `/admin/zone-message` body (base64url the payload). */\nexport function encodeMessage(\n\tname: K,\n\tvalue: Messages[K]\n): { readonly typeId: number; readonly payload: Uint8Array } {\n\tconst codec = MESSAGES[name];\n\treturn { typeId: codec.id, payload: codec.encode(value) };\n}\n"); + } + + /// Payload interfaces and layouts of events (`suffix` `Event`) or zone + /// inbox messages (`Message`, constants prefixed `MESSAGE_`). + fn payload_types(&self, out: &mut String, items: &[Event], suffix: &str, prefix: &str) { + for event in items { + let type_name = format!("{}{suffix}", pascal(&event.name)); + out.push('\n'); + match &event.payload { + Payload::Fields { fields } if fields.is_empty() => { + let _ = writeln!( + out, + "/** {suffix} `{}` (id {}): no payload. */\nexport type {type_name} = Readonly>;", + event.name, event.id + ); + let _ = writeln!( + out, + "\nconst {prefix}{}_LAYOUT: PayloadLayout = [];", + constant(&event.name) + ); + } + Payload::Fields { fields } => { + out.push_str(&self.interface( + &type_name, + &format!("{suffix} `{}` (id {}).", event.name, event.id), + fields, + )); + let _ = writeln!( + out, + "\nconst {prefix}{}_LAYOUT: PayloadLayout = {};", + constant(&event.name), + self.layout(fields) + ); + } + Payload::Value { ty } => { + let _ = writeln!( + out, + "/** {suffix} `{}` (id {}). */\nexport type {type_name} = {};", + event.name, + event.id, + self.ts_type(ty) + ); + let _ = writeln!( + out, + "\nconst {prefix}{}_PAYLOAD: WireType = {};", + constant(&event.name), + self.wire_literal(ty) + ); + } + } + } + } + /// Records an event payload (or the input) encodes, directly or nested. fn encoded_records(&self) -> BTreeSet<&'a str> { fn walk<'r>(ty: &'r Wire, out: &mut Vec<&'r str>) { @@ -1323,7 +1383,7 @@ impl<'a> Gen<'a> { } } let mut pending = Vec::new(); - for event in &self.surface.events { + for event in self.surface.events.iter().chain(&self.surface.messages) { match &event.payload { Payload::Fields { fields } => fields.iter().for_each(|f| walk(&f.ty, &mut pending)), Payload::Value { ty } => walk(ty, &mut pending), diff --git a/distributed_cli/tests/fixtures/game-client/sample/events.ts b/distributed_cli/tests/fixtures/game-client/sample/events.ts index d94e2bddc..3febc862a 100644 --- a/distributed_cli/tests/fixtures/game-client/sample/events.ts +++ b/distributed_cli/tests/fixtures/game-client/sample/events.ts @@ -130,6 +130,22 @@ const PING_LAYOUT: PayloadLayout = [ ['seq', { type: 'u32' }], ]; +/** Message `announce` (id 1). */ +export interface AnnounceMessage { + readonly text: string; + readonly urgent: boolean; +} + +const MESSAGE_ANNOUNCE_LAYOUT: PayloadLayout = [ + ['text', { type: 'str' }], + ['urgent', { type: 'bool' }], +]; + +/** Message `set_wind` (id 2). */ +export type SetWindMessage = SampleWind; + +const MESSAGE_SET_WIND_PAYLOAD: WireType = SAMPLE_WIND_TYPE; + /** Event type ids by name. */ export const EVENT = { hello: 1, @@ -198,3 +214,40 @@ export function onEvent( ): () => void { return onGameEvent(client, EVENTS, name, handler); } + +/** Zone inbox message type ids by name. */ +export const MESSAGE = { + announce: 1, + setWind: 2, +} as const; + +/** Zone inbox message payload types by name. */ +export interface Messages { + announce: AnnounceMessage; + setWind: SetWindMessage; +} + +/** Codecs of every zone inbox message. */ +export const MESSAGES: GameEventCodecs = { + announce: { + id: 1, + name: 'announce', + encode: (value) => encodePayload(MESSAGE_ANNOUNCE_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_ANNOUNCE_LAYOUT, payload) as unknown as AnnounceMessage + }, + setWind: { + id: 2, + name: 'set_wind', + encode: (value) => encodeValue(MESSAGE_SET_WIND_PAYLOAD, value), + decode: (payload) => decodeValue(MESSAGE_SET_WIND_PAYLOAD, payload) as SetWindMessage + }, +}; + +/** Encode a zone inbox message: `typeId` and `payload` of a `/admin/zone-message` body (base64url the payload). */ +export function encodeMessage( + name: K, + value: Messages[K] +): { readonly typeId: number; readonly payload: Uint8Array } { + const codec = MESSAGES[name]; + return { typeId: codec.id, payload: codec.encode(value) }; +} diff --git a/distributed_cli/tests/fixtures/game-client/sample/surface.json b/distributed_cli/tests/fixtures/game-client/sample/surface.json index 6629d2703..892008aaa 100644 --- a/distributed_cli/tests/fixtures/game-client/sample/surface.json +++ b/distributed_cli/tests/fixtures/game-client/sample/surface.json @@ -426,6 +426,40 @@ "priority": 1.0 } ], + "messages": [ + { + "id": 1, + "name": "announce", + "payload": { + "fields": [ + { + "name": "text", + "type": { + "type": "str" + } + }, + { + "name": "urgent", + "type": { + "type": "bool" + } + } + ], + "kind": "fields" + } + }, + { + "id": 2, + "name": "set_wind", + "payload": { + "kind": "value", + "type": { + "name": "SampleWind", + "type": "record" + } + } + } + ], "name": "sample", "predict": { "abi": 1, diff --git a/distributed_cli/tests/fixtures/game-surface/src/lib.rs b/distributed_cli/tests/fixtures/game-surface/src/lib.rs index 2712e56c4..75cdfdb42 100644 --- a/distributed_cli/tests/fixtures/game-surface/src/lib.rs +++ b/distributed_cli/tests/fixtures/game-surface/src/lib.rs @@ -62,6 +62,7 @@ impl Zone for Orbs { type Config = (); type Input = Steer; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![OrbView] diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index c96d43464..8b2e35f47 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -68,6 +68,11 @@ name = "net_e2e" path = "tests/net_e2e/main.rs" required-features = ["net"] +[[test]] +name = "inbox" +path = "tests/inbox/main.rs" +required-features = ["net"] + [[test]] name = "lifecycle" path = "tests/lifecycle/main.rs" diff --git a/distributed_game/README.md b/distributed_game/README.md index 81855fa7b..5a14c9493 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -169,6 +169,7 @@ impl Zone for Room { type Config = (); type Input = Push; type Timer = (); + type Message = NoMessages; // or a #[derive(GameEvent)] type (zone inbox) fn schema() -> EntitySchema { schema![DotView] } @@ -1008,6 +1009,23 @@ principals commit concurrently; settlements arrive in completion order. 2 Hz while its neighborhoods run at 30 Hz); the world content version is applied afterwards. +**Zone inbox.** Services talk to a running zone through typed messages: +`type Message = MyMessage` on the zone (a `#[derive(GameEvent)]` struct or +enum, so it has stable ids and the event payload codec; `NoMessages` for +none) and `Zone::on_message(ctx, message)`, which runs on the zone's task +between ticks, never during one. Deliver with `GameServer::send(zone, +ZoneInbound::Message(m))` locally, `POST /admin/zone-message` (admin +secret; `{zone, message: {type_id, payload}}` or `{zone, kick: +{principal, reason}}`) from peers, or `net::ZoneInboxClient` from a +service: it finds the owner through the `ZoneDirectory` and follows `421 +wrong_owner` answers. The built-in `ZoneInbound::Kick { principal, reason +}` removes every connection of the principal (`LeaveReason::Kicked`, and a +`PlayerLeft` lifecycle fact). A message for a zone that is not running is +refused (`404 inactive` / `InboxError::Inactive`) and never activates it. +Messages appear in the game surface under `messages`; the generated +TypeScript has `MESSAGE`, `MESSAGES` and `encodeMessage` for TS services. +Recorded and replayed like the other entry points (`Recorder::deliver`). + **Zone factory and allocation.** `ZoneFactory::new(|activation| async move { … })` may await storage: it runs on the activation task after the ownership claim and snapshot load, before the zone exists. It receives a diff --git a/distributed_game/examples/echo_arena.rs b/distributed_game/examples/echo_arena.rs index 98c2b88f5..7b417f64b 100644 --- a/distributed_game/examples/echo_arena.rs +++ b/distributed_game/examples/echo_arena.rs @@ -33,6 +33,7 @@ impl Zone for EchoArena { type Config = (); type Input = (i8, i8); type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { let pos = FieldType::QuantF32 { diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index 3c95be79c..c76f6195b 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -34,6 +34,7 @@ use crate::authoring::GameEvent; use crate::clock::Clock; use crate::facts::{EmitError, Fact, FactCommand, FactQueue, FactSettlement}; use crate::handoff::{HandoffError, HandoffRequest, Transfer, TransferIssuer, MAX_CARRIED_BYTES}; +use crate::inbox::{Delivered, ZoneInbound}; use crate::input::{ConsumePolicy, InputBuffer, PushOutcome, SequencedInput}; use crate::interest::{InterestConfig, View}; use crate::lifecycle::{ @@ -1551,6 +1552,48 @@ impl ZoneHost { self.send_event(to, event.event_id(), event.to_payload()); } + /// Deliver a service message ([`crate::inbox`]): a kick removes every + /// connection of the principal (players leave with + /// [`LeaveReason::Kicked`]), a game message runs + /// [`Zone::on_message`]. Call between ticks (the host is not ticking + /// while the caller holds `&mut self`). + pub fn deliver(&mut self, inbound: ZoneInbound) -> Delivered { + let mut delivered = Delivered::default(); + match inbound { + ZoneInbound::Kick { principal, reason } => { + let players: Vec<_> = self + .core + .players + .values() + .filter(|slot| slot.info.principal == principal) + .map(|slot| slot.info.id) + .collect(); + delivered.players_kicked = players.len(); + for player in players { + self.core.pending_kicks.push((player, reason.clone())); + } + let observers: Vec<_> = self + .core + .observers + .values() + .filter(|slot| slot.info.principal == principal) + .map(|slot| slot.info.id) + .collect(); + delivered.observers_kicked = observers.len(); + let bye = ServerMessage::Kick { + code: kick::BY_ZONE, + message: reason, + }; + for observer in observers { + self.remove_observer(observer, Some(&bye)); + } + } + ZoneInbound::Message(message) => hook!(self.on_message(message)), + } + self.process_kicks(); + delivered + } + /// Kick a player now. pub fn kick(&mut self, player: PlayerId, message: impl Into) { self.core.pending_kicks.push((player, message.into())); diff --git a/distributed_game/src/inbox.rs b/distributed_game/src/inbox.rs new file mode 100644 index 000000000..89e795f0b --- /dev/null +++ b/distributed_game/src/inbox.rs @@ -0,0 +1,69 @@ +//! The zone inbox: typed messages from services into a running zone. +//! +//! A zone declares [`Zone::Message`](crate::Zone::Message) — a +//! `#[derive(GameEvent)]` struct or enum, so messages have stable type ids +//! and the same typed binary payload codec as events (and appear in the +//! zone's [`GameSurface`](crate::GameSurface) under `messages`) — and +//! handles them in [`Zone::on_message`](crate::Zone::on_message). Zones that +//! take none use [`NoMessages`]. +//! +//! Delivery ([`ZoneHost::deliver`](crate::ZoneHost::deliver)) runs on the +//! zone's own task between ticks, never concurrently with one. The host +//! handles the built-in [`ZoneInbound::Kick`] itself (the player leaves with +//! `LeaveReason::Kicked` and, with lifecycle facts on, a `PlayerLeft` +//! fact). Over the network: `GameServer::send` locally, `POST +//! /admin/zone-message` for peers and services (admin secret), and +//! `net::ZoneInboxClient`, which finds the owning host through the +//! `ZoneDirectory` and follows `wrong_owner` answers. A message for a zone +//! that is not active is refused; it never activates the zone. + +use crate::authoring::GameEvent; +use crate::protocol::wire::{DecodeError, Writer}; +use crate::surface::{EventSurface, RecordSurface}; + +/// What a service can deliver into a zone. +#[derive(Debug, Clone, PartialEq)] +pub enum ZoneInbound { + /// Remove every connection of `principal` (players leave with + /// `LeaveReason::Kicked`, observers are unsubscribed). Clients get + /// `Kick(BY_ZONE, reason)`. + Kick { + /// Verified principal to remove (a service decision, e.g. a ban). + principal: String, + /// Shown to the client. + reason: String, + }, + /// A game message for [`Zone::on_message`](crate::Zone::on_message). + Message(M), +} + +/// Outcome of a delivery. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Delivered { + /// Players removed by a kick. + pub players_kicked: usize, + /// Observers removed by a kick. + pub observers_kicked: usize, +} + +/// The message type of a zone that takes none. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NoMessages {} + +impl GameEvent for NoMessages { + fn event_id(&self) -> u32 { + match *self {} + } + + fn encode_payload(&self, _w: &mut Writer) { + match *self {} + } + + fn decode_event(_type_id: u32, _payload: &[u8]) -> Result, DecodeError> { + Ok(None) + } + + fn event_surfaces(_records: &mut Vec) -> Vec { + Vec::new() + } +} diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index fed2fa28a..c9b2f6598 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -60,6 +60,7 @@ pub mod codec; pub mod facts; pub mod handoff; pub mod host; +pub mod inbox; pub mod input; pub mod interest; pub mod lifecycle; @@ -99,6 +100,7 @@ pub use facts::{ }; pub use handoff::{HandoffError, HandoffRequest, Transfer}; pub use host::{AdvanceReport, Ctx, FrameStats, HostConfig, JoinError, Joined, Outbound, ZoneHost}; +pub use inbox::{Delivered, NoMessages, ZoneInbound}; pub use input::{ConsumePolicy, InputBuffer, InputCredits, PushOutcome, SequencedInput}; pub use interest::{InterestConfig, View}; pub use prediction::{PredictContext, PredictStep}; diff --git a/distributed_game/src/metrics.rs b/distributed_game/src/metrics.rs index 19c66e27e..b4093ff71 100644 --- a/distributed_game/src/metrics.rs +++ b/distributed_game/src/metrics.rs @@ -642,6 +642,7 @@ mod tests { type Config = (); type Input = u8; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { EntitySchema::new().kind(KindDef::new(1, "dot").field("v", FieldType::U8)) diff --git a/distributed_game/src/net/inbox.rs b/distributed_game/src/net/inbox.rs new file mode 100644 index 000000000..85290e356 --- /dev/null +++ b/distributed_game/src/net/inbox.rs @@ -0,0 +1,300 @@ +//! Zone inbox over the network: [`GameServer::send`](super::GameServer::send) +//! locally, `POST /admin/zone-message` for peers and services, and +//! [`ZoneInboxClient`] for services (see [`crate::inbox`]). +//! +//! `POST /admin/zone-message` (header `x-game-admin-secret`), JSON body: +//! +//! ```text +//! {"zone": "pit-1", "kick": {"principal": "u-42", "reason": "banned"}} +//! {"zone": "pit-1", "message": {"type_id": 3, "payload": ""}} +//! ``` +//! +//! | Status | Body | Meaning | +//! |---|---|---| +//! | 200 | `{ok, players_kicked, observers_kicked}` | delivered on the zone task | +//! | 400 | `{error}` | malformed body, unknown message id or bad payload | +//! | 401 | `{error: "unauthorized"}` | missing / wrong admin secret (or none configured) | +//! | 404 | `{error: "inactive"}` | the zone is not running here (never activated by a message) | +//! | 421 | `{error: "wrong_owner", owner, admin_url}` | another host owns the zone (directory) | + +use std::sync::Arc; +use std::time::Duration; + +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use serde_json::{json, Value}; + +use super::peer::post_json; +use crate::authoring::GameEvent; +use crate::inbox::{Delivered, ZoneInbound}; +use crate::world::{HostAddr, ZoneDirectory}; + +/// Route path. +pub const ZONE_MESSAGE_PATH: &str = "/admin/zone-message"; + +/// Why a message was not delivered. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InboxError { + /// The zone is not active on the host asked (it is not activated). + Inactive, + /// Another host owns the zone. + WrongOwner(HostAddr), + /// Malformed message (unknown id, payload that does not decode). + Invalid(String), + /// The admin secret was refused. + Unauthorized, + /// Transport failure or an unexpected answer. + Unavailable(String), +} + +impl std::fmt::Display for InboxError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Inactive => f.write_str("zone not active"), + Self::WrongOwner(host) => write!(f, "zone owned by {}", host.id), + Self::Invalid(why) => write!(f, "invalid zone message: {why}"), + Self::Unauthorized => f.write_str("admin secret refused"), + Self::Unavailable(why) => write!(f, "zone inbox unavailable: {why}"), + } + } +} + +impl std::error::Error for InboxError {} + +/// A zone message on the wire between tasks and hosts: decoded into +/// `Z::Message` on the zone's own task. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum WireInbound { + Kick { principal: String, reason: String }, + Message { type_id: u32, payload: Vec }, +} + +impl WireInbound { + pub(crate) fn of(inbound: &ZoneInbound) -> Self { + match inbound { + ZoneInbound::Kick { principal, reason } => Self::Kick { + principal: principal.clone(), + reason: reason.clone(), + }, + ZoneInbound::Message(message) => Self::Message { + type_id: message.event_id(), + payload: message.to_payload(), + }, + } + } + + pub(crate) fn decode(self) -> Result, InboxError> { + match self { + Self::Kick { principal, reason } => { + if principal.is_empty() { + return Err(InboxError::Invalid("kick needs a principal".into())); + } + Ok(ZoneInbound::Kick { principal, reason }) + } + Self::Message { type_id, payload } => match M::decode_event(type_id, &payload) { + Ok(Some(message)) => Ok(ZoneInbound::Message(message)), + Ok(None) => Err(InboxError::Invalid(format!( + "this zone declares no message {type_id}" + ))), + Err(e) => Err(InboxError::Invalid(format!("message {type_id}: {e:?}"))), + }, + } + } + + pub(crate) fn to_json(&self, zone: &str) -> Value { + match self { + Self::Kick { principal, reason } => { + json!({ "zone": zone, "kick": { "principal": principal, "reason": reason } }) + } + Self::Message { type_id, payload } => json!({ + "zone": zone, + "message": { "type_id": type_id, "payload": URL_SAFE_NO_PAD.encode(payload) }, + }), + } + } + + /// `(zone, inbound)` from a route body. + pub(crate) fn from_json(body: &Value) -> Result<(String, Self), String> { + let zone = body + .get("zone") + .and_then(Value::as_str) + .filter(|z| !z.is_empty()) + .ok_or("zone")? + .to_string(); + let text = |v: &Value, name: &str| v.get(name).and_then(Value::as_str).map(str::to_string); + let inbound = match (body.get("kick"), body.get("message")) { + (Some(kick), None) => Self::Kick { + principal: text(kick, "principal").ok_or("kick.principal")?, + reason: text(kick, "reason").unwrap_or_default(), + }, + (None, Some(message)) => Self::Message { + type_id: message + .get("type_id") + .and_then(Value::as_u64) + .and_then(|id| u32::try_from(id).ok()) + .ok_or("message.type_id")?, + payload: URL_SAFE_NO_PAD + .decode(text(message, "payload").ok_or("message.payload")?) + .map_err(|_| "message.payload")?, + }, + _ => return Err("exactly one of `kick` or `message`".into()), + }; + Ok((zone, inbound)) + } +} + +pub(crate) fn delivered_json(delivered: Delivered) -> Value { + json!({ + "ok": true, + "players_kicked": delivered.players_kicked, + "observers_kicked": delivered.observers_kicked, + }) +} + +/// Sends zone messages from a service: finds the owning host through the +/// [`ZoneDirectory`], posts to its `/admin/zone-message` and follows +/// `wrong_owner` answers (ownership moved) a bounded number of times. +#[derive(Clone)] +pub struct ZoneInboxClient { + directory: Arc, + secret: String, + timeout: Duration, + max_redirects: u32, +} + +impl std::fmt::Debug for ZoneInboxClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ZoneInboxClient") + .field("timeout", &self.timeout) + .finish_non_exhaustive() + } +} + +impl ZoneInboxClient { + /// Client over `directory` with the hosts' shared admin secret. + pub fn new(directory: Arc, admin_secret: impl Into) -> Self { + Self { + directory, + secret: admin_secret.into(), + timeout: Duration::from_secs(3), + max_redirects: 3, + } + } + + /// Per-request timeout (default 3 s). + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + /// Deliver a typed game message to `zone`. + pub async fn send(&self, zone: &str, message: &M) -> Result<(), InboxError> { + self.post( + zone, + WireInbound::Message { + type_id: message.event_id(), + payload: message.to_payload(), + }, + ) + .await + .map(drop) + } + + /// Remove every connection of `principal` from `zone`. + pub async fn kick( + &self, + zone: &str, + principal: &str, + reason: &str, + ) -> Result { + self.post( + zone, + WireInbound::Kick { + principal: principal.into(), + reason: reason.into(), + }, + ) + .await + } + + async fn post(&self, zone: &str, inbound: WireInbound) -> Result { + let body = inbound.to_json(zone); + let mut admin_url = self + .directory + .locate(zone) + .ok_or(InboxError::Inactive)? + .admin_url; + for _ in 0..=self.max_redirects { + let (status, reply) = post_json( + &admin_url, + ZONE_MESSAGE_PATH, + &self.secret, + &body, + self.timeout, + ) + .await + .map_err(InboxError::Unavailable)?; + let error = || { + reply + .get("error") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + match status { + 200 => { + let count = + |name: &str| reply.get(name).and_then(Value::as_u64).unwrap_or(0) as usize; + return Ok(Delivered { + players_kicked: count("players_kicked"), + observers_kicked: count("observers_kicked"), + }); + } + 421 => { + admin_url = reply + .get("admin_url") + .and_then(Value::as_str) + .ok_or_else(|| { + InboxError::Unavailable("wrong_owner without admin_url".into()) + })? + .to_string(); + } + 400 => return Err(InboxError::Invalid(error())), + 401 => return Err(InboxError::Unauthorized), + 404 => return Err(InboxError::Inactive), + other => return Err(InboxError::Unavailable(format!("{other}: {}", error()))), + } + } + Err(InboxError::Unavailable( + "too many wrong_owner redirects".into(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn route_bodies_round_trip() { + for inbound in [ + WireInbound::Kick { + principal: "u-1".into(), + reason: "banned".into(), + }, + WireInbound::Message { + type_id: 7, + payload: vec![1, 2, 255], + }, + ] { + let (zone, back) = WireInbound::from_json(&inbound.to_json("pit-1")).unwrap(); + assert_eq!((zone.as_str(), back), ("pit-1", inbound)); + } + assert!(WireInbound::from_json(&json!({"zone": "z", "kick": {}})).is_err()); + assert!(WireInbound::from_json(&json!({"zone": "z"})).is_err()); + assert!(WireInbound::from_json( + &json!({"zone": "z", "kick": {"principal": "p"}, "message": {"type_id": 1, "payload": ""}}) + ) + .is_err()); + } +} diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index e0f72e89f..892c30324 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -43,6 +43,7 @@ //! - **Barrier**: [`GameServer::prepare_barrier`] / `commit_barrier` / //! `abort_barrier`, driven by a [`world::WorldCoordinator`]. +pub mod inbox; pub mod peer; pub mod ticket; pub mod transfer; @@ -66,6 +67,7 @@ use tokio::task::JoinHandle; use tokio::time::Instant; pub use crate::facts::BoxFuture; +pub use inbox::{InboxError, ZoneInboxClient, ZONE_MESSAGE_PATH}; pub use ticket::{ decode_hs256, NonceCache, SharedTicketVerifier, TicketError, TicketKey, TicketVerifier, VerifiedTicket, @@ -339,6 +341,11 @@ enum ZoneCmd { client: ClientId, graceful: bool, }, + /// A zone inbox message (decoded into `Z::Message` on the zone task). + Deliver { + inbound: inbox::WireInbound, + reply: oneshot::Sender>, + }, } enum ConnOut { @@ -578,6 +585,49 @@ where rx.await.ok() } + /// Deliver a zone inbox message ([`crate::inbox`]) to `zone_id` on this + /// server. Runs on the zone's task between ticks. Refused with + /// [`InboxError::Inactive`] when the zone is not running here (a message + /// never activates a zone) and [`InboxError::WrongOwner`] when the + /// directory places it on another host. + pub async fn send( + &self, + zone_id: &str, + inbound: crate::inbox::ZoneInbound, + ) -> Result { + self.deliver_wire(zone_id, inbox::WireInbound::of(&inbound)) + .await + } + + pub(crate) async fn deliver_wire( + &self, + zone_id: &str, + inbound: inbox::WireInbound, + ) -> Result { + if let Some(owner) = self + .inner + .directory + .as_ref() + .and_then(|d| d.locate(zone_id)) + .filter(|h| h.id != self.inner.host_id) + { + return Err(InboxError::WrongOwner(owner)); + } + let tx = { + let zones = self.inner.zones.lock().expect("zones"); + zones + .get(zone_id) + .filter(|e| !e.tx.is_closed()) + .map(|e| e.tx.clone()) + .ok_or(InboxError::Inactive)? + }; + let (reply, rx) = oneshot::channel(); + tx.send(ZoneCmd::Deliver { inbound, reply }) + .await + .map_err(|_| InboxError::Inactive)?; + rx.await.map_err(|_| InboxError::Inactive)? + } + /// Start activating `zone_id` locally without waiting (it is registered /// at once, so barriers and joins already reach it). False only while /// shutting down; an unknown zone unregisters itself once its factory @@ -727,6 +777,7 @@ where router = router .route("/admin/transfer", post(world::admin_transfer::)) .route("/admin/child-event", post(world::admin_child_event::)) + .route(ZONE_MESSAGE_PATH, post(world::admin_zone_message::)) .route("/admin/barrier/{phase}", post(world::admin_barrier::)); } router.with_state(self.inner.clone()) @@ -1046,6 +1097,9 @@ fn refuse_activation( ZoneCmd::Freeze { ack, .. } => { let _ = ack.send(()); } + ZoneCmd::Deliver { reply, .. } => { + let _ = reply.send(Err(InboxError::Inactive)); + } _ => {} } } @@ -1332,6 +1386,11 @@ async fn run_zone( if matches!(cmd, ZoneCmd::Input { .. } | ZoneCmd::Disconnect { .. }) { continue; } + // Messages never activate a zone. + if let ZoneCmd::Deliver { reply, .. } = cmd { + let _ = reply.send(Err(InboxError::Inactive)); + continue; + } if let Some(tx) = activate_local(&inner, &zone_id) { let _ = tx.send(cmd).await; } @@ -1464,6 +1523,9 @@ fn handle_cmd( host.disconnect(client, graceful); } ZoneCmd::ChildEvent(event) => host.deliver_child_event(event), + ZoneCmd::Deliver { inbound, reply } => { + let _ = reply.send(inbound.decode::().map(|m| host.deliver(m))); + } ZoneCmd::Freeze { notice, ack } => { if host.request_freeze(notice) == FreezeState::Frozen { let _ = ack.send(()); diff --git a/distributed_game/src/net/world.rs b/distributed_game/src/net/world.rs index a0795d507..ed0a7a74e 100644 --- a/distributed_game/src/net/world.rs +++ b/distributed_game/src/net/world.rs @@ -8,6 +8,7 @@ //! |---|---|---| //! | `/admin/transfer` | `{op: reserve\|confirm\|release\|return, nonce}` | transfer record at the issuing host | //! | `/admin/child-event` | `{zone, event}` | deliver a tier-up event to a local zone | +//! | `/admin/zone-message` | `{zone, kick: {principal, reason}}` or `{zone, message: {type_id, payload}}` | zone inbox ([`super::inbox`]): 404 inactive, 421 wrong owner | //! | `/admin/barrier/prepare` | `{barrier, reason, version, notes, timeout_ms}` | freeze all zones, wait for acks (409 + `missing`) | //! | `/admin/barrier/commit` | `{barrier, version, notes, mode: rolling\|hard}` | apply version, resume | //! | `/admin/barrier/abort` | `{barrier, notes}` | resume on the old version | @@ -46,6 +47,7 @@ use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; use serde_json::{json, Value}; +use super::inbox::{delivered_json, InboxError, WireInbound}; use super::peer::{post_json, secret_eq, ADMIN_SECRET_HEADER}; use super::{activate_local, transfer_op, unix_now, GameServer, Inner, TransferOp, ZoneCmd}; use crate::facts::BoxFuture; @@ -133,6 +135,41 @@ where } } +pub(super) async fn admin_zone_message( + State(inner): State>>, + headers: HeaderMap, + body: Bytes, +) -> Reply +where + Z: Zone, + Z::Config: Send, +{ + let body = match authorize(&inner, &headers, &body) { + Ok(body) => body, + Err(r) => return r, + }; + let (zone, inbound) = match WireInbound::from_json(&body) { + Ok(parsed) => parsed, + Err(e) => return reply(StatusCode::BAD_REQUEST, json!({"ok": false, "error": e})), + }; + let server = GameServer { inner }; + match server.deliver_wire(&zone, inbound).await { + Ok(delivered) => reply(StatusCode::OK, delivered_json(delivered)), + Err(InboxError::WrongOwner(owner)) => reply( + StatusCode::MISDIRECTED_REQUEST, + json!({"ok": false, "error": "wrong_owner", "owner": owner.id, "admin_url": owner.admin_url}), + ), + Err(InboxError::Inactive) => reply( + StatusCode::NOT_FOUND, + json!({"ok": false, "error": "inactive"}), + ), + Err(e) => reply( + StatusCode::BAD_REQUEST, + json!({"ok": false, "error": e.to_string()}), + ), + } +} + pub(super) async fn admin_child_event( State(inner): State>>, headers: HeaderMap, diff --git a/distributed_game/src/replay.rs b/distributed_game/src/replay.rs index 364ebd20e..4cbd1ecc0 100644 --- a/distributed_game/src/replay.rs +++ b/distributed_game/src/replay.rs @@ -223,6 +223,24 @@ pub enum Event { /// New version. version: String, }, + /// [`ZoneHost::deliver`]: a kick of `principal`. + DeliverKick { + /// Clock. + at: Duration, + /// Principal kicked. + principal: String, + /// Reason. + reason: String, + }, + /// [`ZoneHost::deliver`]: a game message (encoded `Z::Message`). + DeliverMessage { + /// Clock. + at: Duration, + /// Message type id. + type_id: u32, + /// Encoded payload. + payload: Vec, + }, /// [`ZoneHost::request_freeze`]. RequestFreeze { /// Clock. @@ -277,6 +295,8 @@ impl Event { | Self::ReconnectVersion { at, .. } | Self::SetContentVersion { at, .. } | Self::SetConfigVersion { at, .. } + | Self::DeliverKick { at, .. } + | Self::DeliverMessage { at, .. } | Self::RequestFreeze { at, .. } | Self::CancelFreeze { at, .. } | Self::ReloadStale { at } @@ -347,6 +367,12 @@ impl fmt::Display for Event { Self::SetConfigVersion { version, .. } => { write!(f, "@{ms:.1}ms config version {version}") } + Self::DeliverKick { principal, .. } => { + write!(f, "@{ms:.1}ms inbox kick {principal}") + } + Self::DeliverMessage { type_id, .. } => { + write!(f, "@{ms:.1}ms inbox message {type_id}") + } Self::RequestFreeze { notice, .. } => { write!(f, "@{ms:.1}ms request freeze {}", notice.reason) } @@ -811,6 +837,16 @@ impl Recording { } => { w.u8(16).bytes(state).string(saved_version); } + Event::DeliverKick { + principal, reason, .. + } => { + w.u8(18).string(principal).string(reason); + } + Event::DeliverMessage { + type_id, payload, .. + } => { + w.u8(19).varuint(*type_id).bytes(payload); + } Event::SetConfigVersion { version, .. } => { w.u8(17).string(version); } @@ -968,6 +1004,16 @@ impl Recording { at, version: r.string()?, }, + 18 => Event::DeliverKick { + at, + principal: r.string()?, + reason: r.string()?, + }, + 19 => Event::DeliverMessage { + at, + type_id: r.varuint()?, + payload: r.bytes()?.to_vec(), + }, _ => return Err(bad("event tag")), }; events.push(event); @@ -1282,6 +1328,29 @@ impl Recorder { }); } + /// [`ZoneHost::deliver`]. + pub fn deliver( + &mut self, + inbound: crate::inbox::ZoneInbound, + ) -> crate::inbox::Delivered { + let at = self.now(); + let event = match &inbound { + crate::inbox::ZoneInbound::Kick { principal, reason } => Event::DeliverKick { + at, + principal: principal.clone(), + reason: reason.clone(), + }, + crate::inbox::ZoneInbound::Message(message) => Event::DeliverMessage { + at, + type_id: crate::GameEvent::event_id(message), + payload: crate::GameEvent::to_payload(message), + }, + }; + let delivered = self.host.deliver(inbound); + self.recording.events.push(event); + delivered + } + /// [`ZoneHost::request_freeze`]. pub fn request_freeze(&mut self, notice: Notice) -> crate::world::FreezeState { let at = self.now(); @@ -1613,6 +1682,23 @@ pub fn replay_with( } Event::SetContentVersion { version, .. } => host.set_content_version(version.clone()), Event::SetConfigVersion { version, .. } => host.set_config_version(version.clone()), + Event::DeliverKick { + principal, reason, .. + } => { + host.deliver(crate::inbox::ZoneInbound::Kick { + principal: principal.clone(), + reason: reason.clone(), + }); + } + Event::DeliverMessage { + type_id, payload, .. + } => { + if let Ok(Some(message)) = + ::decode_event(*type_id, payload) + { + host.deliver(crate::inbox::ZoneInbound::Message(message)); + } + } Event::RequestFreeze { notice, .. } => { host.request_freeze(notice.clone()); } diff --git a/distributed_game/src/surface.rs b/distributed_game/src/surface.rs index 5e744068a..3b787da0b 100644 --- a/distributed_game/src/surface.rs +++ b/distributed_game/src/surface.rs @@ -67,6 +67,11 @@ pub struct GameSurface { /// (`encode` / `decode`), by name. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub codecs: Vec, + /// Zone inbox messages ([`Zone::Message`], [`crate::inbox`]) by id: + /// what services may send into a running zone. Same payload encoding + /// as events. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub messages: Vec, } /// A pure function compiled to WebAssembly ([`crate::pure::GamePure`]). @@ -242,16 +247,42 @@ impl GameSurface { pures: Vec::new(), constants: Vec::new(), codecs: Vec::new(), + messages: Vec::new(), } } - /// Kinds from the zone's [`Zone::schema`] and its typed input. + /// Kinds from the zone's [`Zone::schema`], its typed input and its + /// inbox messages ([`Zone::Message`]). pub fn for_zone(name: impl Into) -> Self where Z: Zone, Z::Input: GameInput, { - Self::new(name).schema(&Z::schema()).input::() + Self::new(name) + .schema(&Z::schema()) + .input::() + .messages::() + } + + /// Add the zone inbox message type (done by [`for_zone`](Self::for_zone)). + /// Message ids and names must be unique among messages. + pub fn messages(mut self) -> Self { + let mut records = Vec::new(); + for message in M::event_surfaces(&mut records) { + assert!( + self.messages + .iter() + .all(|m| m.id != message.id && m.name != message.name), + "duplicate message id {} / name {} in game surface {}", + message.id, + message.name, + self.name + ); + self.messages.push(message); + } + self.messages.sort_by_key(|m| m.id); + self.add_records(records); + self } /// Add every kind of `schema` (derived or hand-written). diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 3431125df..2b01125dd 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -209,6 +209,10 @@ pub trait Zone: Sized + Send + 'static { type Input: Send + 'static; /// Timer payload. type Timer: Clone + Send + 'static; + /// Messages services deliver into the running zone + /// ([`on_message`](Self::on_message), [`crate::inbox`]): a + /// `#[derive(GameEvent)]` struct or enum, or [`NoMessages`](crate::NoMessages). + type Message: crate::GameEvent + Send + 'static; /// Replicated entity schema (called once at creation). fn schema() -> EntitySchema; @@ -272,6 +276,11 @@ pub trait Zone: Sized + Send + 'static { /// One fixed simulation step (`ctx.dt()` is the step length of the tier). fn on_tick(&mut self, ctx: &mut Ctx<'_, Self>); + /// A message from a service ([`crate::inbox`]), delivered on the zone's + /// task between ticks (never during one). The built-in kick is handled + /// by the host and never reaches this hook. + fn on_message(&mut self, _ctx: &mut Ctx<'_, Self>, _message: Self::Message) {} + /// A timer fired (before `on_tick` of the same tick). fn on_timer(&mut self, _ctx: &mut Ctx<'_, Self>, _id: TimerId, _timer: Self::Timer) {} diff --git a/distributed_game/tests/authoring/main.rs b/distributed_game/tests/authoring/main.rs index 21b18449f..c4e4c5163 100644 --- a/distributed_game/tests/authoring/main.rs +++ b/distributed_game/tests/authoring/main.rs @@ -127,6 +127,19 @@ pub enum SampleEvent { }, } +/// What services may send into the sample zone (zone inbox). +#[derive(Debug, Clone, PartialEq, GameEvent)] +pub enum SampleMessage { + #[game_event(id = 1)] + Announce { + #[long] + text: String, + urgent: bool, + }, + #[game_event(id = 2)] + SetWind(SampleWind), +} + #[derive(Debug, Clone, PartialEq, GameEvent)] #[game_event(id = 9)] pub struct Ping { @@ -243,6 +256,7 @@ impl Zone for Sample { type Config = (); type Input = SampleInput; type Timer = (); + type Message = SampleMessage; fn schema() -> EntitySchema { schema![PlayerView, MarkerView, StatsView] @@ -753,6 +767,21 @@ fn golden_fixture_is_current() { "sanitized": {"mx": sanitized.mx, "my": sanitized.my, "buttons": sanitized.buttons, "aim": aim_wire, "power": 0.25, "viewTick": 123_456}, }, "events": events, + // Zone inbox messages (`/admin/zone-message` payloads). + "messages": [ + { + "id": 1, + "name": "announce", + "value": { "text": "server restart in 5", "urgent": true }, + "hex": hex(&SampleMessage::Announce { text: "server restart in 5".into(), urgent: true }.to_payload()), + }, + { + "id": 2, + "name": "set_wind", + "value": { "speed": 2.5, "gusts": 7 }, + "hex": hex(&SampleMessage::SetWind(SampleWind { speed: 2.5, gusts: 7 }).to_payload()), + }, + ], // Each constant's encoding: the generated literal must decode-equal it. "constants": { "sample_board": hex(&distributed_game::codec::to_bytes(&sample_board())), diff --git a/distributed_game/tests/fixtures/authoring.json b/distributed_game/tests/fixtures/authoring.json index 5983f862c..492d1eb2d 100644 --- a/distributed_game/tests/fixtures/authoring.json +++ b/distributed_game/tests/fixtures/authoring.json @@ -107,6 +107,26 @@ "viewTick": 123456 } }, + "messages": [ + { + "hex": "13736572766572207265737461727420696e203501", + "id": 1, + "name": "announce", + "value": { + "text": "server restart in 5", + "urgent": true + } + }, + { + "hex": "0000204007", + "id": 2, + "name": "set_wind", + "value": { + "gusts": 7, + "speed": 2.5 + } + } + ], "pures": [ { "arg": "0000c03fffff03", @@ -547,6 +567,40 @@ "priority": 1.0 } ], + "messages": [ + { + "id": 1, + "name": "announce", + "payload": { + "fields": [ + { + "name": "text", + "type": { + "type": "str" + } + }, + { + "name": "urgent", + "type": { + "type": "bool" + } + } + ], + "kind": "fields" + } + }, + { + "id": 2, + "name": "set_wind", + "payload": { + "kind": "value", + "type": { + "name": "SampleWind", + "type": "record" + } + } + } + ], "name": "sample", "predict": { "abi": 1, diff --git a/distributed_game/tests/inbox/main.rs b/distributed_game/tests/inbox/main.rs new file mode 100644 index 000000000..9d3c08437 --- /dev/null +++ b/distributed_game/tests/inbox/main.rs @@ -0,0 +1,398 @@ +//! The zone inbox: typed messages and the built-in kick, delivered on the +//! zone task between ticks; `GameServer::send`, `/admin/zone-message` and +//! `ZoneInboxClient` (auth, inactive zones, wrong owner). + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use distributed_game::lifecycle::{LeftReason, Lifecycle}; +use distributed_game::net::peer::post_json; +use distributed_game::net::{ + GameServer, InboxError, ServerConfig, TicketKey, ZoneFactory, ZoneInboxClient, + ZONE_MESSAGE_PATH, +}; +use distributed_game::protocol::kick; +use distributed_game::world::{HostAddr, StaticDirectory}; +use distributed_game::*; +use serde_json::json; + +const SECRET: &str = "inbox-secret"; +const ADMIN: &str = "inbox-admin"; + +/// Messages a service may send into a hall. +#[derive(Debug, Clone, PartialEq, GameEvent)] +pub enum HallMessage { + /// Change the message of the day. + #[game_event(id = 1)] + SetMotd { + /// New text. + #[long] + motd: String, + }, + /// Numbered probe (ordering / tick checks). + #[game_event(id = 2)] + Probe { + /// Sequence. + n: u32, + }, +} + +#[derive(Default)] +struct Shared { + /// `(n, tick seen, delivered inside a tick?)` per probe. + probes: Vec<(u32, u64, bool)>, +} + +struct Hall { + motd: String, + in_tick: bool, + ticks: u64, + left: Vec, + shared: Arc>, +} + +impl Zone for Hall { + type Config = Arc>; + type Input = u8; + type Timer = (); + type Message = HallMessage; + + fn schema() -> EntitySchema { + EntitySchema::new() + } + + fn on_create(_ctx: &mut Ctx<'_, Self>, shared: Arc>) -> Self { + Self { + motd: String::new(), + in_tick: false, + ticks: 0, + left: Vec::new(), + shared, + } + } + + fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option { + payload.first().copied() + } + + fn on_tick(&mut self, _ctx: &mut Ctx<'_, Self>) { + self.in_tick = true; + self.ticks += 1; + // A little work so a concurrent delivery would have a window. + std::thread::sleep(Duration::from_micros(200)); + self.in_tick = false; + } + + fn on_message(&mut self, ctx: &mut Ctx<'_, Self>, message: HallMessage) { + match message { + HallMessage::SetMotd { motd } => self.motd = motd, + HallMessage::Probe { n } => { + self.shared + .lock() + .unwrap() + .probes + .push((n, ctx.tick(), self.in_tick)); + } + } + } + + fn on_leave(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId, reason: LeaveReason) { + self.left.push(reason); + } + + /// Keep ticking at the Hot rate with nobody connected. + fn has_pending_work(&self) -> bool { + true + } + + fn replicate(&self, _out: &mut WorldView) {} +} + +fn claims(principal: &str, nonce: &str) -> TicketClaims { + TicketClaims::new(principal, "hall", u64::MAX, nonce, Role::Player) +} + +#[test] +fn kick_removes_the_principal_with_a_lifecycle_fact_and_messages_reach_on_message() { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "hall", + HostConfig { + lifecycle_facts: true, + ..HostConfig::default() + }, + Arc::new(clock.clone()), + Arc::default(), + ) + .with_facts(queue); + let ana = h.join(&claims("ana", "a"), 0, "").unwrap(); + h.join(&claims("bo", "b"), 0, "").unwrap(); + h.drain_outbound(); + + let delivered = h.deliver(ZoneInbound::Kick { + principal: "ana".into(), + reason: "banned".into(), + }); + assert_eq!(delivered.players_kicked, 1); + assert_eq!(h.zone().left, [LeaveReason::Kicked]); + assert_eq!(h.player_count(), 1); + let out = h.drain_outbound(); + let kicked = out.iter().any(|o| match o { + Outbound::Frame { client, frame } if *client == ana.client => matches!( + ServerMessage::decode(frame, &()), + Ok(ServerMessage::Kick { code, ref message }) if code == kick::BY_ZONE && message == "banned" + ), + _ => false, + }); + assert!(kicked, "the client is told why"); + // Kicking someone absent is a no-op. + assert_eq!( + h.deliver(ZoneInbound::Kick { + principal: "nobody".into(), + reason: String::new(), + }), + Delivered::default() + ); + h.deliver(ZoneInbound::Message(HallMessage::SetMotd { + motd: "welcome".into(), + })); + assert_eq!(h.zone().motd, "welcome"); + + drop(h); + let sink = MemoryFactSink::new(); + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + let mut run = std::pin::pin!(feed.run(sink.clone(), |_| async {})); + while std::future::Future::poll(run.as_mut(), &mut cx).is_pending() {} + let ana: Vec<_> = sink + .committed() + .iter() + .filter(|f| f.principal() == Some("ana")) + .filter_map(Lifecycle::of) + .collect(); + assert!( + matches!(ana.last(), Some(Lifecycle::Left(l)) if l.reason == LeftReason::Kicked), + "{ana:?}" + ); +} + +#[test] +fn messages_are_in_the_surface_with_their_codec() { + let surface = GameSurface::new("hall").messages::<::Message>(); + let names: Vec<_> = surface.messages.iter().map(|m| m.name.as_str()).collect(); + assert_eq!(names, ["set_motd", "probe"]); + assert!(surface.to_json().contains("\"messages\"")); + // No messages: nothing in the JSON (older surfaces stay identical). + let quiet = GameSurface::new("quiet").messages::(); + assert!(!quiet.to_json().contains("messages")); + let message = HallMessage::Probe { n: 7 }; + let back = HallMessage::decode_event(message.event_id(), &message.to_payload()).unwrap(); + assert_eq!(back, Some(message)); +} + +async fn serve( + server: &GameServer, + listener: tokio::net::TcpListener, +) -> tokio::task::JoinHandle<()> { + let serving = server.clone(); + tokio::spawn(async move { + serving + .serve(listener, std::future::pending()) + .await + .unwrap(); + }) +} + +fn server(shared: Arc>) -> GameServer { + let mut config = ServerConfig::new(TicketKey::new(SECRET)); + config.admin_secret = Some(ADMIN.into()); + config.host.tick_rate = 120; + GameServer::new( + config, + ZoneFactory::sync(move |a| a.zone_id.starts_with("hall").then(|| shared.clone())), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn on_message_runs_on_the_zone_task_between_ticks() { + let shared = Arc::new(Mutex::new(Shared::default())); + let server = server(shared.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let handle = serve(&server, listener).await; + assert!(server.activate("hall").await); + let senders: Vec<_> = (0..64u32) + .map(|n| { + let server = server.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(u64::from(n % 16) * 3)).await; + server + .send("hall", ZoneInbound::Message(HallMessage::Probe { n })) + .await + }) + }) + .collect(); + for sender in senders { + sender.await.unwrap().unwrap(); + } + let probes = shared.lock().unwrap().probes.clone(); + assert_eq!(probes.len(), 64); + assert!( + probes.iter().all(|(_, _, in_tick)| !in_tick), + "never inside a tick" + ); + let ticks: std::collections::BTreeSet<_> = probes.iter().map(|(_, t, _)| *t).collect(); + assert!(ticks.len() > 1, "delivered across several ticks: {ticks:?}"); + + // A message for a zone that is not running is refused, never activates it. + assert_eq!( + server + .send( + "hall-cold", + ZoneInbound::Message(HallMessage::Probe { n: 0 }) + ) + .await, + Err(InboxError::Inactive) + ); + assert!(!server.is_active("hall-cold")); + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .unwrap() + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn admin_route_authenticates_refuses_inactive_and_redirects_to_the_owner() { + // Two hosts; the shared directory places `hall.a` on A. + let (la, lb) = ( + tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(), + tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(), + ); + let a_addr = HostAddr::new("A", la.local_addr().unwrap().to_string()); + let b_addr = HostAddr::new("B", lb.local_addr().unwrap().to_string()); + let directory = Arc::new(StaticDirectory::new()); + directory.add_host(a_addr.clone()).add_host(b_addr.clone()); + directory.assign("hall.a", "A").assign("hall.cold", "A"); + let shared = Arc::new(Mutex::new(Shared::default())); + let a = server(shared.clone()) + .with_host_id("A") + .with_directory(directory.clone()); + let b = server(shared.clone()) + .with_host_id("B") + .with_directory(directory.clone()); + let (ha, hb) = (serve(&a, la).await, serve(&b, lb).await); + assert!(a.activate("hall.a").await); + let probe = |n: u32| { + let m = HallMessage::Probe { n }; + json!({ + "zone": "hall.a", + "message": { + "type_id": m.event_id(), + "payload": base64_url(&m.to_payload()), + } + }) + }; + let post = |url: String, secret: &'static str, body: serde_json::Value| async move { + post_json( + &url, + ZONE_MESSAGE_PATH, + secret, + &body, + Duration::from_secs(3), + ) + .await + .unwrap() + }; + + // Unauthenticated (wrong or empty secret) is refused before anything. + for secret in ["", "wrong"] { + let (status, body) = post(a_addr.admin_url.clone(), secret, probe(1)).await; + assert_eq!( + (status, body["error"].as_str()), + (401, Some("unauthorized")) + ); + } + // The owner delivers. + let (status, _) = post(a_addr.admin_url.clone(), ADMIN, probe(2)).await; + assert_eq!(status, 200); + // The wrong host answers with the owner instead of delivering. + let (status, body) = post(b_addr.admin_url.clone(), ADMIN, probe(3)).await; + assert_eq!(status, 421, "{body}"); + assert_eq!(body["error"], "wrong_owner"); + assert_eq!(body["owner"], "A"); + assert_eq!(body["admin_url"], a_addr.admin_url); + // Inactive zones: 404, and still inactive. + let mut cold = probe(4); + cold["zone"] = json!("hall.cold"); + let (status, body) = post(a_addr.admin_url.clone(), ADMIN, cold).await; + assert_eq!((status, body["error"].as_str()), (404, Some("inactive"))); + assert!(!a.is_active("hall.cold")); + // Unknown message ids and malformed payloads: 400. + let bad = json!({"zone": "hall.a", "message": {"type_id": 99, "payload": ""}}); + let (status, _) = post(a_addr.admin_url.clone(), ADMIN, bad).await; + assert_eq!(status, 400); + let short = json!({"zone": "hall.a", "message": {"type_id": 2, "payload": ""}}); + let (status, _) = post(a_addr.admin_url.clone(), ADMIN, short).await; + assert_eq!(status, 400); + + // A service whose directory is stale (says B) follows the redirect. + let stale = Arc::new(StaticDirectory::new()); + stale.add_host(b_addr.clone()).assign("hall.a", "B"); + let client = ZoneInboxClient::new(stale, ADMIN); + client + .send("hall.a", &HallMessage::SetMotd { motd: "hi".into() }) + .await + .unwrap(); + assert_eq!( + a.inspect("hall.a", |h| h.zone().motd.clone()) + .await + .unwrap(), + "hi" + ); + client + .send("hall.a", &HallMessage::Probe { n: 5 }) + .await + .unwrap(); + assert_eq!( + client.kick("hall.a", "nobody", "bye").await.unwrap(), + Delivered::default() + ); + assert_eq!( + client.send("hall.cold", &HallMessage::Probe { n: 6 }).await, + Err(InboxError::Inactive) + ); + let unauthorized = ZoneInboxClient::new(directory.clone(), "wrong"); + assert_eq!( + unauthorized + .send("hall.a", &HallMessage::Probe { n: 7 }) + .await, + Err(InboxError::Unauthorized) + ); + let ns: Vec<_> = shared.lock().unwrap().probes.iter().map(|p| p.0).collect(); + assert_eq!(ns, [2, 5], "only the authorized, owned deliveries ran"); + + a.shutdown(); + b.shutdown(); + for handle in [ha, hb] { + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .unwrap() + .unwrap(); + } +} + +fn base64_url(bytes: &[u8]) -> String { + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + let mut out = String::new(); + for chunk in bytes.chunks(3) { + let n = chunk + .iter() + .enumerate() + .fold(0u32, |acc, (i, b)| acc | (u32::from(*b) << (16 - 8 * i))); + for i in 0..=chunk.len() { + out.push(ALPHABET[((n >> (18 - 6 * i)) & 63) as usize] as char); + } + } + out +} diff --git a/distributed_game/tests/netcode/main.rs b/distributed_game/tests/netcode/main.rs index 7a4a985c8..f5683d491 100644 --- a/distributed_game/tests/netcode/main.rs +++ b/distributed_game/tests/netcode/main.rs @@ -53,6 +53,7 @@ impl Zone for Range { type Config = (); type Input = (f32, u16); type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { EntitySchema::new().kind(KindDef::new(1, "target").field("x", FieldType::F32)) diff --git a/distributed_game/tests/world/realm.rs b/distributed_game/tests/world/realm.rs index 1609c0ef2..36188a219 100644 --- a/distributed_game/tests/world/realm.rs +++ b/distributed_game/tests/world/realm.rs @@ -55,6 +55,7 @@ impl Zone for Realm { type Config = RealmConfig; type Input = u8; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { EntitySchema::new().kind(KindDef::new(1, "realm").field("counter", FieldType::U32)) diff --git a/distributed_game/tests/zone_runtime/arena.rs b/distributed_game/tests/zone_runtime/arena.rs index 7a72b2620..dbce5d8fc 100644 --- a/distributed_game/tests/zone_runtime/arena.rs +++ b/distributed_game/tests/zone_runtime/arena.rs @@ -83,6 +83,7 @@ impl Zone for Arena { type Config = ArenaConfig; type Input = (i8, i8); type Timer = ArenaTimer; + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { let pos = FieldType::QuantF32 { diff --git a/distributed_game/tests/zone_runtime/main.rs b/distributed_game/tests/zone_runtime/main.rs index 7d4d6f5df..2a9d45ae6 100644 --- a/distributed_game/tests/zone_runtime/main.rs +++ b/distributed_game/tests/zone_runtime/main.rs @@ -721,6 +721,7 @@ impl distributed_game::Zone for Paced { type Config = (); type Input = u8; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> distributed_game::EntitySchema { distributed_game::EntitySchema::new() diff --git a/examples/game/server/src/arena/mod.rs b/examples/game/server/src/arena/mod.rs index 2c6e5719d..409ce1710 100644 --- a/examples/game/server/src/arena/mod.rs +++ b/examples/game/server/src/arena/mod.rs @@ -208,6 +208,7 @@ impl Zone for Arena { type Config = ArenaConfig; type Input = MoveInput; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![PlayerView, DroneView, PillarView] diff --git a/examples/game/server/src/freeze/mod.rs b/examples/game/server/src/freeze/mod.rs index 10a477e1a..a1b4b0559 100644 --- a/examples/game/server/src/freeze/mod.rs +++ b/examples/game/server/src/freeze/mod.rs @@ -357,6 +357,7 @@ impl Zone for FreezeZone { type Config = FreezeConfig; type Input = MoveInput; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![PlayerView, SlimeView, GemView, RitualView] diff --git a/examples/game/server/src/gallery/mod.rs b/examples/game/server/src/gallery/mod.rs index a1f6898dc..664d364c7 100644 --- a/examples/game/server/src/gallery/mod.rs +++ b/examples/game/server/src/gallery/mod.rs @@ -484,6 +484,7 @@ impl Zone for Gallery { type Config = GalleryConfig; type Input = ShotInput; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![ShooterView, TargetView] diff --git a/examples/game/server/src/grid/mod.rs b/examples/game/server/src/grid/mod.rs index 3b2002f2f..0905a0be8 100644 --- a/examples/game/server/src/grid/mod.rs +++ b/examples/game/server/src/grid/mod.rs @@ -404,6 +404,7 @@ impl Zone for GridZone { type Config = GridConfig; type Input = MoveInput; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![WalkerView, CritterView, MarkerView] diff --git a/examples/game/server/src/mobpit/zone.rs b/examples/game/server/src/mobpit/zone.rs index d615146bc..42f4feb52 100644 --- a/examples/game/server/src/mobpit/zone.rs +++ b/examples/game/server/src/mobpit/zone.rs @@ -822,6 +822,7 @@ impl Zone for MobPit { type Config = MobPitConfig; type Input = PitInput; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![PlayerView, MobView, PillarView, PitView] diff --git a/examples/game/server/src/ripple/mod.rs b/examples/game/server/src/ripple/mod.rs index 8e86be046..17f1c9cef 100644 --- a/examples/game/server/src/ripple/mod.rs +++ b/examples/game/server/src/ripple/mod.rs @@ -641,6 +641,7 @@ impl Zone for Ripple { type Config = RippleConfig; type Input = HoodInput; type Timer = RippleTimer; + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![TierView, BossView, FighterView] diff --git a/js/tests/game-generated.test.mjs b/js/tests/game-generated.test.mjs index f4304238a..d87c756a5 100644 --- a/js/tests/game-generated.test.mjs +++ b/js/tests/game-generated.test.mjs @@ -90,6 +90,18 @@ test('every event encodes and decodes like Rust', () => { assert.throws(() => gen.EVENTS.hello.decode(fromHex('01'))); }); +test('zone inbox messages encode and decode like Rust', () => { + assert.ok(fixture.messages.length > 0); + for (const sample of fixture.messages) { + const name = camel(sample.name); + assert.equal(gen.MESSAGE[name], sample.id); + const { typeId, payload } = gen.encodeMessage(name, sample.value); + assert.equal(typeId, sample.id); + assert.equal(toHex(payload), sample.hex, `${sample.name} encodes`); + assert.deepEqual(normalize(gen.MESSAGES[name].decode(fromHex(sample.hex))), normalize(sample.value)); + } +}); + test('kinds, typed field accessors and correction defaults follow the surface', () => { const surface = fixture.surface; for (const kind of surface.kinds) assert.equal(gen.KIND[camel(kind.name)], kind.id); From 4de4347f7f82434a44ecf2f2e6f8988b3b787495 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 10:05:08 -0500 Subject: [PATCH 07/40] feat(cell-host): re-armable cell alarms sharing the DO alarm Command handlers arm, re-arm and cancel cell alarms with their events (ctx.arm_alarm / ctx.cancel_alarm). Alarm rows live in the cell's own SQL (migration 11, cell_alarms) and commit in the command's transaction. When due, the cell dispatches the alarm's command into itself through its command ledger with a (name, generation) command id and a trusted cell-alarm session, so each arm has its effects at most once, even across a crash before settlement. On celld the one Durable Object alarm is shared with the outbox watchdog: every setAlarm targets the earliest of the next domain alarm and the next drain. AggregateCell::alarm(&env) fires due alarms, then drains. The in-process reference host and native cell-sqlite cells take a ManualClock (CellStreamStore::with_clock) and fire_due_alarms(). Proven on celld dev 0.4.0 and 0.6.2 by tests/celld/alarms.mjs (atomic arm, periodic re-arm, watchdog vs domain alarm, restart); CI runs it in the celld storage job. [[tasks/gr-p28-cell-alarms]] --- .github/workflows/README.md | 2 +- .github/workflows/integration-celld.yaml | 11 +- .gitignore | 3 + Cargo.toml | 2 +- distributed_cli/src/contracts/tests.rs | 2 +- docs/coordination-primitives.md | 58 ++ migrations/inventory.json | 12 + migrations/postgres/0011_cell_alarms.sql | 19 + migrations/sqlite/0011_cell_alarms.sql | 20 + src/commit_builder/mod.rs | 1 + src/in_memory_repo/repository.rs | 1 + src/microsvc/causal.rs | 25 + src/microsvc/cell_host/alarm.rs | 773 ++++++++++++++++++ src/microsvc/cell_host/cell.rs | 189 ++++- src/microsvc/cell_host/celld_outbox.rs | 179 +++- src/microsvc/cell_host/mod.rs | 13 +- src/microsvc/cell_host/sql_store.rs | 186 +++++ src/microsvc/cell_host/store.rs | 169 ++++ src/microsvc/cell_host/tests.rs | 1 + src/microsvc/cell_host/tests/alarms.rs | 519 ++++++++++++ src/microsvc/service/handlers.rs | 26 + src/outbox/commit.rs | 1 + src/repository/migrations.rs | 2 +- src/repository/traits.rs | 4 + src/sqlx_repo/repo/backend.rs | 18 +- src/sqlx_repo/repo/commit.rs | 1 + tests/celld/README.md | 13 +- tests/celld/alarm-worker/Cargo.toml | 23 + tests/celld/alarm-worker/src/lib.rs | 458 +++++++++++ tests/celld/alarm-worker/wrangler.jsonc | 22 + tests/celld/alarms.mjs | 251 ++++++ tests/celld/main.rs | 32 + tests/celld/worker/src/lib.rs | 33 +- .../read_models.rs | 1 + .../scenario.rs | 1 + tests/postgres_repository/main.rs | 3 +- tests/sqlite_repository/main.rs | 5 +- 37 files changed, 3033 insertions(+), 46 deletions(-) create mode 100644 migrations/postgres/0011_cell_alarms.sql create mode 100644 migrations/sqlite/0011_cell_alarms.sql create mode 100644 src/microsvc/cell_host/alarm.rs create mode 100644 src/microsvc/cell_host/tests/alarms.rs create mode 100644 tests/celld/alarm-worker/Cargo.toml create mode 100644 tests/celld/alarm-worker/src/lib.rs create mode 100644 tests/celld/alarm-worker/wrangler.jsonc create mode 100644 tests/celld/alarms.mjs diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3c7c411de..8e49cd5e1 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,7 +13,7 @@ existing unbounded-tech quality provider plus the integration/* jobs below — | [`test-all-features.yaml`](./test-all-features.yaml) | yes | **This repo:** workspace `--all-features` | | [`integration-*.yaml`](./) | yes | **This repo:** broker / DB / CLI / observability / GraphQL identity+OIDC | | [`integration-e2e-ui.yaml`](./integration-e2e-ui.yaml) | yes | **This repo:** `tests/e2e-ui` offline suite, compose Playwright e2e, and kind local-GitOps Playwright e2e | -| [`integration-celld.yaml`](./integration-celld.yaml) | yes | **This repo:** `tests/e2e-celld` workspace tests + live Azurite+celld+NATS | +| [`integration-celld.yaml`](./integration-celld.yaml) | yes | **This repo:** `tests/e2e-celld` workspace tests, celld storage and cell-alarm proofs (`celld dev`), live celld+NATS | | [`integration-js.yaml`](./integration-js.yaml) | yes | **This repo:** install, typecheck, test, build, and packed-consumer smoke test for `js/` | | [`on-pr-quality.yaml`](./on-pr-quality.yaml) | entry | **This repo** PR gate (not the consumer quality contract) | | [`on-push-main-version-and-tag.yaml`](./on-push-main-version-and-tag.yaml) | entry | **This repo** main → **vnext** tag | diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index 2d4f00953..74ae5b51e 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -39,7 +39,9 @@ jobs: targets: wasm32-unknown-unknown - uses: Swatinem/rust-cache@v2 with: - workspaces: tests/celld/worker -> tests/celld/worker/target + workspaces: | + tests/celld/worker -> tests/celld/worker/target + tests/celld/alarm-worker -> tests/celld/alarm-worker/target shared-key: celld-storage - uses: actions/setup-node@v4 with: @@ -56,6 +58,11 @@ jobs: run: worker-build --release --features storage-conformance - name: Prove SQL atomicity, fencing, autonomous recovery and growth run: node tests/celld/storage-conformance.mjs + - name: Build the cell-alarm Ticker Worker + working-directory: tests/celld/alarm-worker + run: worker-build --release + - name: Prove re-armable cell alarms (atomic arm, cadence, watchdog, restart) + run: node tests/celld/alarms.mjs - name: Retain storage proof results and logs if: always() uses: actions/upload-artifact@v4 @@ -64,6 +71,8 @@ jobs: path: | /tmp/distributed-cell-storage-*/results.json /tmp/distributed-cell-storage-*/celld-*.log + /tmp/distributed-cell-alarms-*/results.json + /tmp/distributed-cell-alarms-*/celld-*.log if-no-files-found: ignore e2e-celld: diff --git a/.gitignore b/.gitignore index 28784b1d2..fb273dae0 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,9 @@ tests/celld/worker/build/ tests/celld/worker/.celld/ tests/celld/relay-worker/target/ tests/celld/relay-worker/build/ +tests/celld/alarm-worker/target/ +tests/celld/alarm-worker/build/ +tests/celld/alarm-worker/.celld/ # Build output of CLI test fixture crates /distributed_cli/tests/fixtures/*/target/ diff --git a/Cargo.toml b/Cargo.toml index d6793212e..2c1e670d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = ["distributed_macros", "distributed_cli", "distributed_game", "distributed_game_macros"] -exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "examples/game/server"] +exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "examples/game/server"] resolver = "2" [workspace.package] diff --git a/distributed_cli/src/contracts/tests.rs b/distributed_cli/src/contracts/tests.rs index a23582ee8..26deeacac 100644 --- a/distributed_cli/src/contracts/tests.rs +++ b/distributed_cli/src/contracts/tests.rs @@ -1259,7 +1259,7 @@ fn migration_inventory_is_deterministic_and_preserves_runtime_order() { .iter() .map(|migration| migration.version) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); assert_eq!( inventory.canonical_bytes().expect("canonical inventory"), inventory diff --git a/docs/coordination-primitives.md b/docs/coordination-primitives.md index 608a31b5c..848377992 100644 --- a/docs/coordination-primitives.md +++ b/docs/coordination-primitives.md @@ -10,6 +10,7 @@ twice. |------|-----------|--------| | Change state in two cells with rollback | Saga / process manager | `distributed::saga` | | Run something at a time in the future, across restarts | `OutboxMessage::deliver_at` | `distributed::outbox` | +| A cell schedules its own future or recurring work | Re-armable cell alarm | `distributed::cell_host` | | One owner per key, with stale writers rejected | Lease with fencing token | `distributed::lease` | | Deterministic time in tests | `Clock`, `ManualClock` | `distributed::clock` | @@ -45,6 +46,56 @@ retry) resets availability to now, so a timer that failed to publish is retried promptly instead of being re-delayed. `OutboxMessage.deliver_at` is `#[serde(default)]`. +## Cell alarms: a cell schedules its own work + +`deliver_at` leaves the cell and needs a consumer to send a command back. +When a cell must act on itself later (a matchmaking tick, an invite expiry, a +daily reset, a lease expiry), a causal command handler arms a **cell alarm** +with its events: + +```rust +// in a causal command handler +ctx.arm_alarm(CellAlarm::after("tick", Duration::from_secs(5), "pool.tick", &TickInput { pool })?)?; +ctx.cancel_alarm("invite-expiry")?; + +// in the alarm's own handler: re-arm from the fired due time (no drift) +let due = ctx.alarm_due().expect("fired by an alarm") + period; +ctx.arm_alarm(CellAlarm::at("tick", due, "pool.tick", &input)?)?; +``` + +- The alarm row (`cell_alarms`, migration 11) commits in the **same + transaction** as the command's events, ledger receipt and outbox rows. A + failed command arms nothing. `CellAlarm::after` resolves against the + cell's clock at commit; handlers have no wall clock of their own. +- One row per alarm name. Arming a name again replaces it and bumps its + `generation`; `cancel_alarm` disarms it. +- When due, the cell dispatches the alarm's command **into itself** through + its command ledger, with a command id derived from `(name, generation)` and + the trusted session from `cell_alarm_session` (role `cell-alarm`, plus + claims only the host sets). Mount alarm-only commands with + `guard: cell_alarm_guard` so callers cannot invoke them. A crash after the + handler committed but before the alarm settled replays the receipt; the + handler never runs twice for one arm. +- Rejections disarm the alarm (reported as `failed`). Retryable failures back + off from 1 s, doubling, up to 5 min (`retried`). At most 32 alarms fire per + wake, 16 alarm writes per command and 1,024 armed alarms per cell. +- Only aggregate cells have an alarm slot. Every other repository rejects a + batch carrying alarm writes. + +**celld.** A Durable Object has one alarm. The outbox watchdog +(`CelldOutbox`) and domain alarms share it: every `setAlarm` targets the +earliest of the next domain alarm and the next drain, so a watchdog rearm +never postpones a domain alarm and a far domain alarm never postpones a +drain. The Worker's `alarm()` handler calls `AggregateCell::alarm(&env)`, +which fires due alarms, then drains the outbox and re-plans the wake. +Commands that arm an alarm move the wake earlier right after they commit. + +**Reference host.** The in-process host and native `cell-sqlite` cells have +no timer. Give the store a `ManualClock` +(`CellStreamStore::with_clock`), advance it and call +`AggregateCell::fire_due_alarms()`: the same ledger and settlement path runs +deterministically. + ## Leases with fencing tokens ```rust @@ -147,6 +198,13 @@ A test network delivers outbox rows whose `deliver_at` is due on a ```sh cargo test --features sqlite,postgres --test saga_transfer --test coordination_lease --test scheduled_outbox cargo test --features sqlite --lib -- outbox lease clock +cargo test --features cell-sqlite,workers-rs --lib cell_host +(cd tests/celld/alarm-worker && worker-build --release) && node tests/celld/alarms.mjs ``` +`tests/celld/alarms.mjs` owns its own `celld dev` process (no Docker; set +`CELLD` to choose the binary) and proves the alarm row commits atomically, +periodic re-arm at the alarm cadence, that the watchdog and domain alarms do +not starve each other, and exactly-once firing across a celld restart. + The Postgres cases skip when `DATABASE_URL` is unset (see `compose.yaml`). diff --git a/migrations/inventory.json b/migrations/inventory.json index 1b923d11e..249bdf2e3 100644 --- a/migrations/inventory.json +++ b/migrations/inventory.json @@ -120,6 +120,18 @@ "path": "migrations/postgres/0010_coordination_leases.sql", "sha256": "f5494517f00ba6f4a00ca140fcb77ac49444d6d7bdf3ac575d00e043f8c01072" } + }, + { + "version": 11, + "description": "cell alarms", + "sqlite": { + "path": "migrations/sqlite/0011_cell_alarms.sql", + "sha256": "5d581afcf6238c3a251acb85f28d045d71cce1ca09b98649c180d7caeb228bc7" + }, + "postgres": { + "path": "migrations/postgres/0011_cell_alarms.sql", + "sha256": "572da24fac96e56263b5dfdf17be26d99f7a330de5585325ba091a4fe3e6caf8" + } } ] } diff --git a/migrations/postgres/0011_cell_alarms.sql b/migrations/postgres/0011_cell_alarms.sql new file mode 100644 index 000000000..d5715a351 --- /dev/null +++ b/migrations/postgres/0011_cell_alarms.sql @@ -0,0 +1,19 @@ +-- Re-armable aggregate-cell alarms (see `distributed::cell_host::CellAlarm`). +-- Only aggregate cells read or write this table; it exists here so the +-- inventory stays identical across dialects. One row per alarm name; +-- `generation` increments on every arm and is never reset. `due_at_ms` is +-- epoch milliseconds; `armed` is 1 while the alarm is scheduled. +CREATE TABLE IF NOT EXISTS cell_alarms ( + name TEXT NOT NULL PRIMARY KEY, + command TEXT NOT NULL, + input TEXT NOT NULL, + due_at_ms BIGINT NOT NULL, + generation BIGINT NOT NULL, + attempts BIGINT NOT NULL DEFAULT 0, + armed SMALLINT NOT NULL, + CHECK (name <> ''), + CHECK (command <> ''), + CHECK (generation > 0), + CHECK (armed IN (0, 1)) +); +CREATE INDEX IF NOT EXISTS cell_alarms_armed_due ON cell_alarms (armed, due_at_ms); diff --git a/migrations/sqlite/0011_cell_alarms.sql b/migrations/sqlite/0011_cell_alarms.sql new file mode 100644 index 000000000..d3e741236 --- /dev/null +++ b/migrations/sqlite/0011_cell_alarms.sql @@ -0,0 +1,20 @@ +-- Re-armable aggregate-cell alarms (see `distributed::cell_host::CellAlarm`). +-- Only aggregate cells (Durable Object `storage.sql`, native `cell-sqlite`) +-- read or write this table. One row per alarm name; `generation` increments +-- on every arm and is never reset (disarming keeps the row), so the command id +-- derived from (name, generation) is unique per arm. `due_at_ms` is epoch +-- milliseconds; `armed` is 1 while the alarm is scheduled. +CREATE TABLE IF NOT EXISTS cell_alarms ( + name TEXT NOT NULL PRIMARY KEY, + command TEXT NOT NULL, + input TEXT NOT NULL, + due_at_ms INTEGER NOT NULL, + generation INTEGER NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + armed INTEGER NOT NULL, + CHECK (name <> ''), + CHECK (command <> ''), + CHECK (generation > 0), + CHECK (armed IN (0, 1)) +); +CREATE INDEX IF NOT EXISTS cell_alarms_armed_due ON cell_alarms (armed, due_at_ms); diff --git a/src/commit_builder/mod.rs b/src/commit_builder/mod.rs index d0e43a2f7..f7741f792 100644 --- a/src/commit_builder/mod.rs +++ b/src/commit_builder/mod.rs @@ -182,6 +182,7 @@ impl<'a, R> CommitBuilder<'a, R> { read_model_plans: self.read_model_plans, snapshots: Vec::new(), inbox_receipts: Vec::new(), + alarms: Vec::new(), }) .await } diff --git a/src/in_memory_repo/repository.rs b/src/in_memory_repo/repository.rs index c2da67630..7608907bb 100644 --- a/src/in_memory_repo/repository.rs +++ b/src/in_memory_repo/repository.rs @@ -296,6 +296,7 @@ impl InMemoryRepository { mut completion: Option, direct_projection: Option, ) -> Result<(), InMemoryCommitError> { + crate::microsvc::cell_host::alarm::reject_alarm_writes(&batch.alarms)?; let prepared = validate_commit_batch(&batch)?; if let Some(direct_projection) = &direct_projection { direct_projection.validate()?; diff --git a/src/microsvc/causal.rs b/src/microsvc/causal.rs index 966db41dd..ff509b06a 100644 --- a/src/microsvc/causal.rs +++ b/src/microsvc/causal.rs @@ -24,6 +24,7 @@ use crate::command::{ }; use crate::command_ledger::CausalGetStream; use crate::domain_event::{DomainEventCaptureError, DomainEventCommitGuardError}; +use crate::microsvc::cell_host::CellAlarmWrite; use crate::outbox::{OutboxMessage, PreparedDomainEvent}; use crate::projection::lower::{ DirectCandidate, LoweredProjectionPlan, ProjectionDescriptor, @@ -155,11 +156,13 @@ pub(crate) enum CausalWorkspaceError { DomainPublicationsAlreadyPrepared, DomainPublicationsNotPrepared, CommitBatchAlreadyPrepared, + Alarm(crate::microsvc::cell_host::CellAlarmError), } impl std::fmt::Display for CausalWorkspaceError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Self::Alarm(error) => write!(formatter, "cell alarm staging failed: {error}"), Self::Repository(error) => write!(formatter, "causal aggregate load failed: {error}"), Self::Table(error) => write!(formatter, "causal read-model staging failed: {error}"), Self::DomainEventCapture(error) => { @@ -216,6 +219,7 @@ impl std::error::Error for CausalWorkspaceError { Self::DomainEventCapture(error) => Some(error), Self::DomainEventGuard(error) => Some(error), Self::Prepare(error) => Some(error), + Self::Alarm(error) => Some(error), Self::Poisoned | Self::IdentityChanged { .. } | Self::CommittedVersionChanged { .. } @@ -280,6 +284,7 @@ struct WorkspaceState { outbox_messages: Vec, read_model_plans: Vec, snapshots: Vec, + alarms: Vec, projection_staged: bool, modeled_direct_projection: Option, } @@ -292,6 +297,7 @@ impl Default for WorkspaceState { outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots: Vec::new(), + alarms: Vec::new(), projection_staged: false, modeled_direct_projection: None, } @@ -414,6 +420,22 @@ where Ok(()) } + /// Stage an alarm write. It commits with the command's events or not at + /// all; the batch is bounded by `MAX_CELL_ALARM_WRITES_PER_COMMAND`. + pub(crate) fn stage_alarm(&self, write: CellAlarmWrite) -> Result<(), CausalWorkspaceError> { + let mut state = self + .state + .lock() + .map_err(|_| CausalWorkspaceError::Poisoned)?; + if state.alarms.len() >= crate::microsvc::cell_host::MAX_CELL_ALARM_WRITES_PER_COMMAND { + return Err(CausalWorkspaceError::Alarm( + crate::microsvc::cell_host::CellAlarmError::TooManyWrites, + )); + } + state.alarms.push(write); + Ok(()) + } + pub(crate) fn stage_read_models( &self, builder: ReadModelWritePlanBuilder, @@ -545,6 +567,7 @@ where outbox_messages: state.outbox_messages, read_model_plans: state.read_model_plans, snapshots: state.snapshots, + alarms: state.alarms, domain_event_occurrences: Vec::new(), unpublished_direct_occurrences: HashSet::new(), modeled_direct_projection: state.modeled_direct_projection, @@ -562,6 +585,7 @@ pub(crate) struct CausalWorkspaceParts { outbox_messages: Vec, read_model_plans: Vec, snapshots: Vec, + alarms: Vec, /// Exact ledger-causation-stamped outward occurrences in publication order. domain_event_occurrences: Vec, /// Captured occurrences consumed only by a direct modeled projection. @@ -783,6 +807,7 @@ where batch.outbox_messages = std::mem::take(&mut self.outbox_messages); batch.read_model_plans = std::mem::take(&mut self.read_model_plans); batch.snapshots = std::mem::take(&mut self.snapshots); + batch.alarms = std::mem::take(&mut self.alarms); Ok(batch) } diff --git a/src/microsvc/cell_host/alarm.rs b/src/microsvc/cell_host/alarm.rs new file mode 100644 index 000000000..18270ddc9 --- /dev/null +++ b/src/microsvc/cell_host/alarm.rs @@ -0,0 +1,773 @@ +//! Re-armable cell alarms. +//! +//! A cell command handler arms an alarm with its events: +//! +//! ```rust,ignore +//! ctx.arm_alarm(CellAlarm::at("tick", due, "matchmaker.tick", &TickInput { pool })?)?; +//! ``` +//! +//! The alarm row commits in the **same transaction** as the command's events, +//! ledger receipt and outbox rows, so a failed command leaves no alarm. At +//! `due` the cell dispatches `command` into itself with a trusted system +//! session ([`cell_alarm_session`]); that handler may arm the same name again, +//! which gives a periodic tick without a global cron. +//! +//! Semantics, identical on celld and on the in-process reference host: +//! +//! - One row per alarm **name**. Arming an existing name replaces its due +//! time, command and input and bumps its `generation`; cancelling disarms it. +//! - A fired alarm runs through the cell's command ledger with a command id +//! derived from `(name, generation)`. A crash after the handler committed but +//! before the alarm was settled replays the recorded receipt instead of +//! running the handler twice, so each arm has its effects at most once. +//! - Settlement only disarms the generation that fired. A handler that re-arms +//! its own name produced a newer generation, which stays armed. +//! - Rejections (domain errors, guard failures, bad input) are terminal for +//! that arm. Retryable failures (in-progress, internal) back off +//! exponentially from one second up to five minutes. +//! +//! On celld the Durable Object has a single alarm slot, owned jointly with the +//! outbox watchdog (`celld_outbox.rs`): every `setAlarm` targets the earliest +//! of the next domain alarm and the watchdog's next drain. Neither replaces the +//! other. + +use std::time::{Duration, SystemTime}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use super::causal::{CellCommandIdentity, CellDispatchError}; +use crate::microsvc::session::{Session, ROLE_KEY, USER_ID_KEY}; +use crate::repository::RepositoryError; + +/// Role carried by the trusted session of a fired alarm. Commands that only +/// alarms may run declare `roles: ["cell-alarm"]` and +/// `guard: cell_alarm_guard`. +pub const CELL_ALARM_ROLE: &str = "cell-alarm"; +/// User id of the trusted session of a fired alarm. +pub const CELL_ALARM_USER_ID: &str = "system:cell-alarm"; +/// Session claim naming the alarm that fired. Set only by the cell host. +pub const CELL_ALARM_NAME_CLAIM: &str = "x-distributed-cell-alarm"; +/// Session claim with the fired alarm's due time (milliseconds since the Unix +/// epoch). Periodic handlers re-arm relative to it so ticks do not drift. +pub const CELL_ALARM_DUE_CLAIM: &str = "x-distributed-cell-alarm-due-ms"; +/// Session claim with the fired alarm's generation. +pub const CELL_ALARM_GENERATION_CLAIM: &str = "x-distributed-cell-alarm-generation"; +/// Command-ledger service id of fired alarms. +pub const CELL_ALARM_SERVICE_ID: &str = "distributed.cell-alarm"; +/// Longest alarm name. +pub const MAX_CELL_ALARM_NAME_BYTES: usize = 128; +/// Largest serialized alarm input. +pub const MAX_CELL_ALARM_INPUT_BYTES: usize = 16 * 1024; +/// Most alarm writes one command may stage. +pub const MAX_CELL_ALARM_WRITES_PER_COMMAND: usize = 16; +/// Most alarms one cell may hold armed at once. +pub const MAX_ARMED_CELL_ALARMS: usize = 1024; +/// Most due alarms fired per wake; the rest fire on the next wake. +pub const CELL_ALARM_FIRE_BATCH: usize = 32; +/// First retry delay after a retryable alarm failure. +pub const CELL_ALARM_RETRY_BASE: Duration = Duration::from_secs(1); +/// Longest retry delay after repeated retryable alarm failures. +pub const CELL_ALARM_RETRY_MAX: Duration = Duration::from_secs(300); + +/// Why an alarm could not be armed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CellAlarmError { + InvalidName, + EmptyCommand, + InvalidInput, + DueOutOfRange, + TooManyWrites, + TooManyArmed, +} + +impl std::fmt::Display for CellAlarmError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidName => write!( + f, + "cell alarm name must be 1..={MAX_CELL_ALARM_NAME_BYTES} bytes of [A-Za-z0-9._:-]" + ), + Self::EmptyCommand => f.write_str("cell alarm command must not be empty"), + Self::InvalidInput => write!( + f, + "cell alarm input must be a JSON object of at most {MAX_CELL_ALARM_INPUT_BYTES} bytes" + ), + Self::DueOutOfRange => f.write_str("cell alarm due time is out of range"), + Self::TooManyWrites => write!( + f, + "a command may stage at most {MAX_CELL_ALARM_WRITES_PER_COMMAND} cell alarm writes" + ), + Self::TooManyArmed => write!( + f, + "a cell may hold at most {MAX_ARMED_CELL_ALARMS} armed alarms" + ), + } + } +} + +impl std::error::Error for CellAlarmError {} + +impl From for RepositoryError { + fn from(error: CellAlarmError) -> Self { + RepositoryError::Model(error.to_string()) + } +} + +/// When an alarm is due. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AlarmDue { + /// Absolute epoch milliseconds. + At(i64), + /// Milliseconds after the commit, resolved by the cell's clock. + After(u64), +} + +/// An alarm a command arms: when due, dispatch `command` with `input`. +#[derive(Clone, Debug, PartialEq)] +pub struct CellAlarm { + name: String, + command: String, + input: Value, + due: AlarmDue, +} + +impl CellAlarm { + /// Arm `name` to dispatch `command` with `input` into this cell at `due`. + /// + /// Periodic handlers re-arm from the fired due time + /// (`ctx.alarm_due()`), so ticks do not drift with handler latency. + pub fn at( + name: impl Into, + due: SystemTime, + command: impl Into, + input: &impl Serialize, + ) -> Result { + Self::build(name, AlarmDue::At(system_time_to_ms(due)?), command, input) + } + + /// Arm `name` to fire `delay` after the command commits, measured by the + /// cell's clock (the Durable Object clock on celld, the injected + /// [`Clock`](crate::Clock) on the reference host). Domain handlers have no + /// wall clock of their own; this is how they schedule relative work. + pub fn after( + name: impl Into, + delay: Duration, + command: impl Into, + input: &impl Serialize, + ) -> Result { + let delay = u64::try_from(delay.as_millis()).map_err(|_| CellAlarmError::DueOutOfRange)?; + Self::build(name, AlarmDue::After(delay), command, input) + } + + fn build( + name: impl Into, + due: AlarmDue, + command: impl Into, + input: &impl Serialize, + ) -> Result { + let input = serde_json::to_value(input).map_err(|_| CellAlarmError::InvalidInput)?; + let alarm = Self { + name: name.into(), + command: command.into(), + input, + due, + }; + alarm.validate()?; + Ok(alarm) + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn command(&self) -> &str { + &self.command + } + + pub fn input(&self) -> &Value { + &self.input + } + + /// Absolute due time, or `None` for a relative alarm not yet committed. + pub fn due(&self) -> Option { + match self.due { + AlarmDue::At(ms) => Some(ms_to_system_time(ms)), + AlarmDue::After(_) => None, + } + } + + /// Resolve a relative due time against the committing cell's clock. + pub(crate) fn resolve(&mut self, now: SystemTime) -> Result<(), CellAlarmError> { + if let AlarmDue::After(delay) = self.due { + let now = system_time_to_ms(now)?; + let due = now + .checked_add(i64::try_from(delay).map_err(|_| CellAlarmError::DueOutOfRange)?) + .ok_or(CellAlarmError::DueOutOfRange)?; + if due > MAX_JS_SAFE_MS { + return Err(CellAlarmError::DueOutOfRange); + } + self.due = AlarmDue::At(due); + } + Ok(()) + } + + /// Absolute due time; storage refuses an unresolved relative alarm. + pub(crate) fn due_at_ms(&self) -> Result { + match self.due { + AlarmDue::At(ms) => Ok(ms), + AlarmDue::After(_) => Err(RepositoryError::Model( + "relative cell alarm reached storage without being resolved".into(), + )), + } + } + + pub(crate) fn input_json(&self) -> Result { + serde_json::to_string(&self.input) + .map_err(|error| RepositoryError::Model(format!("cell alarm input: {error}"))) + } + + pub(crate) fn validate(&self) -> Result<(), CellAlarmError> { + validate_name(&self.name)?; + if self.command.trim().is_empty() { + return Err(CellAlarmError::EmptyCommand); + } + let size = serde_json::to_vec(&self.input) + .map_err(|_| CellAlarmError::InvalidInput)? + .len(); + if !self.input.is_object() || size > MAX_CELL_ALARM_INPUT_BYTES { + return Err(CellAlarmError::InvalidInput); + } + match self.due { + AlarmDue::At(ms) if !(0..=MAX_JS_SAFE_MS).contains(&ms) => { + Err(CellAlarmError::DueOutOfRange) + } + AlarmDue::After(delay) if delay > MAX_JS_SAFE_MS as u64 => { + Err(CellAlarmError::DueOutOfRange) + } + _ => Ok(()), + } + } +} + +/// Resolve every relative alarm of a batch against the cell clock. +pub(crate) fn resolve_alarm_writes( + writes: &mut [CellAlarmWrite], + now: SystemTime, +) -> Result<(), RepositoryError> { + for write in writes { + if let CellAlarmWrite::Arm(alarm) = write { + alarm.resolve(now)?; + } + } + Ok(()) +} + +/// Durable Object SQL binds integers through JS numbers. +const MAX_JS_SAFE_MS: i64 = 9_007_199_254_740_991; + +/// One alarm write staged by a command and committed with its events. +#[derive(Clone, Debug, PartialEq)] +pub enum CellAlarmWrite { + /// Arm (or re-arm) the alarm's name. + Arm(CellAlarm), + /// Disarm a name. Cancelling a name that is not armed is a no-op. + Cancel(String), +} + +impl CellAlarmWrite { + pub fn name(&self) -> &str { + match self { + Self::Arm(alarm) => alarm.name(), + Self::Cancel(name) => name, + } + } +} + +/// Validate a batch's alarm writes before any storage participant runs. +pub(crate) fn validate_alarm_writes(writes: &[CellAlarmWrite]) -> Result<(), RepositoryError> { + if writes.len() > MAX_CELL_ALARM_WRITES_PER_COMMAND { + return Err(CellAlarmError::TooManyWrites.into()); + } + for write in writes { + match write { + CellAlarmWrite::Arm(alarm) => alarm.validate()?, + CellAlarmWrite::Cancel(name) => validate_name(name)?, + } + } + Ok(()) +} + +/// Ordinary repositories have no alarm slot; only aggregate cells do. +pub(crate) fn reject_alarm_writes(writes: &[CellAlarmWrite]) -> Result<(), RepositoryError> { + if writes.is_empty() { + Ok(()) + } else { + Err(RepositoryError::Model( + "cell alarms require an aggregate cell host".into(), + )) + } +} + +/// An armed alarm as stored by the cell. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ArmedCellAlarm { + pub name: String, + pub command: String, + pub input: Value, + pub due_at_ms: i64, + pub generation: u64, + pub attempts: u32, +} + +impl ArmedCellAlarm { + pub fn due(&self) -> SystemTime { + ms_to_system_time(self.due_at_ms) + } + + /// Stable command-ledger identity of this arm. + pub fn command_identity(&self) -> Result { + CellCommandIdentity::new( + CELL_ALARM_SERVICE_ID, + CELL_ALARM_SERVICE_ID, + alarm_command_id(&self.name, self.generation), + ) + } +} + +/// How a fired alarm settled. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum AlarmSettlement { + /// Ran (or replayed) or failed terminally: disarm this generation. + Done, + /// Retry this generation at `due_at_ms`. + Retry { due_at_ms: i64 }, +} + +/// What one wake did with the due alarms. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CellAlarmFireReport { + /// Alarms whose command committed (or replayed a committed receipt). + pub fired: Vec, + /// Alarms rescheduled after a retryable failure. + pub retried: Vec, + /// Alarms disarmed after a terminal failure. + pub failed: Vec, +} + +impl CellAlarmFireReport { + pub fn is_empty(&self) -> bool { + self.fired.is_empty() && self.retried.is_empty() && self.failed.is_empty() + } +} + +/// A fired alarm. +#[derive(Clone, Debug, PartialEq)] +pub struct FiredCellAlarm { + pub name: String, + pub generation: u64, + pub command: String, + /// The command had already committed; its receipt was replayed. + pub replayed: bool, + pub payload: Value, +} + +/// An alarm whose command failed. +#[derive(Clone, Debug, PartialEq)] +pub struct FailedCellAlarm { + pub name: String, + pub generation: u64, + pub command: String, + pub code: &'static str, + pub message: String, +} + +/// The trusted system session a fired alarm dispatches with. +pub fn cell_alarm_session(alarm: &ArmedCellAlarm) -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, CELL_ALARM_USER_ID); + session.set(ROLE_KEY, CELL_ALARM_ROLE); + session.set(CELL_ALARM_NAME_CLAIM, alarm.name.clone()); + session.set(CELL_ALARM_DUE_CLAIM, alarm.due_at_ms.to_string()); + session.set(CELL_ALARM_GENERATION_CLAIM, alarm.generation.to_string()); + session +} + +/// Whether `session` is the trusted session of a fired alarm. +pub fn is_cell_alarm_session(session: &Session) -> bool { + session.user_id() == Some(CELL_ALARM_USER_ID) + && session.has_role(CELL_ALARM_ROLE) + && session.get(CELL_ALARM_NAME_CLAIM).is_some() +} + +/// Due time of the alarm that fired this command, when it was an alarm. +pub fn cell_alarm_due(session: &Session) -> Option { + if !is_cell_alarm_session(session) { + return None; + } + session + .get(CELL_ALARM_DUE_CLAIM)? + .parse::() + .ok() + .filter(|ms| *ms >= 0) + .map(ms_to_system_time) +} + +/// `portable_command!` guard admitting only fired alarms. +pub fn cell_alarm_guard(ctx: &crate::microsvc::CausalCommandContext<'_, A>) -> bool +where + A: crate::Aggregate + Send + Sync + 'static, +{ + is_cell_alarm_session(ctx.session()) +} + +/// Deterministic UUIDv7-shaped command id of one alarm arm. +pub(crate) fn alarm_command_id(name: &str, generation: u64) -> String { + let mut hasher = Sha256::new(); + hasher.update(CELL_ALARM_SERVICE_ID.as_bytes()); + hasher.update([0u8]); + hasher.update(name.as_bytes()); + hasher.update([0u8]); + hasher.update(generation.to_be_bytes()); + let digest = hasher.finalize(); + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&digest[..16]); + bytes[6] = (bytes[6] & 0x0f) | 0x70; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + uuid::Uuid::from_bytes(bytes).hyphenated().to_string() +} + +/// Exponential retry delay for the `attempts`-th failure (0-based). +pub(crate) fn retry_delay(attempts: u32) -> Duration { + let factor = 1u32.checked_shl(attempts.min(16)).unwrap_or(u32::MAX); + CELL_ALARM_RETRY_BASE + .checked_mul(factor) + .unwrap_or(CELL_ALARM_RETRY_MAX) + .min(CELL_ALARM_RETRY_MAX) +} + +/// Whether a dispatch failure should be retried rather than disarm the alarm. +pub(crate) fn is_retryable(error: &CellDispatchError) -> bool { + matches!( + error, + CellDispatchError::InProgress | CellDispatchError::Internal(_) + ) +} + +pub(crate) fn system_time_to_ms(time: SystemTime) -> Result { + let millis = time + .duration_since(SystemTime::UNIX_EPOCH) + .map_err(|_| CellAlarmError::DueOutOfRange)? + .as_millis(); + if millis > MAX_JS_SAFE_MS as u128 { + return Err(CellAlarmError::DueOutOfRange); + } + Ok(millis as i64) +} + +pub(crate) fn ms_to_system_time(ms: i64) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_millis(ms.max(0) as u64) +} + +fn validate_name(name: &str) -> Result<(), CellAlarmError> { + if name.is_empty() + || name.len() > MAX_CELL_ALARM_NAME_BYTES + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) + { + return Err(CellAlarmError::InvalidName); + } + Ok(()) +} + +/// The in-process reference host's alarm table, with the same semantics as +/// the SQL table used by Durable Object and native SQLite cells. +#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] +#[derive(Clone, Debug, Default)] +pub(crate) struct MemoryCellAlarms { + rows: std::collections::BTreeMap, +} + +#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] +#[derive(Clone, Debug)] +struct MemoryAlarmRow { + alarm: ArmedCellAlarm, + armed: bool, +} + +#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] +impl MemoryCellAlarms { + /// Apply one command's writes all-or-nothing, as the SQL transaction does. + pub(crate) fn plan(&self, writes: &[CellAlarmWrite]) -> Result { + let mut next = self.clone(); + for write in writes { + match write { + CellAlarmWrite::Arm(alarm) => { + let generation = next + .rows + .get(alarm.name()) + .map_or(1, |row| row.alarm.generation + 1); + next.rows.insert( + alarm.name().to_string(), + MemoryAlarmRow { + alarm: ArmedCellAlarm { + name: alarm.name().to_string(), + command: alarm.command().to_string(), + input: alarm.input().clone(), + due_at_ms: alarm.due_at_ms()?, + generation, + attempts: 0, + }, + armed: true, + }, + ); + } + CellAlarmWrite::Cancel(name) => { + if let Some(row) = next.rows.get_mut(name) { + row.armed = false; + } + } + } + } + if next.rows.values().filter(|row| row.armed).count() > MAX_ARMED_CELL_ALARMS { + return Err(CellAlarmError::TooManyArmed.into()); + } + Ok(next) + } + + pub(crate) fn armed(&self) -> Vec { + let mut armed = self + .rows + .values() + .filter(|row| row.armed) + .map(|row| row.alarm.clone()) + .collect::>(); + armed.sort_by(|a, b| (a.due_at_ms, &a.name).cmp(&(b.due_at_ms, &b.name))); + armed + } + + pub(crate) fn due(&self, now_ms: i64, limit: usize) -> Vec { + self.armed() + .into_iter() + .filter(|alarm| alarm.due_at_ms <= now_ms) + .take(limit) + .collect() + } + + pub(crate) fn next_due_ms(&self) -> Option { + self.rows + .values() + .filter(|row| row.armed) + .map(|row| row.alarm.due_at_ms) + .min() + } + + pub(crate) fn settle(&mut self, name: &str, generation: u64, settlement: &AlarmSettlement) { + let Some(row) = self.rows.get_mut(name) else { + return; + }; + if !row.armed || row.alarm.generation != generation { + return; + } + match settlement { + AlarmSettlement::Done => { + row.armed = false; + row.alarm.attempts = 0; + } + AlarmSettlement::Retry { due_at_ms } => { + row.alarm.attempts = row.alarm.attempts.saturating_add(1); + row.alarm.due_at_ms = *due_at_ms; + } + } + } + + /// Every row (armed or not), for reference-host restart exports. + pub(crate) fn export(&self) -> Vec { + self.rows + .values() + .map(|row| DurableCellAlarm { + alarm: row.alarm.clone(), + armed: row.armed, + }) + .collect() + } + + pub(crate) fn restore(rows: Vec) -> Self { + Self { + rows: rows + .into_iter() + .map(|row| { + ( + row.alarm.name.clone(), + MemoryAlarmRow { + alarm: row.alarm, + armed: row.armed, + }, + ) + }) + .collect(), + } + } +} + +/// One alarm row in a reference-host restart export. Disarmed rows are kept so +/// generations never repeat after a restore. +#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DurableCellAlarm { + pub alarm: ArmedCellAlarm, + pub armed: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn alarm(name: &str, due_ms: u64) -> CellAlarm { + CellAlarm::at( + name, + SystemTime::UNIX_EPOCH + Duration::from_millis(due_ms), + "test.tick", + &serde_json::json!({ "n": 1 }), + ) + .unwrap() + } + + #[test] + fn validates_names_inputs_and_write_counts() { + let due = SystemTime::UNIX_EPOCH + Duration::from_secs(1); + assert_eq!( + CellAlarm::at("", due, "c", &serde_json::json!({})).unwrap_err(), + CellAlarmError::InvalidName + ); + assert_eq!( + CellAlarm::at("a b", due, "c", &serde_json::json!({})).unwrap_err(), + CellAlarmError::InvalidName + ); + assert_eq!( + CellAlarm::at("ok", due, " ", &serde_json::json!({})).unwrap_err(), + CellAlarmError::EmptyCommand + ); + assert_eq!( + CellAlarm::at("ok", due, "c", &1).unwrap_err(), + CellAlarmError::InvalidInput + ); + let big = serde_json::json!({ "x": "y".repeat(MAX_CELL_ALARM_INPUT_BYTES) }); + assert_eq!( + CellAlarm::at("ok", due, "c", &big).unwrap_err(), + CellAlarmError::InvalidInput + ); + let writes = (0..=MAX_CELL_ALARM_WRITES_PER_COMMAND) + .map(|index| CellAlarmWrite::Cancel(format!("a{index}"))) + .collect::>(); + assert!(validate_alarm_writes(&writes).is_err()); + assert!(validate_alarm_writes(&writes[1..]).is_ok()); + assert!(reject_alarm_writes(&writes).is_err()); + assert!(reject_alarm_writes(&[]).is_ok()); + } + + #[test] + fn command_ids_are_uuidv7_and_unique_per_generation() { + let first = alarm_command_id("tick", 1); + assert_eq!(first, alarm_command_id("tick", 1)); + assert_ne!(first, alarm_command_id("tick", 2)); + assert_ne!(first, alarm_command_id("tock", 1)); + let parsed = uuid::Uuid::parse_str(&first).unwrap(); + assert_eq!(parsed.get_version_num(), 7); + let armed = ArmedCellAlarm { + name: "tick".into(), + command: "c".into(), + input: serde_json::json!({}), + due_at_ms: 0, + generation: 1, + attempts: 0, + }; + assert_eq!(armed.command_identity().unwrap().command_id(), first); + } + + #[test] + fn retry_delay_doubles_and_caps() { + assert_eq!(retry_delay(0), Duration::from_secs(1)); + assert_eq!(retry_delay(3), Duration::from_secs(8)); + assert_eq!(retry_delay(40), CELL_ALARM_RETRY_MAX); + } + + #[test] + fn alarm_session_is_recognized_and_carries_due() { + let armed = ArmedCellAlarm { + name: "tick".into(), + command: "c".into(), + input: serde_json::json!({}), + due_at_ms: 1_234, + generation: 7, + attempts: 0, + }; + let session = cell_alarm_session(&armed); + assert!(is_cell_alarm_session(&session)); + assert_eq!( + cell_alarm_due(&session), + Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_234)) + ); + let mut forged = Session::new(); + forged.set(USER_ID_KEY, "player-1"); + forged.set(ROLE_KEY, CELL_ALARM_ROLE); + forged.set(CELL_ALARM_NAME_CLAIM, "tick"); + assert!(!is_cell_alarm_session(&forged)); + assert_eq!(cell_alarm_due(&forged), None); + } + + #[test] + fn memory_table_rearms_with_new_generation_and_settles_only_its_own() { + let table = MemoryCellAlarms::default(); + let table = table + .plan(&[CellAlarmWrite::Arm(alarm("tick", 10))]) + .unwrap(); + assert_eq!(table.armed()[0].generation, 1); + let mut table = table + .plan(&[CellAlarmWrite::Arm(alarm("tick", 20))]) + .unwrap(); + assert_eq!(table.armed()[0].generation, 2); + assert_eq!(table.next_due_ms(), Some(20)); + assert!(table.due(19, 10).is_empty()); + table.settle("tick", 1, &AlarmSettlement::Done); + assert_eq!(table.armed().len(), 1, "stale generation must not disarm"); + table.settle("tick", 2, &AlarmSettlement::Retry { due_at_ms: 50 }); + assert_eq!(table.armed()[0].attempts, 1); + assert_eq!(table.next_due_ms(), Some(50)); + table.settle("tick", 2, &AlarmSettlement::Done); + assert!(table.armed().is_empty()); + let table = table + .plan(&[CellAlarmWrite::Arm(alarm("tick", 60))]) + .unwrap(); + assert_eq!( + table.armed()[0].generation, + 3, + "generations never repeat after a disarm" + ); + let restored = MemoryCellAlarms::restore(table.export()); + assert_eq!(restored.armed(), table.armed()); + let cancelled = restored + .plan(&[CellAlarmWrite::Cancel("tick".into())]) + .unwrap(); + assert!(cancelled.armed().is_empty()); + } + + #[test] + fn memory_table_caps_armed_alarms() { + let mut table = MemoryCellAlarms::default(); + for chunk in 0..(MAX_ARMED_CELL_ALARMS / 16) { + let writes = (0..16) + .map(|index| CellAlarmWrite::Arm(alarm(&format!("a{chunk}-{index}"), 1))) + .collect::>(); + table = table.plan(&writes).unwrap(); + } + let error = table + .plan(&[CellAlarmWrite::Arm(alarm("overflow", 1))]) + .unwrap_err(); + assert!(error.to_string().contains("armed alarms")); + } +} diff --git a/src/microsvc/cell_host/cell.rs b/src/microsvc/cell_host/cell.rs index 0eb92d796..7943826dc 100644 --- a/src/microsvc/cell_host/cell.rs +++ b/src/microsvc/cell_host/cell.rs @@ -2,10 +2,11 @@ //! one instance per shard (`{aggregate_type}:{shard}`). use std::collections::HashMap; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use serde_json::Value; +use super::alarm::{AlarmSettlement, ArmedCellAlarm, CellAlarmFireReport}; use super::causal::{CellCommandIdentity, CellDispatchError, CellDispatchResult}; use super::store::CellStreamStore; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] @@ -230,9 +231,13 @@ where .begin_command() .await .map_err(|error| HandlerError::Other(Box::new(error)))?; - self.routes + let result = self + .routes .dispatch_cell_command(command, input, session, &self.shard) - .await + .await; + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + self.reschedule_after_command().await; + result } /// Dispatch a wait-path command through this cell's fenced command ledger. @@ -252,11 +257,109 @@ where .begin_command() .await .map_err(|error| CellDispatchError::Internal(error.to_string()))?; - self.routes + let result = self + .routes .dispatch_cell_causal(command, identity, input, session, &self.shard) + .await; + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + self.reschedule_after_command().await; + result + } + + /// Armed alarms of this cell, earliest first. + pub fn armed_alarms(&self) -> Result, RepositoryError> { + self.routes.repo().repo().armed_alarms() + } + + /// Due time of this cell's earliest armed alarm. + pub fn next_alarm_due(&self) -> Result, RepositoryError> { + Ok(self + .routes + .repo() + .repo() + .next_alarm_due_ms()? + .map(super::alarm::ms_to_system_time)) + } + + /// Fire this cell's alarms that are due now by the cell clock. + /// + /// The in-process reference host has no timer: tests give the store a + /// [`ManualClock`](crate::ManualClock) ([`CellStreamStore::with_clock`]), + /// advance it, and call this, exactly as the celld alarm handler does on + /// every Durable Object wake. + pub async fn fire_due_alarms(&self) -> Result { + self.fire_due_alarms_at(self.routes.repo().repo().now()) .await } + /// Fire up to [`CELL_ALARM_FIRE_BATCH`](super::CELL_ALARM_FIRE_BATCH) + /// alarms due at or before `now`, earliest first. + /// + /// Each one dispatches its command through this cell's command ledger with + /// a stable `(name, generation)` command id and the trusted alarm session. + /// The due set is read once, so a handler that re-arms in the past fires on + /// the next wake, not in a loop. + pub async fn fire_due_alarms_at( + &self, + now: SystemTime, + ) -> Result { + let store = self.routes.repo().repo(); + let now_ms = super::alarm::system_time_to_ms(now)?; + let mut report = CellAlarmFireReport::default(); + for alarm in store.due_alarms(now_ms, super::CELL_ALARM_FIRE_BATCH)? { + let result = match alarm.command_identity() { + Ok(identity) => { + self.dispatch_idempotent( + &alarm.command, + &identity, + alarm.input.clone(), + super::cell_alarm_session(&alarm), + ) + .await + } + Err(error) => Err(error), + }; + let settlement = match result { + Ok(dispatch) => { + report.fired.push(super::FiredCellAlarm { + name: alarm.name.clone(), + generation: alarm.generation, + command: alarm.command.clone(), + replayed: dispatch.replayed(), + payload: dispatch.payload().clone(), + }); + AlarmSettlement::Done + } + Err(error) => { + let retry = super::alarm::is_retryable(&error); + let failed = super::FailedCellAlarm { + name: alarm.name.clone(), + generation: alarm.generation, + command: alarm.command.clone(), + code: error.code(), + message: match &error { + CellDispatchError::Internal(message) => message.clone(), + other => other.client_message(), + }, + }; + if retry { + report.retried.push(failed); + AlarmSettlement::Retry { + due_at_ms: now_ms.saturating_add( + super::alarm::retry_delay(alarm.attempts).as_millis() as i64, + ), + } + } else { + report.failed.push(failed); + AlarmSettlement::Done + } + } + }; + store.settle_alarm(&alarm.name, alarm.generation, &settlement)?; + } + Ok(report) + } + /// Load this cell's aggregate from the private stream store. /// /// HTTP GET on the cell host is a stream load, not a GraphQL/projector @@ -335,6 +438,19 @@ where ) } + /// Rows of this cell's outbox still awaiting delivery. + pub async fn outbox_backlog( + &self, + ) -> Result { + use crate::outbox_worker::OutboxStore; + self.routes + .repo() + .repo() + .outbox_store() + .backlog_stats() + .await + } + /// Attach the celld Queue binding and drain policy used by this cell host. #[cfg(feature = "workers-rs")] pub fn with_celld_outbox(mut self, outbox: super::celld_outbox::CelldOutbox) -> Self { @@ -350,7 +466,62 @@ where &self, env: &worker::Env, ) -> Result { - self.outbox_config()?.drain(self, env, &self.storage).await + let next_alarm = self.next_alarm_due().map_err(alarm_transport_error)?; + self.outbox_config()? + .drain(self, env, &self.storage, next_alarm) + .await + } + + /// The Durable Object alarm handler: fire due domain alarms, then drain + /// the outbox. The one Durable Object alarm slot is shared: every wake is + /// set to the earliest of the next domain alarm and the outbox watchdog, + /// so neither starves the other. + /// + /// ```rust,ignore + /// async fn alarm(&self) -> Result { + /// let wake = self.cell.alarm(&self.env).await?; + /// Response::ok("ok") + /// } + /// ``` + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + pub async fn alarm( + &self, + env: &worker::Env, + ) -> Result { + let alarms = self + .fire_due_alarms() + .await + .map_err(alarm_transport_error)?; + let drain = self.drain_outbox(env).await?; + Ok(super::celld_outbox::CelldAlarmOutcome { alarms, drain }) + } + + /// A command may have armed an alarm due before the prearmed watchdog + /// wake; move the one Durable Object alarm earlier if so. Failure is only + /// logged: the prearmed watchdog still wakes the cell, which re-plans. + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + async fn reschedule_after_command(&self) { + let Ok(outbox) = self.outbox_config() else { + return; + }; + let next_alarm = match self.next_alarm_due() { + Ok(next_alarm) => next_alarm, + Err(error) => { + worker::console_error!("cell alarm lookup failed after command: {error}"); + return; + } + }; + if let Err(error) = outbox + .schedule( + &super::celld_outbox::WorkerAlarm(&self.storage), + crate::time::now(), + true, + next_alarm, + ) + .await + { + worker::console_error!("cell alarm reschedule failed after command: {error}"); + } } #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] @@ -368,10 +539,13 @@ where async fn begin_command( &self, ) -> Result { + let next_alarm = self.next_alarm_due().map_err(alarm_transport_error)?; self.outbox_config()? .begin_command( &self.activity, &super::celld_outbox::WorkerAlarm(&self.storage), + crate::time::now(), + next_alarm, ) .await } @@ -592,6 +766,11 @@ where } } +#[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] +fn alarm_transport_error(error: RepositoryError) -> crate::bus::TransportError { + crate::bus::TransportError::retryable(format!("cell alarm storage: {error}")) +} + /// Cell instance name: `{aggregate_type}:{shard_id}`. pub fn instance_name(shard_id: &str) -> String { format!("{}:{shard_id}", A::aggregate_type()) diff --git a/src/microsvc/cell_host/celld_outbox.rs b/src/microsvc/cell_host/celld_outbox.rs index 691a7c674..ea33c6e64 100644 --- a/src/microsvc/cell_host/celld_outbox.rs +++ b/src/microsvc/cell_host/celld_outbox.rs @@ -6,6 +6,8 @@ use std::sync::{ Arc, }; use std::time::Duration; +#[cfg(any(target_arch = "wasm32", test))] +use std::time::SystemTime; use worker::Env; #[cfg(target_arch = "wasm32")] @@ -49,6 +51,18 @@ pub struct CelldOutboxDrainOutcome { pub deferred: Vec, } +/// Result of one Durable Object alarm wake ([`AggregateCell::alarm`]). +/// +/// [`AggregateCell::alarm`]: super::AggregateCell +#[must_use = "inspect fired alarms and deferred drain diagnostics"] +#[derive(Debug, Default)] +pub struct CelldAlarmOutcome { + /// Domain alarms fired, retried or failed by this wake. + pub alarms: super::CellAlarmFireReport, + /// The outbox drain that followed. + pub drain: CelldOutboxDrainOutcome, +} + impl CelldOutboxDrainOutcome { /// Whether the durable watchdog still owns work from this drain attempt. pub fn is_deferred(&self) -> bool { @@ -173,6 +187,7 @@ impl CelldOutbox { cell: &AggregateCell, env: &Env, storage: &Storage, + next_alarm: Option, ) -> Result where A: Aggregate + Send + Sync + 'static, @@ -181,20 +196,51 @@ impl CelldOutbox { cell, || CelldQueuePublisher::from_env(env, &self.binding), &WorkerAlarm(storage), + crate::time::now(), + next_alarm, ) .await } + /// The one Durable Object alarm slot is shared by this watchdog and the + /// cell's domain alarms. Every write targets the earliest due time, so a + /// watchdog rearm never postpones a domain alarm and a far domain alarm + /// never postpones a drain. Returns `false` (and leaves the slot alone: + /// never delete an alarm) when neither wants a wake. + #[cfg(any(target_arch = "wasm32", test))] + pub(super) async fn schedule( + &self, + alarm: &W, + now: SystemTime, + watchdog: bool, + next_alarm: Option, + ) -> Result { + let watchdog = watchdog.then(|| now + self.drain_interval); + let target = match (watchdog, next_alarm) { + (Some(watchdog), Some(domain)) => Some(watchdog.min(domain)), + (watchdog, domain) => watchdog.or(domain), + }; + match target { + Some(target) => { + alarm.set(target, now).await?; + Ok(true) + } + None => Ok(false), + } + } + #[cfg(any(target_arch = "wasm32", test))] pub(super) async fn begin_command( &self, activity: &CommandActivity, alarm: &W, + now: SystemTime, + next_alarm: Option, ) -> Result { // Count the command before yielding to setAlarm. An alarm firing while // this command is suspended must keep the next wake scheduled. let guard = activity.enter()?; - alarm.arm(self.drain_interval).await?; + self.schedule(alarm, now, true, next_alarm).await?; Ok(guard) } @@ -204,6 +250,8 @@ impl CelldOutbox { cell: &AggregateCell, publisher: PF, alarm: &W, + now: SystemTime, + next_alarm: Option, ) -> Result where A: Aggregate + Send + Sync + 'static, @@ -212,15 +260,13 @@ impl CelldOutbox { W: CelldOutboxAlarm, { let pending = has_pending(cell).await?; - if !pending && !cell.command_activity().is_active() { - // Never delete an alarm: doing so can erase a concurrent command's - // prearmed wake. The last scheduled alarm simply finds no work. - return Ok(CelldOutboxDrainOutcome::default()); - } - + let watchdog = pending || cell.command_activity().is_active(); // Arm before Queue I/O, including during an alarm invocation. A crash - // after Queue acceptance but before deletion retries the stable event ID. - alarm.arm(self.drain_interval).await?; + // after Queue acceptance but before deletion retries the stable event + // ID. With no outbox work and no domain alarm, never delete an alarm: + // doing so can erase a concurrent command's prearmed wake. The last + // scheduled alarm simply finds no work. + self.schedule(alarm, now, watchdog, next_alarm).await?; if !pending { return Ok(CelldOutboxDrainOutcome::default()); } @@ -285,7 +331,9 @@ impl Drop for CommandGuard { #[cfg(any(target_arch = "wasm32", test))] #[async_trait::async_trait(?Send)] pub(super) trait CelldOutboxAlarm { - async fn arm(&self, interval: Duration) -> Result<(), TransportError>; + /// Set the single Durable Object alarm to `at` (`now` is the caller's + /// clock reading; a past `at` fires as soon as possible). + async fn set(&self, at: SystemTime, now: SystemTime) -> Result<(), TransportError>; } #[cfg(target_arch = "wasm32")] @@ -293,9 +341,10 @@ pub(super) struct WorkerAlarm<'a>(pub &'a Storage); #[cfg(target_arch = "wasm32")] #[async_trait::async_trait(?Send)] impl CelldOutboxAlarm for WorkerAlarm<'_> { - async fn arm(&self, interval: Duration) -> Result<(), TransportError> { - self.0.set_alarm(interval).await.map_err(|error| { - TransportError::retryable(format!("cannot arm celld outbox watchdog: {error}")) + async fn set(&self, at: SystemTime, now: SystemTime) -> Result<(), TransportError> { + let offset = at.duration_since(now).unwrap_or(Duration::ZERO); + self.0.set_alarm(offset).await.map_err(|error| { + TransportError::retryable(format!("cannot arm celld cell alarm: {error}")) }) } } @@ -367,12 +416,17 @@ mod tests { } struct RecordingAlarm { log: Arc>>, + targets: Mutex>, fail: bool, } #[async_trait::async_trait(?Send)] impl CelldOutboxAlarm for RecordingAlarm { - async fn arm(&self, _: Duration) -> Result<(), TransportError> { + async fn set(&self, at: SystemTime, now: SystemTime) -> Result<(), TransportError> { self.log.lock().unwrap().push("arm"); + self.targets + .lock() + .unwrap() + .push(at.duration_since(now).unwrap_or(Duration::ZERO)); if self.fail { Err(TransportError::retryable("alarm unavailable")) } else { @@ -380,6 +434,8 @@ mod tests { } } } + const NOW: SystemTime = SystemTime::UNIX_EPOCH; + fn cell(pending: bool) -> AggregateCell { let cell = AggregateCell::new("aggregate-1").unwrap(); if pending { @@ -399,6 +455,7 @@ mod tests { CelldOutbox::new("OUTBOX").unwrap(), RecordingAlarm { log: log.clone(), + targets: Mutex::new(Vec::new()), fail: false, }, RecordingPublisher { log, fail: false }, @@ -435,13 +492,13 @@ mod tests { let cell = cell(true); let (outbox, alarm, publisher) = fixtures(); let guard = outbox - .begin_command(cell.command_activity(), &alarm) + .begin_command(cell.command_activity(), &alarm, NOW, None) .await .unwrap(); alarm.log.lock().unwrap().push("command"); drop(guard); let result = outbox - .drain_with(&cell, || Ok(publisher), &alarm) + .drain_with(&cell, || Ok(publisher), &alarm, NOW, None) .await .unwrap(); assert_eq!(result.dispatch.published, 1); @@ -453,7 +510,7 @@ mod tests { ); let (_, _, publisher) = fixtures(); assert!(!outbox - .drain_with(&cell, || Ok(publisher), &alarm) + .drain_with(&cell, || Ok(publisher), &alarm, NOW, None) .await .unwrap() .is_deferred()); @@ -471,7 +528,7 @@ mod tests { let (outbox, alarm, mut publisher) = fixtures(); publisher.fail = true; let result = outbox - .drain_with(&cell, || Ok(publisher), &alarm) + .drain_with(&cell, || Ok(publisher), &alarm, NOW, None) .await .unwrap(); assert_eq!(result.dispatch.released, 1); @@ -488,7 +545,7 @@ mod tests { let (outbox, mut alarm, _) = fixtures(); alarm.fail = true; assert!(outbox - .begin_command(cell.command_activity(), &alarm) + .begin_command(cell.command_activity(), &alarm, NOW, None) .await .is_err()); assert!(!cell.command_activity().is_active()); @@ -501,28 +558,28 @@ mod tests { let cell = cell(false); let (outbox, alarm, publisher) = fixtures(); let first = outbox - .begin_command(cell.command_activity(), &alarm) + .begin_command(cell.command_activity(), &alarm, NOW, None) .await .unwrap(); assert!(!outbox - .drain_with(&cell, || Ok(publisher.clone()), &alarm) + .drain_with(&cell, || Ok(publisher.clone()), &alarm, NOW, None) .await .unwrap() .is_deferred()); let second = outbox - .begin_command(cell.command_activity(), &alarm) + .begin_command(cell.command_activity(), &alarm, NOW, None) .await .unwrap(); drop(first); assert!(!outbox - .drain_with(&cell, || Ok(publisher.clone()), &alarm) + .drain_with(&cell, || Ok(publisher.clone()), &alarm, NOW, None) .await .unwrap() .is_deferred()); assert_eq!(*alarm.log.lock().unwrap(), ["arm", "arm", "arm", "arm"]); drop(second); assert!(!outbox - .drain_with(&cell, || Ok(publisher), &alarm) + .drain_with(&cell, || Ok(publisher), &alarm, NOW, None) .await .unwrap() .is_deferred()); @@ -539,6 +596,8 @@ mod tests { &cell, || Err::(TransportError::permanent("missing Queue")), &alarm, + NOW, + None, ) .await .unwrap(); @@ -554,7 +613,7 @@ mod tests { let (outbox, mut alarm, publisher) = fixtures(); alarm.fail = true; assert!(outbox - .drain_with(&cell, || Ok(publisher), &alarm) + .drain_with(&cell, || Ok(publisher), &alarm, NOW, None) .await .is_err()); assert_eq!(cell.durable_outbox().unwrap().len(), 1); @@ -575,7 +634,7 @@ mod tests { .await .unwrap(); let result = outbox - .drain_with(&cell, || Ok(publisher), &alarm) + .drain_with(&cell, || Ok(publisher), &alarm, NOW, None) .await .unwrap(); assert_eq!(result.dispatch.published, 0); @@ -586,4 +645,72 @@ mod tests { ); }); } + + #[test] + fn domain_alarm_earlier_than_watchdog_wins_the_slot() { + block_on(async { + let cell = cell(true); + let (outbox, alarm, publisher) = fixtures(); + let soon = NOW + Duration::from_millis(250); + let guard = outbox + .begin_command(cell.command_activity(), &alarm, NOW, Some(soon)) + .await + .unwrap(); + drop(guard); + let result = outbox + .drain_with(&cell, || Ok(publisher), &alarm, NOW, Some(soon)) + .await + .unwrap(); + assert_eq!(result.dispatch.published, 1); + assert_eq!( + *alarm.targets.lock().unwrap(), + [Duration::from_millis(250), Duration::from_millis(250)], + "a watchdog rearm must never postpone an earlier domain alarm" + ); + }); + } + + #[test] + fn watchdog_earlier_than_domain_alarm_keeps_draining() { + block_on(async { + let cell = cell(true); + let (outbox, alarm, publisher) = fixtures(); + let later = NOW + Duration::from_secs(3_600); + let result = outbox + .drain_with(&cell, || Ok(publisher), &alarm, NOW, Some(later)) + .await + .unwrap(); + assert_eq!(result.dispatch.published, 1); + assert_eq!( + *alarm.targets.lock().unwrap(), + [outbox.drain_interval()], + "a far domain alarm must not starve the outbox watchdog" + ); + }); + } + + #[test] + fn idle_cell_arms_only_its_next_domain_alarm() { + block_on(async { + let cell = cell(false); + let (outbox, alarm, publisher) = fixtures(); + let later = NOW + Duration::from_secs(60); + assert!(!outbox + .drain_with(&cell, || Ok(publisher.clone()), &alarm, NOW, Some(later)) + .await + .unwrap() + .is_deferred()); + assert_eq!(*alarm.targets.lock().unwrap(), [Duration::from_secs(60)]); + assert!(!outbox + .drain_with(&cell, || Ok(publisher), &alarm, NOW, None) + .await + .unwrap() + .is_deferred()); + assert_eq!( + alarm.targets.lock().unwrap().len(), + 1, + "with no work and no domain alarm the slot is left alone" + ); + }); + } } diff --git a/src/microsvc/cell_host/mod.rs b/src/microsvc/cell_host/mod.rs index 26b354ae1..ed903a0a3 100644 --- a/src/microsvc/cell_host/mod.rs +++ b/src/microsvc/cell_host/mod.rs @@ -21,6 +21,7 @@ //! only; the cell host itself stays ungated (`PCH-DEC-005`). Without it, //! native cells use the in-process reference host. +pub(crate) mod alarm; pub(crate) mod causal; mod cell; #[cfg(feature = "workers-rs")] @@ -45,6 +46,16 @@ mod sql_store; mod store; mod wire; +#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] +pub use alarm::DurableCellAlarm; +pub use alarm::{ + cell_alarm_due, cell_alarm_guard, cell_alarm_session, is_cell_alarm_session, ArmedCellAlarm, + CellAlarm, CellAlarmError, CellAlarmFireReport, CellAlarmWrite, FailedCellAlarm, + FiredCellAlarm, CELL_ALARM_DUE_CLAIM, CELL_ALARM_FIRE_BATCH, CELL_ALARM_GENERATION_CLAIM, + CELL_ALARM_NAME_CLAIM, CELL_ALARM_RETRY_BASE, CELL_ALARM_RETRY_MAX, CELL_ALARM_ROLE, + CELL_ALARM_SERVICE_ID, CELL_ALARM_USER_ID, MAX_ARMED_CELL_ALARMS, MAX_CELL_ALARM_INPUT_BYTES, + MAX_CELL_ALARM_NAME_BYTES, MAX_CELL_ALARM_WRITES_PER_COMMAND, +}; pub use causal::{ CellCommandIdentity, CellDispatchError, CellDispatchResult, CELL_CAUSATION_ID_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, @@ -52,7 +63,7 @@ pub use causal::{ pub use cell::{instance_name, parent_cell_name, AggregateCell, CellNamespace}; #[cfg(feature = "workers-rs")] pub use celld_outbox::{ - CelldOutbox, CelldOutboxDrainOutcome, CELLD_OUTBOX_DEFAULT_BATCH_SIZE, + CelldAlarmOutcome, CelldOutbox, CelldOutboxDrainOutcome, CELLD_OUTBOX_DEFAULT_BATCH_SIZE, CELLD_OUTBOX_DEFAULT_BINDING, CELLD_OUTBOX_DEFAULT_DRAIN_INTERVAL_MS, CELLD_OUTBOX_DEFAULT_LEASE, CELLD_OUTBOX_DEFAULT_MAX_ATTEMPTS, CELLD_OUTBOX_DRAIN_INTERVAL_ENV, }; diff --git a/src/microsvc/cell_host/sql_store.rs b/src/microsvc/cell_host/sql_store.rs index ab2acf98d..43ba47df4 100644 --- a/src/microsvc/cell_host/sql_store.rs +++ b/src/microsvc/cell_host/sql_store.rs @@ -32,6 +32,8 @@ use crate::repository::{ }; use crate::snapshot::SnapshotRecord; +use super::alarm::{AlarmSettlement, ArmedCellAlarm}; + // Durable Object SQL limits bindings per statement, not per transaction. // Both shared insert planners split large commands within the same transaction. const MAX_SQL_BIND_PARAMS: usize = 100; @@ -156,6 +158,7 @@ impl CellSqlRepository { )); } let prepared = validate_commit_batch(&batch)?; + super::alarm::validate_alarm_writes(&batch.alarms)?; self.connection.transaction(|executor| { finish_sql(async { if let Some(completion) = &completion { @@ -201,6 +204,11 @@ impl CellSqlRepository { let SnapshotWrite::Save { identity, record } = snapshot; sql::save_snapshot(executor, identity, record).await?; } + // Alarm rows join the same transaction: a failed command (or a + // lost ledger lease below) leaves no alarm armed. + if !batch.alarms.is_empty() { + alarms::apply(executor, &batch.alarms).await?; + } // Native sealed read-model row: same transaction, so a reducer // error or lost ledger lease rolls the row back with the events. #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] @@ -229,6 +237,42 @@ impl CellSqlRepository { } } +impl CellSqlRepository { + /// Armed alarms, earliest first. + pub(super) fn armed_alarms(&self) -> Result, RepositoryError> { + self.connection + .read(|executor| finish_sql(alarms::armed(executor, None, usize::MAX))) + } + + /// Armed alarms due at or before `now_ms`, earliest first. + pub(super) fn due_alarms( + &self, + now_ms: i64, + limit: usize, + ) -> Result, RepositoryError> { + self.connection + .read(|executor| finish_sql(alarms::armed(executor, Some(now_ms), limit))) + } + + /// Due time of the earliest armed alarm. + pub(super) fn next_alarm_due_ms(&self) -> Result, RepositoryError> { + self.connection + .read(|executor| finish_sql(alarms::next_due(executor))) + } + + /// Settle the fired generation of `name`; a newer arm is left untouched. + pub(super) fn settle_alarm( + &self, + name: &str, + generation: u64, + settlement: &AlarmSettlement, + ) -> Result<(), RepositoryError> { + self.connection.transaction(|executor| { + finish_sql(alarms::settle(executor, name, generation, settlement)) + }) + } +} + impl GetStream for CellSqlRepository { async fn get_stream( &self, @@ -515,3 +559,145 @@ mod native { Ok(()) } } + +/// Cell alarm rows (`cell_alarms`, migration 11). Shared by Durable Object and +/// native SQLite cells; the reference host mirrors these semantics in memory. +mod alarms { + use super::*; + use crate::microsvc::cell_host::alarm::{ + CellAlarmError, CellAlarmWrite, MAX_ARMED_CELL_ALARMS, + }; + + const SELECT: &str = + "SELECT name, command, input, due_at_ms, generation, attempts FROM cell_alarms WHERE armed = 1"; + + pub(super) async fn apply( + executor: &mut impl SqlExecutor, + writes: &[CellAlarmWrite], + ) -> Result<(), RepositoryError> { + for write in writes { + match write { + CellAlarmWrite::Arm(alarm) => { + let input = alarm.input_json()?; + executor + .execute( + Statement::new("INSERT INTO cell_alarms (name, command, input, due_at_ms, generation, attempts, armed) VALUES (") + .bind(alarm.name().into()) + .sql(", ") + .bind(alarm.command().into()) + .sql(", ") + .bind(input.as_str().into()) + .sql(", ") + .bind(alarm.due_at_ms()?.into()) + .sql(", 1, 0, 1) ON CONFLICT (name) DO UPDATE SET command = excluded.command, input = excluded.input, due_at_ms = excluded.due_at_ms, generation = cell_alarms.generation + 1, attempts = 0, armed = 1"), + ) + .await?; + } + CellAlarmWrite::Cancel(name) => { + executor + .execute( + Statement::new( + "UPDATE cell_alarms SET armed = 0, attempts = 0 WHERE name = ", + ) + .bind(name.as_str().into()), + ) + .await?; + } + } + } + let rows = executor + .query(Statement::new( + "SELECT COUNT(*) AS armed FROM cell_alarms WHERE armed = 1", + )) + .await?; + let armed = rows + .first() + .map(|row| row.integer("armed")) + .transpose()? + .unwrap_or(0); + if armed > MAX_ARMED_CELL_ALARMS as i64 { + return Err(CellAlarmError::TooManyArmed.into()); + } + Ok(()) + } + + pub(super) async fn armed( + executor: &mut impl SqlExecutor, + due_by_ms: Option, + limit: usize, + ) -> Result, RepositoryError> { + let limit = (limit as u64).min(9_007_199_254_740_991) as i64; + let mut statement = Statement::new(SELECT); + if let Some(due_by_ms) = due_by_ms { + statement = statement.sql(" AND due_at_ms <= ").bind(due_by_ms.into()); + } + let rows = executor + .query( + statement + .sql(" ORDER BY due_at_ms, name LIMIT ") + .bind(limit.into()), + ) + .await?; + rows.iter() + .map(|row| { + let input = row.text("input")?; + Ok(ArmedCellAlarm { + name: row.text("name")?, + command: row.text("command")?, + input: serde_json::from_str(&input).map_err(|error| { + RepositoryError::Model(format!("cell alarm input is invalid JSON: {error}")) + })?, + due_at_ms: row.integer("due_at_ms")?, + generation: u64::try_from(row.integer("generation")?).map_err(|_| { + RepositoryError::Model("cell alarm generation is negative".into()) + })?, + attempts: u32::try_from(row.integer("attempts")?).unwrap_or(u32::MAX), + }) + }) + .collect() + } + + pub(super) async fn next_due( + executor: &mut impl SqlExecutor, + ) -> Result, RepositoryError> { + let rows = executor + .query(Statement::new( + "SELECT MIN(due_at_ms) AS due FROM cell_alarms WHERE armed = 1", + )) + .await?; + Ok(rows + .first() + .map(|row| row.optional_integer("due")) + .transpose()? + .flatten()) + } + + pub(super) async fn settle( + executor: &mut impl SqlExecutor, + name: &str, + generation: u64, + settlement: &AlarmSettlement, + ) -> Result<(), RepositoryError> { + let generation = i64::try_from(generation) + .map_err(|_| RepositoryError::Model("cell alarm generation overflow".into()))?; + let statement = match settlement { + AlarmSettlement::Done => { + Statement::new("UPDATE cell_alarms SET armed = 0, attempts = 0 WHERE name = ") + } + AlarmSettlement::Retry { due_at_ms } => { + Statement::new("UPDATE cell_alarms SET attempts = attempts + 1, due_at_ms = ") + .bind((*due_at_ms).into()) + .sql(" WHERE name = ") + } + }; + executor + .execute( + statement + .bind(name.into()) + .sql(" AND armed = 1 AND generation = ") + .bind(generation.into()), + ) + .await?; + Ok(()) + } +} diff --git a/src/microsvc/cell_host/store.rs b/src/microsvc/cell_host/store.rs index 13bd07805..62d7bd9f2 100644 --- a/src/microsvc/cell_host/store.rs +++ b/src/microsvc/cell_host/store.rs @@ -115,6 +115,9 @@ pub struct DurableAggregateCellState { pub commands: Vec, pub outbox: Vec, pub sealed_row: Option, + /// Alarm rows, including disarmed ones so generations never repeat. + #[serde(default)] + pub alarms: Vec, } /// Private command-side repository for one cell instance (`{aggregate_type}:{shard}`). @@ -135,10 +138,15 @@ pub struct DurableAggregateCellState { pub struct CellStreamStore { ownership: CellOwnership, inner: CellRepository, + /// Resolves relative alarm due times at commit and drives alarm firing. + clock: Arc, #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] sealed_row: Arc>>, #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] sealed_row_reducer: Option, + /// Reference-host alarm table (SQL cells keep alarms in their database). + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + alarms: Arc>, } impl CellStreamStore { @@ -151,6 +159,7 @@ impl CellStreamStore { Ok(Self { ownership: CellOwnership::Exclusive(identity), inner: CellSqlRepository::from_state(state)?, + clock: Arc::new(crate::SystemClock), }) } @@ -169,6 +178,7 @@ impl CellStreamStore { owns: Arc::new(owns), }, inner: CellSqlRepository::from_state(state)?, + clock: Arc::new(crate::SystemClock), }) } @@ -202,8 +212,10 @@ impl CellStreamStore { Self { ownership, inner, + clock: Arc::new(crate::SystemClock), sealed_row: Arc::new(Mutex::new(None)), sealed_row_reducer: None, + alarms: Arc::new(Mutex::new(Default::default())), } } @@ -314,6 +326,22 @@ impl CellStreamStore { )?)) } + /// Replace the cell clock (default: the system clock). The reference host + /// takes a [`ManualClock`](crate::ManualClock) so alarm tests are + /// deterministic; it resolves [`CellAlarm::after`](super::CellAlarm::after) + /// at commit and is what [`AggregateCell::fire_due_alarms`] reads. + /// + /// [`AggregateCell::fire_due_alarms`]: super::AggregateCell::fire_due_alarms + pub fn with_clock(mut self, clock: impl crate::Clock + 'static) -> Self { + self.clock = Arc::new(clock); + self + } + + /// The cell clock's current time. + pub fn now(&self) -> std::time::SystemTime { + self.clock.now() + } + /// Cell instance name (`type:id`). pub fn instance_name(&self) -> String { match &self.ownership { @@ -508,6 +536,7 @@ impl CellStreamStore { commands: self.durable_commands()?, outbox: self.durable_outbox()?, sealed_row: self.sealed_row()?, + alarms: self.lock_alarms()?.export(), }) } @@ -535,6 +564,130 @@ impl CellStreamStore { .lock() .map_err(|_| RepositoryError::Model("cell sealed row lock poisoned".into()))?; *sealed = state.sealed_row; + *self.lock_alarms()? = super::alarm::MemoryCellAlarms::restore(state.alarms); + Ok(()) + } + + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + fn lock_alarms( + &self, + ) -> Result, RepositoryError> { + self.alarms + .lock() + .map_err(|_| RepositoryError::Model("cell alarm table lock poisoned".into())) + } + + /// The SQL database holding alarms (Durable Object or native SQLite); + /// `None` on the reference host, which keeps them in memory. + #[cfg(any( + all(feature = "workers-rs", target_arch = "wasm32"), + all(feature = "cell-sqlite", not(target_arch = "wasm32")) + ))] + fn alarm_sql(&self) -> Option<&super::sql_store::CellSqlRepository> { + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + return Some(&self.inner); + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + self.inner.sql() + } + + #[cfg(not(any( + all(feature = "workers-rs", target_arch = "wasm32"), + all(feature = "cell-sqlite", not(target_arch = "wasm32")) + )))] + fn alarm_sql(&self) -> Option<&std::convert::Infallible> { + None + } + + /// Armed alarms, earliest first. + pub fn armed_alarms(&self) -> Result, RepositoryError> { + #[cfg(any( + all(feature = "workers-rs", target_arch = "wasm32"), + all(feature = "cell-sqlite", not(target_arch = "wasm32")) + ))] + if let Some(sql) = self.alarm_sql() { + return sql.armed_alarms(); + } + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + return Ok(self.lock_alarms()?.armed()); + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + Ok(Vec::new()) + } + + /// Armed alarms due at or before `now_ms`, earliest first, at most `limit`. + pub(super) fn due_alarms( + &self, + now_ms: i64, + limit: usize, + ) -> Result, RepositoryError> { + #[cfg(any( + all(feature = "workers-rs", target_arch = "wasm32"), + all(feature = "cell-sqlite", not(target_arch = "wasm32")) + ))] + if let Some(sql) = self.alarm_sql() { + return sql.due_alarms(now_ms, limit); + } + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + return Ok(self.lock_alarms()?.due(now_ms, limit)); + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + Ok(Vec::new()) + } + + /// Due time (epoch ms) of the earliest armed alarm. + pub(super) fn next_alarm_due_ms(&self) -> Result, RepositoryError> { + #[cfg(any( + all(feature = "workers-rs", target_arch = "wasm32"), + all(feature = "cell-sqlite", not(target_arch = "wasm32")) + ))] + if let Some(sql) = self.alarm_sql() { + return sql.next_alarm_due_ms(); + } + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + return Ok(self.lock_alarms()?.next_due_ms()); + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + Ok(None) + } + + pub(super) fn settle_alarm( + &self, + name: &str, + generation: u64, + settlement: &super::alarm::AlarmSettlement, + ) -> Result<(), RepositoryError> { + #[cfg(any( + all(feature = "workers-rs", target_arch = "wasm32"), + all(feature = "cell-sqlite", not(target_arch = "wasm32")) + ))] + if let Some(sql) = self.alarm_sql() { + return sql.settle_alarm(name, generation, settlement); + } + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + self.lock_alarms()?.settle(name, generation, settlement); + Ok(()) + } + + /// Reference host: validate and plan alarm writes before the commit so a + /// rejected plan aborts it; install the plan only after the commit. + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + fn plan_memory_alarms( + &self, + batch: &mut CommitBatch<'_>, + ) -> Result, RepositoryError> { + if self.alarm_sql().is_some() || batch.alarms.is_empty() { + return Ok(None); + } + let writes = std::mem::take(&mut batch.alarms); + super::alarm::validate_alarm_writes(&writes)?; + self.lock_alarms()?.plan(&writes).map(Some) + } + + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + fn install_memory_alarms( + &self, + planned: Option, + ) -> Result<(), RepositoryError> { + if let Some(planned) = planned { + *self.lock_alarms()? = planned; + } Ok(()) } @@ -693,15 +846,21 @@ impl TransactionalCommit for CellStreamStore { &'a self, batch: CommitBatch<'a>, ) -> impl Future> + Send + 'a { + let mut batch = batch; async move { self.ensure_batch(&batch)?; + super::alarm::resolve_alarm_writes(&mut batch.alarms, self.clock.now())?; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] let sealed = self.reduce_memory_sealed_row(&batch)?; + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + let alarms = self.plan_memory_alarms(&mut batch)?; TransactionalCommit::commit_batch(&self.inner, batch).await?; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] if let Some(row) = sealed { self.replace_sealed_row(row)?; } + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + self.install_memory_alarms(alarms)?; Ok(()) } } @@ -712,19 +871,29 @@ impl CausalTransactionalCommit for CellStreamStore { &'a self, batch: CausalCommitBatch<'a>, ) -> impl Future> + Send + 'a { + let mut batch = batch; async move { self.ensure_batch(&batch.domain) .map_err(CommandLedgerError::Storage)?; + super::alarm::resolve_alarm_writes(&mut batch.domain.alarms, self.clock.now()) + .map_err(CommandLedgerError::Storage)?; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] let sealed = self .reduce_memory_sealed_row(&batch.domain) .map_err(CommandLedgerError::Storage)?; + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + let alarms = self + .plan_memory_alarms(&mut batch.domain) + .map_err(CommandLedgerError::Storage)?; CausalTransactionalCommit::commit_causal_batch(&self.inner, batch).await?; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] if let Some(row) = sealed { self.replace_sealed_row(row) .map_err(CommandLedgerError::Storage)?; } + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] + self.install_memory_alarms(alarms) + .map_err(CommandLedgerError::Storage)?; Ok(()) } } diff --git a/src/microsvc/cell_host/tests.rs b/src/microsvc/cell_host/tests.rs index fe773cca8..2c901143c 100644 --- a/src/microsvc/cell_host/tests.rs +++ b/src/microsvc/cell_host/tests.rs @@ -669,5 +669,6 @@ async fn reference_host_applies_the_sealed_row_reducer_on_commit() { .is_err()); } +mod alarms; #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] mod native_sqlite; diff --git a/src/microsvc/cell_host/tests/alarms.rs b/src/microsvc/cell_host/tests/alarms.rs new file mode 100644 index 000000000..4d161c475 --- /dev/null +++ b/src/microsvc/cell_host/tests/alarms.rs @@ -0,0 +1,519 @@ +//! Re-armable cell alarms on the in-process reference host, driven by a +//! `ManualClock`. The same command/ledger/settlement path runs on celld; only +//! the wake source differs (Durable Object alarm vs. the test calling +//! `fire_due_alarms`). + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, SystemTime}; + +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use super::super::{ + cell_alarm_guard, cell_alarm_session, AggregateCell, CellAlarm, CellAlarmWrite, + CellCommandIdentity, CellStreamStore, CELL_ALARM_RETRY_BASE, +}; +use super::owner_session; +use crate::command::{typed_command, PreparedCommand, Succeeded}; +use crate::entity::Entity; +use crate::microsvc::service::{CausalCommandContext, PortableCommand, Routes}; +use crate::microsvc::HandlerError; +use crate::repository::{CommitBatch, StreamIdentity, TransactionalCommit}; +use crate::{sourced, Clock, InMemoryRepository, ManualClock}; + +#[derive(Clone, Default, Serialize, Deserialize, crate::Snapshot)] +struct Ticker { + entity: Entity, + ticks: u32, + limit: u32, + period_ms: u64, +} + +#[sourced(entity, aggregate_type = "Ticker", events = "TickerEvent")] +impl Ticker { + #[event("ticker.started", version = 1)] + fn start(&mut self, id: String, limit: u32, period_ms: u64) { + self.entity.set_id(id); + self.limit = limit; + self.period_ms = period_ms; + } + + #[event("ticker.ticked", version = 1)] + fn tick(&mut self) { + self.ticks += 1; + } +} + +#[derive(Debug, Deserialize, crate::CommandInput)] +struct StartInput { + id: String, + limit: u32, + period_ms: u64, + fail_after_arming: bool, + tick_command: String, +} + +#[derive(Debug, Serialize, Deserialize, crate::CommandInput)] +struct TickInput { + id: String, +} + +#[derive(Debug, Serialize, crate::CommandOutput)] +struct TickerPayload { + id: String, + ticks: u32, +} + +const TICK: &str = "ticker.tick"; + +struct Start; +impl PortableCommand for Start +where + D: crate::microsvc::CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "ticker.start", + )) + .guarded( + |ctx: &CausalCommandContext<'_, Ticker>| ctx.session().user_id().is_some(), + handle_start, + ) + } +} + +async fn handle_start( + ctx: &CausalCommandContext<'_, Ticker>, + input: StartInput, +) -> Result>, HandlerError> { + let repo = ctx.repo(); + let mut ticker = repo.create(); + ticker + .start(input.id.clone(), input.limit, input.period_ms) + .map_err(|error| HandlerError::Rejected(error.to_string()))?; + ctx.arm_alarm( + CellAlarm::after( + "tick", + Duration::from_millis(input.period_ms), + &input.tick_command, + &TickInput { + id: input.id.clone(), + }, + ) + .map_err(|error| HandlerError::Rejected(error.to_string()))?, + )?; + if input.fail_after_arming { + return Err(HandlerError::Rejected("start refused after arming".into())); + } + repo.commit(ticker)?.succeeded(TickerPayload { + id: input.id, + ticks: 0, + }) +} + +struct Tick; +impl PortableCommand for Tick +where + D: crate::microsvc::CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(TICK)) + .guarded(cell_alarm_guard::, handle_tick) + } +} + +async fn handle_tick( + ctx: &CausalCommandContext<'_, Ticker>, + input: TickInput, +) -> Result>, HandlerError> { + let repo = ctx.repo(); + let mut ticker = repo + .get(&input.id) + .await? + .ok_or_else(|| HandlerError::Rejected("no ticker".into()))?; + ticker + .tick() + .map_err(|error| HandlerError::Rejected(error.to_string()))?; + if ticker.ticks < ticker.limit { + // Re-arm from the fired due time, not wall time: no drift. + let due = ctx.alarm_due().expect("alarm session") + Duration::from_millis(ticker.period_ms); + ctx.arm_alarm( + CellAlarm::at("tick", due, TICK, &input) + .map_err(|error| HandlerError::Rejected(error.to_string()))?, + )?; + } + let ticks = ticker.ticks; + repo.commit(ticker)?.succeeded(TickerPayload { + id: input.id, + ticks, + }) +} + +struct Stop; +impl PortableCommand for Stop +where + D: crate::microsvc::CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "ticker.stop", + )) + .guarded( + |ctx: &CausalCommandContext<'_, Ticker>| ctx.session().user_id().is_some(), + handle_stop, + ) + } +} + +async fn handle_stop( + ctx: &CausalCommandContext<'_, Ticker>, + input: TickInput, +) -> Result>, HandlerError> { + let repo = ctx.repo(); + let ticker = repo + .get(&input.id) + .await? + .ok_or_else(|| HandlerError::Rejected("no ticker".into()))?; + ctx.cancel_alarm("tick")?; + let ticks = ticker.ticks; + repo.commit(ticker)?.succeeded(TickerPayload { + id: input.id, + ticks, + }) +} + +static FLAKY_FAILURES: AtomicUsize = AtomicUsize::new(0); + +fn take_flaky_failure() -> bool { + let mut current = FLAKY_FAILURES.load(Ordering::SeqCst); + while current > 0 { + match FLAKY_FAILURES.compare_exchange( + current, + current - 1, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => return true, + Err(actual) => current = actual, + } + } + false +} + +/// Fails with an internal (retryable) error while `FLAKY_FAILURES` > 0. +struct Flaky; +impl PortableCommand for Flaky +where + D: crate::microsvc::CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "ticker.flaky", + )) + .guarded(cell_alarm_guard::, handle_flaky) + } +} + +async fn handle_flaky( + ctx: &CausalCommandContext<'_, Ticker>, + input: TickInput, +) -> Result>, HandlerError> { + if take_flaky_failure() { + return Err(HandlerError::Other("transient dependency outage".into())); + } + handle_tick(ctx, input).await +} + +const START_MS: u64 = 1_700_000_000_000; + +fn clock() -> ManualClock { + ManualClock::new(SystemTime::UNIX_EPOCH + Duration::from_millis(START_MS)) +} + +fn ticker_cell(clock: &ManualClock) -> AggregateCell { + let store = CellStreamStore::for_identity(StreamIdentity::new("Ticker", "t1").unwrap()) + .with_clock(clock.clone()); + mount(AggregateCell::from_store(store).unwrap()) +} + +fn mount(cell: AggregateCell) -> AggregateCell { + cell.mount(Start).mount(Tick).mount(Stop).mount(Flaky) +} + +fn identity(command_id: &str) -> CellCommandIdentity { + CellCommandIdentity::new("ticker-service", "user-1", command_id).unwrap() +} + +async fn start(cell: &AggregateCell, input: serde_json::Value) { + cell.dispatch_idempotent( + "ticker.start", + &identity("0190a000-0000-7000-8000-000000000001"), + input, + owner_session(), + ) + .await + .unwrap(); +} + +async fn ticks(cell: &AggregateCell) -> u32 { + cell.load().await.unwrap().map_or(0, |ticker| ticker.ticks) +} + +#[tokio::test] +async fn alarm_commits_with_the_events_and_is_absent_when_the_command_fails() { + let clock = clock(); + let cell = ticker_cell(&clock); + let refused = cell + .dispatch_idempotent( + "ticker.start", + &identity("0190a000-0000-7000-8000-000000000002"), + json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": true, "tick_command": "ticker.tick" }), + owner_session(), + ) + .await; + assert!(refused.is_err()); + assert!(cell.armed_alarms().unwrap().is_empty()); + assert!(cell.load().await.unwrap().is_none()); + + start(&cell, json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; + let armed = cell.armed_alarms().unwrap(); + assert_eq!(armed.len(), 1); + assert_eq!(armed[0].name, "tick"); + assert_eq!(armed[0].command, TICK); + assert_eq!(armed[0].generation, 1); + assert_eq!( + cell.next_alarm_due().unwrap(), + Some(clock.now() + Duration::from_millis(1000)), + "a relative alarm resolves against the cell clock at commit" + ); +} + +#[tokio::test] +async fn alarm_fires_once_per_arm_with_the_trusted_session() { + let clock = clock(); + let cell = ticker_cell(&clock); + start(&cell, json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; + + clock.advance(Duration::from_millis(999)); + assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + clock.advance(Duration::from_millis(1)); + let report = cell.fire_due_alarms().await.unwrap(); + assert_eq!(report.fired.len(), 1); + assert!(!report.fired[0].replayed); + assert_eq!(report.fired[0].payload["ticks"], 1); + clock.advance(Duration::from_secs(60)); + assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + assert_eq!(ticks(&cell).await, 1); + assert!(cell.armed_alarms().unwrap().is_empty()); + + // The alarm-only command refuses ordinary callers. + let forged = cell + .dispatch_idempotent( + TICK, + &identity("0190a000-0000-7000-8000-000000000003"), + json!({ "id": "t1" }), + owner_session(), + ) + .await; + assert!(forged.is_err()); + assert_eq!(ticks(&cell).await, 1); +} + +#[tokio::test] +async fn rearming_in_the_handler_ticks_one_hundred_times_without_drift() { + let clock = clock(); + let cell = ticker_cell(&clock); + start(&cell, json!({ "id": "t1", "limit": 100, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; + let first_due = cell.next_alarm_due().unwrap().unwrap(); + for tick in 1..=100u64 { + let expected_due = first_due + Duration::from_millis((tick - 1) * 1000); + assert_eq!(cell.next_alarm_due().unwrap(), Some(expected_due)); + // Wake late; the next tick still re-arms from the due time. + clock.set(expected_due + Duration::from_millis(250)); + let report = cell.fire_due_alarms().await.unwrap(); + assert_eq!(report.fired.len(), 1, "tick {tick}"); + assert_eq!(report.fired[0].generation, tick); + assert!(report.failed.is_empty() && report.retried.is_empty()); + } + assert_eq!(ticks(&cell).await, 100); + assert!(cell.armed_alarms().unwrap().is_empty()); + clock.advance(Duration::from_secs(3600)); + assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + assert_eq!(ticks(&cell).await, 100); +} + +#[tokio::test] +async fn a_fired_command_that_committed_before_settlement_replays_instead_of_rerunning() { + let clock = clock(); + let cell = ticker_cell(&clock); + start(&cell, json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; + clock.advance(Duration::from_secs(1)); + + // Crash window: the alarm command committed but its row was not settled. + let alarm = cell.armed_alarms().unwrap().remove(0); + cell.dispatch_idempotent( + &alarm.command, + &alarm.command_identity().unwrap(), + alarm.input.clone(), + cell_alarm_session(&alarm), + ) + .await + .unwrap(); + assert_eq!(ticks(&cell).await, 1); + + let report = cell.fire_due_alarms().await.unwrap(); + assert_eq!(report.fired.len(), 1); + assert!(report.fired[0].replayed, "the ledger replays the receipt"); + assert_eq!(ticks(&cell).await, 1, "the handler did not run twice"); + assert!(cell.armed_alarms().unwrap().is_empty()); +} + +#[tokio::test] +async fn cancel_disarms_and_rejections_are_terminal() { + let clock = clock(); + let cell = ticker_cell(&clock); + start(&cell, json!({ "id": "t1", "limit": 5, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; + cell.dispatch_idempotent( + "ticker.stop", + &identity("0190a000-0000-7000-8000-000000000004"), + json!({ "id": "t1" }), + owner_session(), + ) + .await + .unwrap(); + assert!(cell.armed_alarms().unwrap().is_empty()); + clock.advance(Duration::from_secs(10)); + assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + + let other = ticker_cell(&clock); + start( + &other, + json!({ "id": "t1", "limit": 5, "period_ms": 10, "tick_command": "ticker.missing", "fail_after_arming": false }), + ) + .await; + clock.advance(Duration::from_secs(1)); + let report = other.fire_due_alarms().await.unwrap(); + assert!(report.fired.is_empty()); + assert_eq!(report.failed.len(), 1); + assert_eq!(report.failed[0].code, "BAD_REQUEST"); + assert!(other.armed_alarms().unwrap().is_empty()); +} + +#[tokio::test] +async fn retryable_failures_back_off_and_then_fire() { + FLAKY_FAILURES.store(2, Ordering::SeqCst); + let clock = clock(); + let cell = ticker_cell(&clock); + start( + &cell, + json!({ "id": "t1", "limit": 1, "period_ms": 1000, "tick_command": "ticker.flaky", "fail_after_arming": false }), + ) + .await; + clock.advance(Duration::from_secs(1)); + let first = cell.fire_due_alarms().await.unwrap(); + assert_eq!(first.retried.len(), 1); + assert_eq!(first.retried[0].code, "INTERNAL"); + let armed = cell.armed_alarms().unwrap(); + assert_eq!(armed[0].attempts, 1); + assert_eq!(armed[0].due(), clock.now() + CELL_ALARM_RETRY_BASE); + + clock.advance(CELL_ALARM_RETRY_BASE); + let second = cell.fire_due_alarms().await.unwrap(); + assert_eq!(second.retried.len(), 1); + assert_eq!( + cell.armed_alarms().unwrap()[0].due(), + clock.now() + CELL_ALARM_RETRY_BASE * 2, + "backoff doubles" + ); + clock.advance(CELL_ALARM_RETRY_BASE * 2); + let third = cell.fire_due_alarms().await.unwrap(); + assert_eq!(third.fired.len(), 1); + assert_eq!(ticks(&cell).await, 1); +} + +#[tokio::test] +async fn armed_alarms_survive_a_reference_host_restart() { + let clock = clock(); + let cell = ticker_cell(&clock); + start(&cell, json!({ "id": "t1", "limit": 2, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; + let exported = cell.durable_state().unwrap(); + drop(cell); + + let restarted = ticker_cell(&clock); + restarted.restore_durable_state(exported).unwrap(); + clock.advance(Duration::from_secs(1)); + assert_eq!(restarted.fire_due_alarms().await.unwrap().fired.len(), 1); + clock.advance(Duration::from_secs(1)); + assert_eq!(restarted.fire_due_alarms().await.unwrap().fired.len(), 1); + assert_eq!(ticks(&restarted).await, 2); +} + +#[tokio::test] +async fn repositories_without_an_alarm_slot_reject_alarm_writes() { + let repo = InMemoryRepository::new(); + let mut batch = CommitBatch::empty(); + batch.alarms.push(CellAlarmWrite::Cancel("tick".into())); + let error = repo.commit_batch(batch).await.unwrap_err(); + assert!(error.to_string().contains("aggregate cell"), "{error}"); +} + +/// The SQL alarm table shared with Durable Object cells, on a native SQLite +/// file: same transaction as the events, and armed rows survive a reopen. +#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] +#[tokio::test] +async fn sqlite_cell_alarms_commit_atomically_and_survive_reopening() { + use super::super::CellSqlitePath; + + let root = + std::env::temp_dir().join(format!("distributed-cell-alarms-{}", uuid::Uuid::now_v7())); + let location = CellSqlitePath::root(&root); + let clock = clock(); + let open = |clock: &ManualClock| { + let store = + CellStreamStore::open_sqlite(&location, StreamIdentity::new("Ticker", "t1").unwrap()) + .unwrap() + .with_clock(clock.clone()); + mount(AggregateCell::from_store(store).unwrap()) + }; + + let cell = open(&clock); + let refused = cell + .dispatch_idempotent( + "ticker.start", + &identity("0190a000-0000-7000-8000-000000000002"), + json!({ "id": "t1", "limit": 3, "period_ms": 1000, "fail_after_arming": true, "tick_command": "ticker.tick" }), + owner_session(), + ) + .await; + assert!(refused.is_err()); + assert!(cell.armed_alarms().unwrap().is_empty()); + start(&cell, json!({ "id": "t1", "limit": 3, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; + drop(cell); + + let cell = open(&clock); + assert_eq!(cell.armed_alarms().unwrap().len(), 1); + clock.advance(Duration::from_secs(1)); + assert_eq!(cell.fire_due_alarms().await.unwrap().fired.len(), 1); + drop(cell); + + // Restart between ticks: the re-armed generation is still there, once. + let cell = open(&clock); + let armed = cell.armed_alarms().unwrap(); + assert_eq!(armed.len(), 1); + assert_eq!(armed[0].generation, 2); + assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + for _ in 0..2 { + clock.advance(Duration::from_secs(1)); + assert_eq!(cell.fire_due_alarms().await.unwrap().fired.len(), 1); + } + assert_eq!(ticks(&cell).await, 3); + assert!(cell.armed_alarms().unwrap().is_empty()); + clock.advance(Duration::from_secs(60)); + assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + drop(cell); + let _ = std::fs::remove_dir_all(root); +} diff --git a/src/microsvc/service/handlers.rs b/src/microsvc/service/handlers.rs index c7d5a891f..acbbf70a0 100644 --- a/src/microsvc/service/handlers.rs +++ b/src/microsvc/service/handlers.rs @@ -686,6 +686,32 @@ where CausalCommitBuilder::empty(self).outbox(message) } + /// Arm (or re-arm) a cell alarm with this command's effects. + /// + /// The alarm row commits in the same transaction as the command's events + /// and receipt; if the command fails, nothing is armed. At the alarm's due + /// time the hosting cell dispatches its command into itself with the + /// trusted session from [`crate::cell_host::cell_alarm_session`]. Only + /// aggregate cells accept alarms: other hosts reject the commit. + pub fn arm_alarm(&self, alarm: crate::cell_host::CellAlarm) -> Result<(), HandlerError> { + self.workspace + .stage_alarm(crate::cell_host::CellAlarmWrite::Arm(alarm)) + .map_err(workspace_handler_error) + } + + /// Disarm a cell alarm with this command's effects (no-op when unarmed). + pub fn cancel_alarm(&self, name: impl Into) -> Result<(), HandlerError> { + self.workspace + .stage_alarm(crate::cell_host::CellAlarmWrite::Cancel(name.into())) + .map_err(workspace_handler_error) + } + + /// Due time of the alarm that fired this command; `None` for any command + /// not dispatched by a cell alarm. + pub fn alarm_due(&self) -> Option { + crate::cell_host::cell_alarm_due(self.session) + } + /// Low-level outbox staging retained for framework internals. #[cfg(all(test, feature = "graphql", feature = "sqlite"))] pub(crate) fn stage_outbox(&self, message: OutboxMessage) -> Result<(), HandlerError> { diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 5b7490a99..fadc13fff 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -276,6 +276,7 @@ where read_model_plans: self.read_model_plans, snapshots, inbox_receipts: Vec::new(), + alarms: Vec::new(), }) .await?; if let Some(version) = snapshot_version { diff --git a/src/repository/migrations.rs b/src/repository/migrations.rs index d08d3f261..188fc912a 100644 --- a/src/repository/migrations.rs +++ b/src/repository/migrations.rs @@ -21,5 +21,5 @@ include!(concat!(env!("OUT_DIR"), "/migration_inventory.rs")); pub(crate) fn cell_migrations() -> impl Iterator { SQLITE_MIGRATIONS .iter() - .filter(|migration| matches!(migration.version, 1 | 2 | 4 | 8)) + .filter(|migration| matches!(migration.version, 1 | 2 | 4 | 8 | 11)) } diff --git a/src/repository/traits.rs b/src/repository/traits.rs index 93d310440..bebecad3f 100644 --- a/src/repository/traits.rs +++ b/src/repository/traits.rs @@ -39,6 +39,9 @@ pub struct CommitBatch<'a> { /// Consumer inbox receipts to record in the same transaction (the optional /// effectively-once effect fence). Empty for the default idempotent path. pub inbox_receipts: Vec, + /// Re-armable cell alarm writes. Only aggregate cells accept them; every + /// other repository rejects a batch that carries any. + pub alarms: Vec, } impl<'a> CommitBatch<'a> { @@ -49,6 +52,7 @@ impl<'a> CommitBatch<'a> { read_model_plans: Vec::new(), snapshots: Vec::new(), inbox_receipts: Vec::new(), + alarms: Vec::new(), } } diff --git a/src/sqlx_repo/repo/backend.rs b/src/sqlx_repo/repo/backend.rs index 1882c062a..2ac2c2e8d 100644 --- a/src/sqlx_repo/repo/backend.rs +++ b/src/sqlx_repo/repo/backend.rs @@ -47,7 +47,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); assert_eq!( descriptions, vec![ @@ -60,7 +60,8 @@ mod tests { "projection program identity", "external command binding", "projection delivery aliases", - "coordination leases" + "coordination leases", + "cell alarms" ] ); assert_eq!( @@ -106,6 +107,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/sqlite/0010_coordination_leases.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/sqlite/0011_cell_alarms.sql" + )), ] ); } @@ -125,7 +130,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); assert_eq!( descriptions, vec![ @@ -138,7 +143,8 @@ mod tests { "projection program identity", "external command binding", "projection delivery aliases", - "coordination leases" + "coordination leases", + "cell alarms" ] ); assert_eq!( @@ -184,6 +190,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/postgres/0010_coordination_leases.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/postgres/0011_cell_alarms.sql" + )), ] ); } diff --git a/src/sqlx_repo/repo/commit.rs b/src/sqlx_repo/repo/commit.rs index 4e7fd6c97..84dc20a62 100644 --- a/src/sqlx_repo/repo/commit.rs +++ b/src/sqlx_repo/repo/commit.rs @@ -43,6 +43,7 @@ where for<'q> Option<&'q str>: Encode<'q, DB> + Type, for<'r> &'r str: sqlx::ColumnIndex, { + crate::microsvc::cell_host::alarm::reject_alarm_writes(&batch.alarms)?; let prepared = validate_commit_batch(&batch)?; for plan in &batch.read_model_plans { validate_sql_write_plan(plan).map_err(RepositoryError::from)?; diff --git a/tests/celld/README.md b/tests/celld/README.md index b9aceb3af..5d248e439 100644 --- a/tests/celld/README.md +++ b/tests/celld/README.md @@ -34,7 +34,18 @@ match cell.drain_outbox(&env).await { Err(error) => worker::console_error!("outbox drain deferred: {error}"), } -// The Durable Object alarm handler also calls drain_outbox(&env). +// The Durable Object alarm handler calls cell.alarm(&env): it fires due cell +// alarms (see docs/coordination-primitives.md) and then drains the outbox. +``` + +Domain cell alarms share the one Durable Object alarm with the outbox +watchdog; every wake is set to the earliest of the two. `alarm-worker/` is a +Ticker cell that arms, re-arms and cancels alarms, and `alarms.mjs` proves it +on an isolated `celld dev` (including a crash and restart): + +```sh +(cd tests/celld/alarm-worker && worker-build --release) +CELLD=celld node tests/celld/alarms.mjs ``` The Queue lives in another cell: its acceptance is **not** part of the diff --git a/tests/celld/alarm-worker/Cargo.toml b/tests/celld/alarm-worker/Cargo.toml new file mode 100644 index 000000000..ac1f35cfb --- /dev/null +++ b/tests/celld/alarm-worker/Cargo.toml @@ -0,0 +1,23 @@ +[workspace] +members = ["."] +resolver = "2" + +[package] +name = "ticker-cell-worker" +version = "0.1.0" +edition = "2021" +publish = false +description = "workers-rs Ticker cell: re-armable cell alarms sharing the Durable Object alarm with the outbox watchdog" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +console_error_panic_hook = "0.1" +distributed = { path = "../../..", default-features = false, features = ["workers-rs"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +worker = { version = "0.8", features = ["queue"] } + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/tests/celld/alarm-worker/src/lib.rs b/tests/celld/alarm-worker/src/lib.rs new file mode 100644 index 000000000..b9b4efce3 --- /dev/null +++ b/tests/celld/alarm-worker/src/lib.rs @@ -0,0 +1,458 @@ +//! Ticker Durable Object backed by `AggregateCell`: re-armable cell alarms on +//! celld, sharing the one Durable Object alarm with the outbox watchdog. +//! +//! Routes (all require `DISTRIBUTED_INTERNAL_SECRET`): +//! - `POST /ticker/{id}/ticker.start` `{ commandId, input }` arms `tick`. +//! - `POST /ticker/{id}/ticker.touch` publishes one outbox event. With +//! `x-distributed-test-defer-drain: 1` it skips the immediate drain, so only +//! the alarm wake (watchdog) can deliver the row. +//! - `POST /ticker/{id}/ticker.tick` is the alarm-only command; ordinary +//! callers are refused. +//! - `GET /ticker/{id}` returns ticks, armed alarms and the outbox backlog. + +use std::time::Duration; + +use distributed::cell_host::{ + cell_alarm_guard, AggregateCell, CellAlarm, CellCommandIdentity, CellDispatchError, + CellWaitPathRequest, CelldOutbox, InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, + CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, +}; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, Session, + ROLE_KEY, USER_ID_KEY, +}; +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use worker::*; + +const TICK: &str = "ticker.tick"; + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct Ticker { + entity: Entity, + ticks: u32, + limit: u32, + period_ms: u64, + touches: u32, +} + +#[sourced(entity, aggregate_type = "Ticker", events = "TickerEvent")] +impl Ticker { + #[event("ticker.started", version = 1)] + fn start(&mut self, id: String, limit: u32, period_ms: u64) { + self.entity.set_id(id); + self.limit = limit; + self.period_ms = period_ms; + } + + #[event("ticker.ticked", version = 1, domain = event)] + fn tick(&mut self) { + self.ticks += 1; + } + + #[event("ticker.touched", version = 1, domain = event)] + fn touch(&mut self) { + self.touches += 1; + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct StartInput { + pub id: String, + pub limit: u32, + pub period_ms: u64, + pub fail_after_arming: bool, +} + +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct TickerInput { + pub id: String, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct TickerPayload { + pub id: String, + pub ticks: u32, +} + +fn rejected(error: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(error.to_string()) +} + +fn is_user(ctx: &CausalCommandContext<'_, Ticker>) -> bool { + ctx.session() + .user_id() + .is_some_and(|user| !user.starts_with("system:")) +} + +struct Start; +impl PortableCommand for Start +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "ticker.start", + )) + .guarded(is_user, handle_start) + } +} + +async fn handle_start( + ctx: &CausalCommandContext<'_, Ticker>, + input: StartInput, +) -> std::result::Result>, HandlerError> { + let repo = ctx.repo(); + if repo.get(&input.id).await?.is_some() { + return Err(rejected("ticker already started")); + } + let mut ticker = repo.create(); + ticker + .start(input.id.clone(), input.limit, input.period_ms) + .map_err(rejected)?; + ctx.arm_alarm( + CellAlarm::after( + "tick", + Duration::from_millis(input.period_ms), + TICK, + &TickerInput { + id: input.id.clone(), + }, + ) + .map_err(rejected)?, + )?; + if input.fail_after_arming { + return Err(rejected("start refused after arming")); + } + repo.commit(ticker)?.succeeded(TickerPayload { + id: input.id, + ticks: 0, + }) +} + +struct Tick; +impl PortableCommand for Tick +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(TICK)) + .guarded(cell_alarm_guard::, handle_tick) + } +} + +async fn handle_tick( + ctx: &CausalCommandContext<'_, Ticker>, + input: TickerInput, +) -> std::result::Result>, HandlerError> { + let repo = ctx.repo(); + let mut ticker = repo + .get(&input.id) + .await? + .ok_or_else(|| rejected("no ticker"))?; + ticker.tick().map_err(rejected)?; + if ticker.ticks < ticker.limit { + let due = ctx + .alarm_due() + .ok_or_else(|| rejected("tick outside an alarm"))? + + Duration::from_millis(ticker.period_ms); + ctx.arm_alarm(CellAlarm::at("tick", due, TICK, &input).map_err(rejected)?)?; + } + let ticks = ticker.ticks; + repo.publish_events() + .commit(ticker)? + .succeeded(TickerPayload { + id: input.id, + ticks, + }) +} + +struct Touch; +impl PortableCommand for Touch +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "ticker.touch", + )) + .guarded(is_user, handle_touch) + } +} + +async fn handle_touch( + ctx: &CausalCommandContext<'_, Ticker>, + input: TickerInput, +) -> std::result::Result>, HandlerError> { + let repo = ctx.repo(); + let mut ticker = repo + .get(&input.id) + .await? + .ok_or_else(|| rejected("no ticker"))?; + ticker.touch().map_err(rejected)?; + let ticks = ticker.ticks; + repo.publish_events() + .commit(ticker)? + .succeeded(TickerPayload { + id: input.id, + ticks, + }) +} + +#[durable_object] +pub struct TickerCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for TickerCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let outbox = CelldOutbox::from_env(&env, "OUTBOX").expect("OUTBOX Queue binding"); + let cell = AggregateCell::::from_state(state) + .expect("ticker cell identity") + .mount(Start) + .mount(Tick) + .mount(Touch) + .with_celld_outbox(outbox); + Self { cell, env } + } + + async fn fetch(&self, mut req: Request) -> Result { + if let Err(error) = authenticate_internal_request(&req, &self.env) { + return error_response(error); + } + let url = req.url()?; + let parts: Vec = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .map(str::to_string) + .collect(); + let id = match parts.get(1) { + Some(id) if parts.first().map(String::as_str) == Some("ticker") => id.clone(), + _ => return json_status(json!({ "error": "missing ticker id" }), 400), + }; + match (req.method(), parts.get(2).map(String::as_str)) { + (Method::Get, None) => self.state(&id).await, + (Method::Post, Some(command)) + if matches!(command, "ticker.start" | "ticker.touch" | TICK) => + { + let command = command.to_string(); + self.command(&id, &command, &mut req).await + } + _ => json_status(json!({ "error": "not found" }), 404), + } + } + + async fn alarm(&self) -> Result { + let wake = self + .cell + .alarm(&self.env) + .await + .map_err(|error| Error::RustError(error.to_string()))?; + for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + worker::console_error!( + "cell alarm {} ({}) failed: {} {}", + failed.name, + failed.command, + failed.code, + failed.message + ); + } + for error in wake.drain.deferred { + worker::console_error!("outbox drain deferred: {error}"); + } + Response::ok("ok") + } +} + +impl TickerCell { + async fn state(&self, id: &str) -> Result { + let ticker = self + .cell + .load() + .await + .map_err(|error| Error::RustError(error.to_string()))?; + let history = self + .cell + .history() + .await + .map_err(|error| Error::RustError(error.to_string()))?; + let armed = self + .cell + .armed_alarms() + .map_err(|error| Error::RustError(error.to_string()))?; + let backlog = self + .cell + .outbox_backlog() + .await + .map_err(|error| Error::RustError(error.to_string()))?; + json_status( + json!({ + "id": id, + "exists": ticker.is_some(), + "ticks": ticker.as_ref().map_or(0, |t| t.ticks), + "touches": ticker.as_ref().map_or(0, |t| t.touches), + "tickEvents": history.iter().filter(|e| e.event_name == "ticker.ticked").count(), + "armed": armed.iter().map(|alarm| json!({ + "name": alarm.name, + "command": alarm.command, + "dueAtMs": alarm.due_at_ms, + "generation": alarm.generation, + "attempts": alarm.attempts, + })).collect::>(), + "outboxPending": backlog.pending, + "nowMs": Date::now().as_millis(), + }), + 200, + ) + } + + async fn command(&self, id: &str, command: &str, req: &mut Request) -> Result { + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => return json_status(json!({ "error": "invalid JSON" }), 400), + }; + let parsed = match CellWaitPathRequest::parse(body) { + Ok(parsed) => parsed, + Err(error) => return error_response(CellDispatchError::BadRequest(error)), + }; + let identity = match request_identity(req, &parsed.command_id) { + Ok(identity) => identity, + Err(error) => return error_response(error), + }; + let mut input = parsed.input; + if let Some(object) = input.as_object_mut() { + object.insert("id".into(), json!(id)); + } + let result = self + .cell + .dispatch_idempotent(command, &identity, input, request_session(req)) + .await; + let defer = req + .headers() + .get("x-distributed-test-defer-drain") + .ok() + .flatten() + .as_deref() + == Some("1"); + if !defer { + match self.cell.drain_outbox(&self.env).await { + Ok(outcome) => { + for error in outcome.deferred { + worker::console_error!("post-commit drain deferred: {error}"); + } + } + Err(error) => worker::console_error!("post-commit drain deferred: {error}"), + } + } + match result { + Ok(dispatch) => json_status( + json!({ + "payload": dispatch.payload(), + "receipt": { "commandId": dispatch.command_id(), "replayed": dispatch.replayed() }, + }), + 200, + ), + Err(error) => error_response(error), + } + } +} + +#[event(fetch)] +async fn main(req: Request, env: Env, _ctx: Context) -> Result { + console_error_panic_hook::set_once(); + let url = req.url()?; + let path = url.path(); + if path == "/" || path == "/health" { + return Response::ok("distributed ticker cell\n"); + } + if let Err(error) = authenticate_internal_request(&req, &env) { + return error_response(error); + } + let parts: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + let id = match (parts.first().copied(), parts.get(1).copied()) { + (Some("ticker"), Some(id)) => id, + _ => return Response::error("ticker: authenticated internal routes\n", 404), + }; + let stub = env.durable_object("TICKER")?.id_from_name(id)?.get_stub()?; + stub.fetch_with_request(req).await +} + +fn authenticate_internal_request( + req: &Request, + env: &Env, +) -> std::result::Result<(), CellDispatchError> { + let configured = env + .secret(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + .or_else(|_| { + env.var(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + }) + .map_err(|_| { + CellDispatchError::Internal(format!("{CELL_INTERNAL_SECRET_ENV} is required")) + })?; + let secret = InternalHttpSecret::new(configured).map_err(CellDispatchError::Internal)?; + let candidate = req + .headers() + .get(CELL_INTERNAL_SECRET_HEADER) + .map_err(|_| CellDispatchError::Unauthorized)? + .ok_or(CellDispatchError::Unauthorized)?; + if secret.matches(&candidate) { + Ok(()) + } else { + Err(CellDispatchError::Unauthorized) + } +} + +fn request_identity( + req: &Request, + command_id: &str, +) -> std::result::Result { + let header = |name: &str| { + req.headers() + .get(name) + .ok() + .flatten() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .ok_or(CellDispatchError::Unauthorized) + }; + CellCommandIdentity::new( + header(CELL_SERVICE_ID_HEADER)?, + header(CELL_PRINCIPAL_PARTITION_HEADER)?, + command_id, + ) +} + +/// Only the verified user and roles cross the HTTP boundary; the alarm claims +/// that `cell_alarm_guard` checks are set by the cell host alone. +fn request_session(req: &Request) -> Session { + let mut session = Session::new(); + for key in [USER_ID_KEY, ROLE_KEY] { + if let Some(value) = req.headers().get(key).ok().flatten() { + if !value.is_empty() { + session.set(key, value); + } + } + } + session +} + +fn error_response(error: CellDispatchError) -> Result { + json_status( + json!({ "error": error.client_message(), "code": error.code() }), + error.status_code(), + ) +} + +fn json_status(body: Value, status: u16) -> Result { + Ok(Response::from_json(&body)?.with_status(status)) +} diff --git a/tests/celld/alarm-worker/wrangler.jsonc b/tests/celld/alarm-worker/wrangler.jsonc new file mode 100644 index 000000000..32502f1f1 --- /dev/null +++ b/tests/celld/alarm-worker/wrangler.jsonc @@ -0,0 +1,22 @@ +{ + "name": "distributed-ticker-cell", + "main": "build/worker/shim.mjs", + "compatibility_date": "2026-01-01", + "durable_objects": { + "bindings": [ + { "name": "TICKER", "class_name": "TickerCell" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["TickerCell"] } + ], + "queues": { + "producers": [ + { "binding": "OUTBOX", "queue": "distributed-ticker-outbox" } + ] + }, + "vars": { + "DISTRIBUTED_INTERNAL_SECRET": "test-only-internal-secret-change-me-2026", + "OUTBOX_DRAIN_INTERVAL_MS": "5000" + } +} diff --git a/tests/celld/alarms.mjs b/tests/celld/alarms.mjs new file mode 100644 index 000000000..a2b1a5a90 --- /dev/null +++ b/tests/celld/alarms.mjs @@ -0,0 +1,251 @@ +// Re-armable cell alarms on celld (`celld dev`, no Docker). +// Build first: (cd tests/celld/alarm-worker && worker-build --release) +// Owns an isolated celld process group, port and object store; it never touches +// a caller's fleet. Artifacts (logs, results.json) are kept in a temp dir. +import assert from "node:assert/strict"; +import { spawn, execFileSync } from "node:child_process"; +import { once } from "node:events"; +import { createWriteStream } from "node:fs"; +import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { createServer } from "node:net"; +import { fileURLToPath } from "node:url"; +import { setTimeout as delay } from "node:timers/promises"; + +const here = dirname(fileURLToPath(import.meta.url)); +const artifacts = await mkdtemp(join(tmpdir(), "distributed-cell-alarms-")); +const project = join(artifacts, "worker"); +await mkdir(project); +await cp(join(here, "alarm-worker/build"), join(project, "build"), { recursive: true }); +const config = JSON.parse(await readFile(join(here, "alarm-worker/wrangler.jsonc"), "utf8")); +await writeFile(join(project, "wrangler.jsonc"), JSON.stringify(config, null, 2)); +const watchdogMs = Number(config.vars.OUTBOX_DRAIN_INTERVAL_MS); + +const server = createServer(); +server.listen(0, "127.0.0.1"); +await once(server, "listening"); +const port = server.address().port; +await new Promise((resolve, reject) => server.close(error => (error ? reject(error) : resolve()))); +const base = `http://127.0.0.1:${port}`; +const headers = { + "content-type": "application/json", + "x-distributed-internal-secret": config.vars.DISTRIBUTED_INTERNAL_SECRET, + "x-distributed-service-id": "cell-alarm-conformance", + "x-distributed-principal-partition": "alice", + "x-user-id": "alice", + "x-roles": "user", +}; +const nonce = Date.now().toString(36); +// `CELLD` selects the binary (default: `celld` on PATH). celld 0.6 watches the +// project by default; this proof owns its build, so it opts out when supported. +const celld = process.env.CELLD || "celld"; +const celldVersion = execFileSync(celld, ["--version"], { encoding: "utf8" }) + .split("\n") + .find(line => line.startsWith("celld ")); +const devHelp = execFileSync(celld, ["dev", "--help"], { encoding: "utf8" }); +const devFlags = devHelp.includes("--no-watch") ? ["--no-watch"] : []; +let child; +let generation = 0; +let sequence = 0; +const results = []; +const commandId = () => `0190a000-0000-7000-8000-${String(++sequence).padStart(12, "0")}`; + +async function until(description, predicate, timeout = 60_000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + const value = await predicate(); + if (value) return value; + if (child?.exitCode !== null && child?.exitCode !== undefined) throw new Error("celld exited"); + await delay(100); + } + throw new Error(`timed out: ${description}; artifacts: ${artifacts}`); +} + +async function start() { + const log = createWriteStream(join(artifacts, `celld-${++generation}.log`)); + child = spawn(celld, ["dev", project, "--host", "127.0.0.1", "--port", String(port), "--logs", ...devFlags], { + detached: true, + env: { ...process.env, RUST_LOG: "warn" }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + child.once("close", () => log.end()); + child.once("error", error => console.error(error)); + await until("Worker readiness", async () => { + try { + return (await fetch(base + "/health", { signal: AbortSignal.timeout(1000) })).ok; + } catch { + return false; + } + }); +} + +// Kill only the celld launcher this script started and its descendants. +async function crash() { + const owned = child; + child = undefined; + if (!owned || owned.exitCode !== null) return; + const exited = once(owned, "close"); + process.kill(owned.pid, "SIGSTOP"); + const processes = execFileSync("ps", ["-axo", "pid=,ppid="], { encoding: "utf8" }) + .trim() + .split("\n") + .map(line => line.trim().split(/\s+/).map(Number)); + const descendants = []; + (function visit(parent) { + for (const [pid, ppid] of processes) { + if (ppid === parent) { + visit(pid); + descendants.push(pid); + } + } + })(owned.pid); + for (const pid of [...descendants, owned.pid]) { + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if (error.code !== "ESRCH") throw error; + } + } + await exited; +} + +async function post(id, command, input, extraHeaders = {}) { + const response = await fetch(`${base}/ticker/${id}/${command}`, { + method: "POST", + headers: { ...headers, ...extraHeaders }, + body: JSON.stringify({ commandId: commandId(), input }), + signal: AbortSignal.timeout(20_000), + }); + return { status: response.status, body: await response.json() }; +} + +async function state(id) { + const response = await fetch(`${base}/ticker/${id}`, { headers, signal: AbortSignal.timeout(10_000) }); + assert.equal(response.status, 200); + return response.json(); +} + +async function startTicker(id, limit, periodMs) { + const started = await post(id, "ticker.start", { limit, period_ms: periodMs, fail_after_arming: false }); + assert.equal(started.status, 200, JSON.stringify(started.body)); + return Date.now(); +} + +async function record(name, evidence) { + results.push({ name, evidence }); + console.log(JSON.stringify({ name, evidence })); + await writeFile(join(artifacts, "results.json"), JSON.stringify(results, null, 2)); +} + +try { + console.log(`cell alarm proof artifacts: ${artifacts} (${celldVersion})`); + await record("celld", { version: celldVersion }); + await start(); + + // 1. The alarm row commits with the command or not at all. + { + const id = `refused-${nonce}`; + const refused = await post(id, "ticker.start", { limit: 1, period_ms: 500, fail_after_arming: true }); + assert.equal(refused.status, 422, JSON.stringify(refused.body)); + const after = await state(id); + assert.equal(after.exists, false); + assert.deepEqual(after.armed, []); + await record("failed command leaves no alarm", { status: refused.status, armed: after.armed }); + } + + // 2. Ordinary callers cannot run the alarm-only command. + { + const id = `forged-${nonce}`; + await startTicker(id, 1, 60_000); + const forged = await post(id, "ticker.tick", {}, { "x-roles": "cell-alarm", "x-user-id": "system:cell-alarm" }); + assert.ok(forged.status >= 400, JSON.stringify(forged.body)); + assert.equal((await state(id)).ticks, 0); + await record("alarm-only command refuses callers", { status: forged.status, code: forged.body.code }); + } + + // 3. Re-arming in the handler gives a periodic tick at the alarm cadence, + // not the 5 s watchdog cadence: domain alarms are never postponed by + // watchdog rearms (each tick publishes an outbox event, so the watchdog + // is armed throughout). + { + const id = `periodic-${nonce}`; + const limit = 8; + const periodMs = 500; + const startedAt = await startTicker(id, limit, periodMs); + const done = await until("periodic ticks", async () => { + const current = await state(id); + return current.ticks >= limit && current; + }, 30_000); + const elapsed = Date.now() - startedAt; + assert.ok(elapsed < limit * periodMs + watchdogMs, `ticks took ${elapsed} ms`); + await delay(2 * periodMs + 500); + const settled = await until("periodic outbox drained", async () => { + const current = await state(id); + return current.outboxPending === 0 && current; + }, 3 * watchdogMs); + assert.equal(settled.ticks, limit, "fires once per arm"); + assert.equal(settled.tickEvents, limit); + assert.deepEqual(settled.armed, []); + await record("periodic re-arm", { limit, periodMs, elapsed, ticks: settled.ticks, tickEvents: settled.tickEvents }); + } + + // 4. A far domain alarm never starves the outbox watchdog: a row committed + // without the immediate drain is still delivered by the alarm wake, and + // the domain alarm stays armed with its original due time. + { + const id = `watchdog-${nonce}`; + await startTicker(id, 1, 3_600_000); + const armed = (await state(id)).armed; + assert.equal(armed.length, 1); + const touched = await post(id, "ticker.touch", {}, { "x-distributed-test-defer-drain": "1" }); + assert.equal(touched.status, 200, JSON.stringify(touched.body)); + const pending = await state(id); + assert.ok(pending.outboxPending >= 1, JSON.stringify(pending)); + const startedAt = Date.now(); + const drained = await until("watchdog drain", async () => { + const current = await state(id); + return current.outboxPending === 0 && current; + }, 3 * watchdogMs); + const elapsed = Date.now() - startedAt; + assert.deepEqual(drained.armed, armed, "the domain alarm survives the watchdog wake"); + assert.equal(drained.ticks, 0); + await record("watchdog not starved by a far domain alarm", { elapsed, pendingBefore: pending.outboxPending }); + } + + // 5. Fires once per arm across a celld restart: the alarm comes due while + // the fleet is down and fires exactly once after it restarts. + { + const id = `restart-${nonce}`; + await startTicker(id, 2, 3_000); + const before = await state(id); + assert.equal(before.armed.length, 1); + await crash(); + await delay(4_000); + await start(); + const fired = await until("overdue alarm after restart", async () => { + const current = await state(id); + return current.ticks >= 1 && current; + }, 30_000); + assert.equal(fired.ticks, 1); + const second = await until("re-armed alarm after restart", async () => { + const current = await state(id); + return current.ticks >= 2 && current; + }, 30_000); + await delay(4_000); + const final = await state(id); + assert.equal(final.ticks, 2, "no duplicate fire after restart"); + assert.equal(final.tickEvents, 2); + assert.deepEqual(final.armed, []); + await record("fires once across restart", { generationBefore: before.armed[0].generation, ticks: final.ticks, secondAt: second.nowMs }); + } + + console.log(`cell alarm proof passed (${results.length} checks)`); +} catch (error) { + console.error(`cell alarm proof FAILED; artifacts: ${artifacts}`); + throw error; +} finally { + await crash(); +} diff --git a/tests/celld/main.rs b/tests/celld/main.rs index 4f1af64dd..368eae7b3 100644 --- a/tests/celld/main.rs +++ b/tests/celld/main.rs @@ -84,6 +84,10 @@ fn worker_declares_sqlite_todo_and_chat_cells() { assert!(source.contains("CelldOutbox::from_env(&env, \"OUTBOX\")")); assert!(source.contains("with_celld_outbox(outbox)")); assert!(source.contains(".drain_outbox(env)")); + assert!( + source.contains("cell\n .alarm(env)"), + "the Durable Object alarm handler fires cell alarms and drains through AggregateCell::alarm" + ); assert!(!source.contains("persist_and_drain_outbox")); assert!(!source.contains("CelldQueuePublisher::from_env")); assert!(!source.contains("drain_outbox_to_queue")); @@ -103,6 +107,34 @@ fn worker_declares_sqlite_todo_and_chat_cells() { assert!(!source.contains("restore_cell_state")); } +#[test] +fn alarm_worker_arms_cell_alarms_and_shares_the_durable_object_alarm() { + let dir = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/celld/alarm-worker" + )); + let wrangler = std::fs::read_to_string(dir.join("wrangler.jsonc")).expect("wrangler.jsonc"); + let spec: Value = serde_json::from_str(&wrangler).expect("wrangler json"); + assert_eq!( + spec["durable_objects"]["bindings"][0]["class_name"], + "TickerCell" + ); + assert_eq!(spec["migrations"][0]["new_sqlite_classes"][0], "TickerCell"); + assert_eq!(spec["queues"]["producers"][0]["binding"], "OUTBOX"); + let source = std::fs::read_to_string(dir.join("src/lib.rs")).expect("lib.rs"); + assert!(source.contains("ctx.arm_alarm(")); + assert!(source.contains("cell_alarm_guard::")); + assert!(source.contains(".alarm(&self.env)")); + assert!(source.contains("with_celld_outbox(outbox)")); + let harness = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/celld/alarms.mjs" + )) + .expect("alarms.mjs"); + assert!(harness.contains("fires once across restart")); + assert!(harness.contains("watchdog not starved by a far domain alarm")); +} + #[test] fn relay_worker_consumes_events_queue_through_native_bus_boundary() { let wrangler = std::fs::read_to_string(relay_worker_dir().join("wrangler.jsonc")) diff --git a/tests/celld/worker/src/lib.rs b/tests/celld/worker/src/lib.rs index 7ff17413d..23678b3b4 100644 --- a/tests/celld/worker/src/lib.rs +++ b/tests/celld/worker/src/lib.rs @@ -42,6 +42,35 @@ where Ok(()) } +/// Durable Object alarm: fire due cell alarms, then drain the outbox. The one +/// alarm slot is shared by both, always set to the earliest due time. +async fn alarm_cell(env: &Env, cell: &AggregateCell) -> Result<()> +where + A: distributed::Aggregate + Send + Sync + 'static, +{ + let wake = cell + .alarm(env) + .await + .map_err(|error| Error::RustError(error.to_string()))?; + for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + worker::console_error!( + "cell alarm {} on {} failed: {} {}", + failed.name, + cell.instance_name(), + failed.code, + failed.message + ); + } + for error in wake.drain.deferred { + worker::console_error!( + "celld outbox alarm drain deferred for {}: {}", + cell.instance_name(), + error + ); + } + Ok(()) +} + async fn drain_after_command(env: &Env, cell: &AggregateCell, request: &Request) where A: distributed::Aggregate + Send + Sync + 'static, @@ -151,7 +180,7 @@ impl DurableObject for TodoCell { } async fn alarm(&self) -> Result { - drain_cell(&self.env, &self.cell).await?; + alarm_cell(&self.env, &self.cell).await?; Response::ok("ok") } } @@ -199,7 +228,7 @@ impl DurableObject for ChatCell { } async fn alarm(&self) -> Result { - drain_cell(&self.env, &self.cell).await?; + alarm_cell(&self.env, &self.cell).await?; Response::ok("ok") } } diff --git a/tests/persistent_repository_conformance/read_models.rs b/tests/persistent_repository_conformance/read_models.rs index ab73c45fb..1ebaa1085 100644 --- a/tests/persistent_repository_conformance/read_models.rs +++ b/tests/persistent_repository_conformance/read_models.rs @@ -179,6 +179,7 @@ where let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), + alarms: Vec::new(), streams: vec![StreamWrite::new(identity, stale.entity_mut())], outbox_messages: Vec::new(), read_model_plans: vec![read_models diff --git a/tests/persistent_repository_conformance/scenario.rs b/tests/persistent_repository_conformance/scenario.rs index 64157340d..a45414f19 100644 --- a/tests/persistent_repository_conformance/scenario.rs +++ b/tests/persistent_repository_conformance/scenario.rs @@ -164,6 +164,7 @@ where let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), + alarms: Vec::new(), streams: vec![ StreamWrite::new(stale_identity, stale.entity_mut()), StreamWrite::new(checkout_identity.clone(), checkout.entity_mut()), diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index af7f7bd62..d6f7fb385 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -234,7 +234,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_and_preserve_checks() { .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 10); + assert_eq!(latest_version, 11); let invalid_service = sqlx::query( r#" @@ -337,6 +337,7 @@ async fn read_model_failure_mid_plan_rolls_back_events_and_outbox() { let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), + alarms: Vec::new(), streams: vec![StreamWrite::new(identity.clone(), projection.entity_mut())], outbox_messages: vec![outbox_message], read_model_plans: vec![read_models.into_write_plan().unwrap()], diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index 3f1511574..f233321ce 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -111,6 +111,7 @@ async fn outbox_backlog_stats_order_sqlite_text_timestamps_numerically() { read_model_plans: Vec::new(), snapshots: Vec::new(), inbox_receipts: Vec::new(), + alarms: Vec::new(), }) .await .expect("outbox-only batch should commit"); @@ -201,7 +202,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_without_schema_drift() .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 10); + assert_eq!(latest_version, 11); let created_at_type: String = sqlx::query_scalar( "SELECT typeof(created_at) FROM command_ledger WHERE service_id = 'service'", @@ -317,6 +318,7 @@ async fn optimistic_conflict_rolls_back_other_stream_and_read_model_plan() { let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), + alarms: Vec::new(), streams: vec![ StreamWrite::new(stale_identity.clone(), stale.entity_mut()), StreamWrite::new(other_identity.clone(), other.entity_mut()), @@ -407,6 +409,7 @@ async fn read_model_failure_mid_plan_rolls_back_events_and_outbox() { let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), + alarms: Vec::new(), streams: vec![StreamWrite::new(identity.clone(), projection.entity_mut())], outbox_messages: vec![outbox_message], read_model_plans: vec![read_models.into_write_plan().unwrap()], From dd35d4a087ebb3a2573bdee6c033bb7803e4ea62 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 10:05:16 -0500 Subject: [PATCH 08/40] feat(saga): fan-out steps to N cells chosen at run time SagaDefinition::action now returns SagaAction: a single command or a FanOut of up to MAX_SAGA_FAN_OUT (64) commands. Each member gets its own command id (type:saga:step:phase:attempt:member), saga-member metadata and timeout timer. The step succeeds when every member succeeds; it settles only after every member replied or timed out, then compensates only the members that applied or timed out, one at a time, highest index first. Late replies after a member timeout are ignored. compensation and on_step_succeeded take the member index. New events (member_succeeded, member_failed, fan_out_settled) replay to the same state. Breaking: action/compensation/on_step_succeeded signatures changed. [[tasks/gr-p27-saga-fanout]] --- docs/coordination-primitives.md | 50 ++++ src/lib.rs | 4 +- src/saga/definition.rs | 74 +++++- src/saga/fan_out_tests.rs | 416 ++++++++++++++++++++++++++++++++ src/saga/message.rs | 14 ++ src/saga/mod.rs | 35 ++- src/saga/state.rs | 382 ++++++++++++++++++++++++++--- tests/saga_transfer/main.rs | 10 +- 8 files changed, 920 insertions(+), 65 deletions(-) create mode 100644 src/saga/fan_out_tests.rs diff --git a/docs/coordination-primitives.md b/docs/coordination-primitives.md index 848377992..c3d031f48 100644 --- a/docs/coordination-primitives.md +++ b/docs/coordination-primitives.md @@ -174,6 +174,56 @@ Guarantees and obligations: the saga id and rejects a late action. - **`on_step_succeeded` is replayed,** so keep it deterministic. +### Fan-out steps + +A step can send commands to N cells chosen at run time, for example one +ticket reservation per party when a match starts. `action` returns +`SagaAction::FanOut(commands)` instead of a single command +(`SagaCommand` converts with `.into()`); `compensation` and +`on_step_succeeded` receive the member index (`0` for single steps). + +```rust +fn action(step: &str, saga_id: &str, d: &MatchStart) -> Result { + match step { + "reserve" => Ok(SagaAction::fan_out( + d.parties.iter().map(|p| SagaCommand::json(format!("party:{p}"), "ticket.reserve", &saga_id)) + .collect::, _>>()?, + )), + _ => Ok(SagaCommand::json("match", "match.confirm", &saga_id)?.into()), + } +} +fn compensation(step: &str, member: usize, saga_id: &str, d: &MatchStart) + -> Result, SagaError> { + match step { + "reserve" => { + SagaCommand::json(format!("party:{}", d.parties[member]), "ticket.release", &saga_id).map(Some) + } + _ => Ok(None), + } +} +``` + +- 1 to `MAX_SAGA_FAN_OUT` (64) members; an empty or larger fan-out is a + `Definition` error. Membership must be a deterministic function of the data. +- Every member gets its own command and timeout timer, both carrying + `saga-member` metadata. Command ids are + `type:saga:step:phase:attempt:member`, so retries dedupe per member. +- The step **succeeds** when every member replies `succeeded`. It + **settles** only after every member has replied or timed out, so the saga + knows which members applied. On any rejection or timeout the step fails and + only the members that applied, or timed out (outcome unknown), are + compensated, one at a time, highest index first. Members that rejected are + not compensated. +- A reply arriving after its member timed out is `Ignored`; the timeout + already settled that member and it will be compensated. +- Events: `saga.member_succeeded`, `saga.member_failed`, + `saga.fan_out_settled`. Replaying the stream reproduces the per-member + state (`Saga::fan_out`, `Saga::applied_members`). + +The tests are in `src/saga/fan_out_tests.rs` (N = 1..16, partial failure, +per-member timeouts and late replies, per-member compensation retries, +bounds, and replay from a repository). + ### Worked example `tests/saga_transfer/main.rs` moves value between wallets in two separate diff --git a/src/lib.rs b/src/lib.rs index 86e92cb02..f73f9d679 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -374,8 +374,8 @@ pub use lease::PostgresLeaseStore; pub use lease::SqliteLeaseStore; pub use lease::{Fence, FencingToken, InMemoryLeaseStore, Lease, LeaseError, LeaseStore}; pub use saga::{ - Saga, SagaCommand, SagaDefinition, SagaError, SagaMessage, SagaOutcome, SagaReply, SagaRunner, - SagaStatus, SagaStep, + Saga, SagaAction, SagaCommand, SagaDefinition, SagaError, SagaMessage, SagaOutcome, SagaReply, + SagaRunner, SagaStatus, SagaStep, }; // Outbox: commit concerns (aggregate + outbox in one commit). diff --git a/src/saga/definition.rs b/src/saga/definition.rs index bef69dd95..c338fa873 100644 --- a/src/saga/definition.rs +++ b/src/saga/definition.rs @@ -73,6 +73,55 @@ impl SagaCommand { } } +/// Most commands one fan-out step may dispatch. +pub const MAX_SAGA_FAN_OUT: usize = 64; + +/// What a forward step dispatches: one command, or a fan-out to N cells +/// decided at run time (for example one ticket reservation per party). +/// +/// A fan-out step succeeds when **every** member replies `succeeded`. It +/// settles only once every member has replied or timed out; if any member +/// failed or timed out, the step fails and compensation runs for the members +/// that applied (or may have, after a timeout) — never for members that +/// definitively rejected. Each member has its own timeout timer and a +/// deterministic command id per `(step, member index)`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SagaAction { + /// A single command (`member` is `None` on its replies). + Single(SagaCommand), + /// `1..=MAX_SAGA_FAN_OUT` commands, member `i` is the `i`-th command. + FanOut(Vec), +} + +impl SagaAction { + /// A fan-out over `commands` (validated when dispatched). + pub fn fan_out(commands: impl IntoIterator) -> Self { + Self::FanOut(commands.into_iter().collect()) + } + + pub(super) fn into_commands(self) -> Result<(Vec, bool), SagaError> { + match self { + Self::Single(command) => Ok((vec![command], false)), + Self::FanOut(commands) if commands.is_empty() => Err(SagaError::Definition( + "a fan-out step needs at least one command".into(), + )), + Self::FanOut(commands) if commands.len() > MAX_SAGA_FAN_OUT => { + Err(SagaError::Definition(format!( + "a fan-out step may dispatch at most {MAX_SAGA_FAN_OUT} commands, not {}", + commands.len() + ))) + } + Self::FanOut(commands) => Ok((commands, true)), + } + } +} + +impl From for SagaAction { + fn from(command: SagaCommand) -> Self { + Self::Single(command) + } +} + /// Static description of a saga (process manager). /// /// The definition is pure: it maps the saga's data to the commands for each @@ -97,25 +146,30 @@ pub trait SagaDefinition: Send + Sync + 'static { /// Forward steps, executed in order. fn steps() -> &'static [SagaStep]; - /// The command that performs `step`. - fn action(step: &str, saga_id: &str, data: &Self::Data) -> Result; + /// What `step` dispatches: a single command (`SagaCommand` converts + /// with `.into()`) or a [`SagaAction::FanOut`]. Must be a deterministic + /// function of the data. + fn action(step: &str, saga_id: &str, data: &Self::Data) -> Result; - /// The command that undoes `step`, or `None` when the step needs no - /// compensation. Compensations must be idempotent **and** must tolerate - /// arriving before (or instead of) the action they undo — the participant - /// should record a tombstone so a late action is rejected. + /// The command that undoes `step` (for fan-out steps, member `member` of + /// it; single steps pass `0`), or `None` when it needs no compensation. + /// Compensations must be idempotent **and** must tolerate arriving before + /// (or instead of) the action they undo — the participant should record a + /// tombstone so a late action is rejected. fn compensation( step: &str, + member: usize, saga_id: &str, data: &Self::Data, ) -> Result, SagaError> { - let _ = (step, saga_id, data); + let _ = (step, member, saga_id, data); Ok(None) } - /// Fold a successful step's reply payload into the saga data. Replayed — + /// Fold a successful reply payload into the saga data (for fan-out steps, + /// once per member, `member` its index; single steps pass `0`). Replayed — /// keep it deterministic. Default: ignore the output. - fn on_step_succeeded(step: &str, data: &mut Self::Data, output: &[u8]) { - let _ = (step, data, output); + fn on_step_succeeded(step: &str, member: usize, data: &mut Self::Data, output: &[u8]) { + let _ = (step, member, data, output); } } diff --git a/src/saga/fan_out_tests.rs b/src/saga/fan_out_tests.rs new file mode 100644 index 000000000..1bba07775 --- /dev/null +++ b/src/saga/fan_out_tests.rs @@ -0,0 +1,416 @@ +//! Fan-out steps: N commands to N cells decided at run time, per-member +//! timeouts, compensation of only the applied members, and replay. + +use std::collections::BTreeMap; +use std::time::{Duration, SystemTime}; + +use serde::{Deserialize, Serialize}; + +use super::{ + Saga, SagaAction, SagaCommand, SagaDefinition, SagaError, SagaMessage, SagaOutcome, SagaPhase, + SagaReply, SagaRunner, SagaStatus, SagaStep, MAX_SAGA_FAN_OUT, META_MEMBER, SAGA_TIMEOUT, +}; +use crate::outbox::OutboxMessage; +use crate::{InMemoryRepository, ManualClock}; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +struct MatchStart { + parties: Vec, + /// Reply payload of each reserved member, folded by `on_step_succeeded`. + reserved: BTreeMap, + confirmed: bool, +} + +struct StartMatch; + +impl SagaDefinition for StartMatch { + const SAGA_TYPE: &'static str = "test.match_start"; + const REPLY_TO: &'static str = "match-start-saga"; + type Data = MatchStart; + + fn steps() -> &'static [SagaStep] { + const STEPS: &[SagaStep] = &[ + SagaStep::new("reserve", Duration::from_secs(5)), + SagaStep::new("confirm", Duration::from_secs(5)), + ]; + STEPS + } + + fn action(step: &str, saga_id: &str, data: &MatchStart) -> Result { + match step { + "reserve" => Ok(SagaAction::fan_out( + data.parties + .iter() + .map(|party| { + SagaCommand::json(format!("party:{party}"), "ticket.reserve", &saga_id) + }) + .collect::, _>>()?, + )), + "confirm" => Ok(SagaCommand::json("match", "match.confirm", &saga_id)?.into()), + other => Err(SagaError::Definition(format!("unknown step {other}"))), + } + } + + fn compensation( + step: &str, + member: usize, + saga_id: &str, + data: &MatchStart, + ) -> Result, SagaError> { + match step { + "reserve" => SagaCommand::json( + format!("party:{}", data.parties[member]), + "ticket.release", + &saga_id, + ) + .map(Some), + _ => Ok(None), + } + } + + fn on_step_succeeded(step: &str, member: usize, data: &mut MatchStart, output: &[u8]) { + match step { + "reserve" => { + data.reserved + .insert(member, String::from_utf8_lossy(output).into_owned()); + } + "confirm" => data.confirmed = true, + _ => {} + } + } +} + +const T0: Duration = Duration::from_secs(1_700_000_000); + +fn now() -> SystemTime { + SystemTime::UNIX_EPOCH + T0 +} + +fn data(n: usize) -> MatchStart { + MatchStart { + parties: (0..n).map(|index| format!("p{index}")).collect(), + ..MatchStart::default() + } +} + +fn commands(messages: &[OutboxMessage]) -> Vec<&OutboxMessage> { + messages + .iter() + .filter(|message| message.event_type != SAGA_TIMEOUT) + .collect() +} + +fn timers(messages: &[OutboxMessage]) -> Vec<&OutboxMessage> { + messages + .iter() + .filter(|message| message.event_type == SAGA_TIMEOUT) + .collect() +} + +fn succeeded(command: &OutboxMessage, output: &str) -> SagaMessage { + let reply = SagaReply::from_outbox(command) + .unwrap() + .succeeded(output.as_bytes().to_vec()) + .unwrap(); + SagaMessage::from_outbox(&reply).unwrap() +} + +fn failed(command: &OutboxMessage, reason: &str) -> SagaMessage { + let reply = SagaReply::from_outbox(command) + .unwrap() + .failed(reason) + .unwrap(); + SagaMessage::from_outbox(&reply).unwrap() +} + +fn timed_out(timer: &OutboxMessage) -> SagaMessage { + SagaMessage::from_outbox(timer).unwrap() +} + +fn member(message: &OutboxMessage) -> usize { + message.metadata[META_MEMBER].parse().unwrap() +} + +/// Drive a compensation stack to the end, acknowledging each compensation +/// and returning the members compensated, in order. +fn drain_compensations( + saga: &mut Saga, + mut messages: Vec, +) -> Vec { + let mut compensated = Vec::new(); + while let Some(command) = commands(&messages).first().copied().cloned() { + assert_eq!(command.event_type, "ticket.release"); + compensated.push(member(&command)); + messages = saga + .handle(&succeeded(&command, "released"), now()) + .unwrap() + .messages; + } + compensated +} + +#[test] +fn fan_out_of_one_to_sixteen_members_completes() { + for n in 1..=16 { + let mut saga = Saga::::new(); + let started = saga.start("m1", data(n), now()).unwrap(); + let sent = commands(&started.messages); + assert_eq!(sent.len(), n, "n={n}"); + assert_eq!(timers(&started.messages).len(), n, "one timer per member"); + let cursor = saga.cursor().unwrap(); + assert_eq!(cursor.fan_out, n as u32); + for (index, command) in sent.iter().enumerate() { + assert_eq!( + command.id(), + format!("test.match_start:m1:reserve:action:1:{index}"), + "deterministic command id per (step, member)" + ); + assert_eq!(member(command), index); + assert_eq!( + command.destination.as_deref(), + Some(&*format!("party:p{index}")) + ); + } + // Replies arrive in any order; the step waits for all of them. + let mut next = Vec::new(); + for command in sent.iter().rev() { + assert_eq!(saga.status(), SagaStatus::Running); + assert!(saga.cursor().is_some_and(|c| c.step == 0)); + next = saga + .handle( + &succeeded(command, &format!("ticket-{}", member(command))), + now(), + ) + .unwrap() + .messages; + } + let confirm = commands(&next); + assert_eq!(confirm.len(), 1); + assert_eq!(confirm[0].event_type, "match.confirm"); + assert!(!confirm[0].metadata.contains_key(META_MEMBER)); + saga.handle(&succeeded(confirm[0], "ok"), now()).unwrap(); + assert_eq!(saga.status(), SagaStatus::Completed); + assert_eq!(saga.data().reserved.len(), n); + assert_eq!(saga.data().reserved[&0], "ticket-0"); + assert!(saga.data().confirmed); + } +} + +#[test] +fn one_rejection_compensates_only_the_applied_members() { + let mut saga = Saga::::new(); + let started = saga.start("m2", data(5), now()).unwrap(); + let sent: Vec = commands(&started.messages).into_iter().cloned().collect(); + for index in [0, 1] { + let transition = saga.handle(&succeeded(&sent[index], "t"), now()).unwrap(); + assert!(transition.messages.is_empty()); + } + // A rejection does not settle the step while member 3 and 4 are pending: + // their outcome must be known before choosing what to compensate. + let rejected = saga.handle(&failed(&sent[2], "party left"), now()).unwrap(); + assert!(rejected.messages.is_empty()); + assert_eq!(saga.status(), SagaStatus::Running); + saga.handle(&succeeded(&sent[3], "t"), now()).unwrap(); + let settled = saga.handle(&succeeded(&sent[4], "t"), now()).unwrap(); + assert_eq!(saga.status(), SagaStatus::Compensating); + let failure = saga.failure().unwrap(); + assert_eq!(failure.step, "reserve"); + assert!(failure.reason.contains("party left")); + assert!(!failure.timed_out); + assert_eq!( + saga.applied_members("reserve") + .unwrap() + .iter() + .copied() + .collect::>(), + [0, 1, 3, 4] + ); + + let compensated = drain_compensations(&mut saga, settled.messages); + assert_eq!(compensated, [4, 3, 1, 0], "never the member that rejected"); + assert_eq!(saga.status(), SagaStatus::Compensated); +} + +#[test] +fn members_time_out_individually_and_late_replies_are_ignored() { + let mut saga = Saga::::new(); + let started = saga.start("m3", data(3), now()).unwrap(); + let sent: Vec = commands(&started.messages).into_iter().cloned().collect(); + let member_timers: Vec = + timers(&started.messages).into_iter().cloned().collect(); + assert_eq!(member(&member_timers[1]), 1); + assert_eq!( + member_timers[1].deliver_at, + Some(now() + Duration::from_secs(5)) + ); + + saga.handle(&succeeded(&sent[0], "t"), now()).unwrap(); + let timeout = timed_out(&member_timers[1]); + assert_eq!( + saga.handle(&timeout, now()).unwrap().outcome, + SagaOutcome::Applied + ); + assert_eq!( + saga.handle(&timeout, now()).unwrap().outcome, + SagaOutcome::Duplicate + ); + // The participant answers after its timer fired: ignored. + assert_eq!( + saga.handle(&succeeded(&sent[1], "late"), now()) + .unwrap() + .outcome, + SagaOutcome::Ignored + ); + // Member 0's timer after its reply: ignored. + assert_eq!( + saga.handle(&timed_out(&member_timers[0]), now()) + .unwrap() + .outcome, + SagaOutcome::Ignored + ); + let settled = saga.handle(&succeeded(&sent[2], "t"), now()).unwrap(); + let failure = saga.failure().unwrap(); + assert!(failure.timed_out); + // The timed-out member may have applied: it is compensated too. + assert_eq!(drain_compensations(&mut saga, settled.messages), [2, 1, 0]); + assert_eq!(saga.status(), SagaStatus::Compensated); + assert!( + !saga.data().reserved.contains_key(&1), + "late output not folded" + ); + + // A late reply after the saga moved on is ignored as well. + assert_eq!( + saga.handle(&succeeded(&sent[1], "later"), now()) + .unwrap() + .outcome, + SagaOutcome::Ignored, + "a terminal saga ignores stragglers" + ); +} + +#[test] +fn a_later_step_failure_compensates_every_fanned_out_member() { + let mut saga = Saga::::new(); + let started = saga.start("m4", data(3), now()).unwrap(); + let mut next = Vec::new(); + for command in commands(&started.messages) { + next = saga + .handle(&succeeded(command, "t"), now()) + .unwrap() + .messages; + } + let confirm = commands(&next)[0].clone(); + let compensating = saga.handle(&failed(&confirm, "zone full"), now()).unwrap(); + assert_eq!( + drain_compensations(&mut saga, compensating.messages), + [2, 1, 0] + ); + assert_eq!(saga.status(), SagaStatus::Compensated); +} + +#[test] +fn member_compensation_retries_carry_the_member() { + let mut saga = Saga::::new(); + let started = saga.start("m5", data(2), now()).unwrap(); + let sent: Vec = commands(&started.messages).into_iter().cloned().collect(); + saga.handle(&succeeded(&sent[0], "t"), now()).unwrap(); + let settled = saga.handle(&failed(&sent[1], "no"), now()).unwrap(); + let release = commands(&settled.messages)[0].clone(); + assert_eq!(release.id(), "test.match_start:m5:reserve:compensation:1:0"); + let retry = saga.handle(&failed(&release, "busy"), now()).unwrap(); + let again = commands(&retry.messages)[0].clone(); + assert_eq!(again.id(), "test.match_start:m5:reserve:compensation:2:0"); + assert_eq!(member(&again), 0); + let cursor = saga.cursor().unwrap(); + assert_eq!( + (cursor.phase, cursor.member, cursor.attempt), + (SagaPhase::Compensation, Some(0), 2) + ); + saga.handle(&succeeded(&again, "released"), now()).unwrap(); + assert_eq!(saga.status(), SagaStatus::Compensated); +} + +#[test] +fn fan_out_is_bounded() { + let mut empty = Saga::::new(); + assert!(matches!( + empty.start("m6", data(0), now()), + Err(SagaError::Definition(_)) + )); + let mut huge = Saga::::new(); + assert!(matches!( + huge.start("m7", data(MAX_SAGA_FAN_OUT + 1), now()), + Err(SagaError::Definition(_)) + )); + let mut max = Saga::::new(); + let started = max.start("m8", data(MAX_SAGA_FAN_OUT), now()).unwrap(); + assert_eq!(commands(&started.messages).len(), MAX_SAGA_FAN_OUT); +} + +#[test] +fn malformed_member_replies_are_rejected() { + let mut saga = Saga::::new(); + let started = saga.start("m9", data(2), now()).unwrap(); + let command = commands(&started.messages)[0].clone(); + let mut reply = succeeded(&command, "t"); + reply.member = None; + assert!(matches!( + saga.handle(&reply, now()), + Err(SagaError::Malformed(_)) + )); + reply.member = Some(7); + assert!(matches!( + saga.handle(&reply, now()), + Err(SagaError::Malformed(_)) + )); +} + +/// Every transition committed through a repository and reloaded from the +/// saga stream reproduces the live state exactly. +#[tokio::test] +async fn replaying_the_saga_stream_reproduces_fan_out_state() { + let clock = ManualClock::new(now()); + let runner = SagaRunner::<_, StartMatch, _>::with_clock(InMemoryRepository::new(), clock); + let mut live = Saga::::new(); + + let started = runner.start("m10", data(4)).await.unwrap(); + live.start("m10", data(4), now()).unwrap(); + let sent: Vec = commands(&started.messages).into_iter().cloned().collect(); + let member_timers: Vec = + timers(&started.messages).into_iter().cloned().collect(); + let mut inbound = vec![ + succeeded(&sent[0], "a"), + failed(&sent[1], "left"), + timed_out(&member_timers[2]), + succeeded(&sent[2], "late"), + succeeded(&sent[3], "d"), + ]; + let mut checkpoints = 0; + while !inbound.is_empty() { + let message = inbound.remove(0); + let committed = runner.handle(&message).await.unwrap(); + let expected = live.handle(&message, now()).unwrap(); + assert_eq!(committed.outcome, expected.outcome); + for command in commands(&committed.messages) { + inbound.push(succeeded(command, "released")); + } + let replayed = runner.get("m10").await.unwrap().unwrap(); + assert_eq!(format!("{replayed:?}"), format!("{live:?}")); + assert_eq!(replayed.data(), live.data()); + assert_eq!(replayed.fan_out(), live.fan_out()); + assert_eq!(replayed.cursor(), live.cursor()); + assert!( + replayed.has_processed(&message.message_id) + || committed.outcome != SagaOutcome::Applied + ); + checkpoints += 1; + } + let replayed = runner.get("m10").await.unwrap().unwrap(); + assert_eq!(replayed.status(), SagaStatus::Compensated); + assert_eq!( + replayed.data().reserved.keys().copied().collect::>(), + [0, 3] + ); + assert!(checkpoints >= 8); +} diff --git a/src/saga/message.rs b/src/saga/message.rs index 596ef2fb5..34c3c06d3 100644 --- a/src/saga/message.rs +++ b/src/saga/message.rs @@ -23,6 +23,8 @@ pub const META_STEP: &str = "saga-step"; pub const META_PHASE: &str = "saga-phase"; /// Metadata key: dispatch attempt (1-based). pub const META_ATTEMPT: &str = "saga-attempt"; +/// Metadata key: fan-out member index (absent on single-command steps). +pub const META_MEMBER: &str = "saga-member"; /// Metadata key: where replies must be sent. pub const META_REPLY_TO: &str = "saga-reply-to"; /// Metadata key: id of the command a reply/timeout belongs to. @@ -80,6 +82,8 @@ pub struct SagaMessage { pub step: String, pub phase: SagaPhase, pub attempt: u32, + /// Fan-out member index; `None` for single-command steps. + pub member: Option, pub signal: SagaSignal, } @@ -130,6 +134,14 @@ impl SagaMessage { let attempt = get(META_ATTEMPT)? .parse::() .map_err(|_| SagaError::Malformed("invalid saga attempt".into()))?; + let member = metadata + .get(META_MEMBER) + .map(|value| { + value + .parse::() + .map_err(|_| SagaError::Malformed("invalid saga member".into())) + }) + .transpose()?; Ok(Self { message_id: message_id.to_string(), saga_type: get(META_SAGA_TYPE)?.to_string(), @@ -137,6 +149,7 @@ impl SagaMessage { step: get(META_STEP)?.to_string(), phase: SagaPhase::parse(get(META_PHASE)?)?, attempt, + member, signal, }) } @@ -198,6 +211,7 @@ impl SagaReply { META_STEP, META_PHASE, META_ATTEMPT, + META_MEMBER, META_REPLY_TO, ] .contains(&key) diff --git a/src/saga/mod.rs b/src/saga/mod.rs index 76590ddce..9a90679ba 100644 --- a/src/saga/mod.rs +++ b/src/saga/mod.rs @@ -28,6 +28,13 @@ //! //! - **Forward**: step *n* succeeds → step *n+1* is dispatched; after the last //! step the saga is [`Completed`](SagaStatus::Completed). +//! - **Fan-out**: a step whose [`SagaDefinition::action`] returns +//! [`SagaAction::FanOut`] dispatches up to [`MAX_SAGA_FAN_OUT`] commands at +//! once, each with its own timer and a command id per `(step, member)`. It +//! succeeds when every member succeeds. It settles only after every member +//! replied or timed out; on any failure the members that applied (or timed +//! out, outcome unknown) are compensated one by one, highest index first, +//! and members that rejected are not. Late replies are ignored. //! - **Rejection**: a `failed` reply means "definitely not applied"; the saga //! compensates the previously applied steps in reverse order. //! - **Timeout**: the outcome is unknown, so the timed-out step is @@ -42,7 +49,8 @@ //! - **Idempotency**: message ids that changed state are stored in the saga //! ([`SagaOutcome::Duplicate`] on redelivery); late replies and superseded //! timers are [`SagaOutcome::Ignored`]. Command ids are deterministic -//! (`type:id:step:phase:attempt`), so participants can dedupe with consumer +//! (`type:id:step:phase:attempt`, plus `:member` on fan-out steps), so +//! participants can dedupe with consumer //! inbox receipts keyed by the command id, and reply ids derive from them. //! - **Concurrency**: the saga stream's optimistic version is the fence; //! [`SagaRunner`] reloads and retries on `ConcurrentWrite`. @@ -62,13 +70,13 @@ //! ]; //! STEPS //! } -//! fn action(step: &str, id: &str, d: &TransferData) -> Result { -//! match step { -//! "debit" => SagaCommand::json(&d.from_cell, "wallet.debit", &Move::new(id, d)), -//! _ => SagaCommand::json(&d.to_cell, "wallet.credit", &Move::new(id, d)), -//! } +//! fn action(step: &str, id: &str, d: &TransferData) -> Result { +//! Ok(match step { +//! "debit" => SagaCommand::json(&d.from_cell, "wallet.debit", &Move::new(id, d))?.into(), +//! _ => SagaCommand::json(&d.to_cell, "wallet.credit", &Move::new(id, d))?.into(), +//! }) //! } -//! fn compensation(step: &str, id: &str, d: &TransferData) +//! fn compensation(step: &str, member: usize, id: &str, d: &TransferData) //! -> Result, SagaError> { /* refund / reverse */ } //! } //! @@ -81,15 +89,20 @@ mod definition; mod error; +#[cfg(test)] +mod fan_out_tests; mod message; mod runner; mod state; -pub use definition::{SagaCommand, SagaDefinition, SagaStep}; +pub use definition::{SagaAction, SagaCommand, SagaDefinition, SagaStep, MAX_SAGA_FAN_OUT}; pub use error::SagaError; pub use message::{ - SagaMessage, SagaPhase, SagaReply, SagaSignal, META_ATTEMPT, META_COMMAND_ID, META_OUTCOME, - META_PHASE, META_REPLY_TO, META_SAGA_ID, META_SAGA_TYPE, META_STEP, SAGA_REPLY, SAGA_TIMEOUT, + SagaMessage, SagaPhase, SagaReply, SagaSignal, META_ATTEMPT, META_COMMAND_ID, META_MEMBER, + META_OUTCOME, META_PHASE, META_REPLY_TO, META_SAGA_ID, META_SAGA_TYPE, META_STEP, SAGA_REPLY, + SAGA_TIMEOUT, }; pub use runner::SagaRunner; -pub use state::{Saga, SagaCursor, SagaFailure, SagaOutcome, SagaStatus, SagaTransition}; +pub use state::{ + Saga, SagaCursor, SagaFailure, SagaFanOut, SagaOutcome, SagaStatus, SagaTransition, +}; diff --git a/src/saga/state.rs b/src/saga/state.rs index 23887dcbe..5053e6b98 100644 --- a/src/saga/state.rs +++ b/src/saga/state.rs @@ -1,6 +1,6 @@ //! The [`Saga`] aggregate: an event-sourced process manager. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::marker::PhantomData; use std::time::SystemTime; @@ -8,8 +8,8 @@ use std::time::SystemTime; use serde::{Deserialize, Serialize}; use super::message::{ - SagaMessage, SagaPhase, SagaSignal, META_ATTEMPT, META_COMMAND_ID, META_PHASE, META_REPLY_TO, - META_SAGA_ID, META_SAGA_TYPE, META_STEP, SAGA_TIMEOUT, + SagaMessage, SagaPhase, SagaSignal, META_ATTEMPT, META_COMMAND_ID, META_MEMBER, META_PHASE, + META_REPLY_TO, META_SAGA_ID, META_SAGA_TYPE, META_STEP, SAGA_TIMEOUT, }; use super::{SagaDefinition, SagaError}; use crate::aggregate::Aggregate; @@ -52,13 +52,52 @@ pub struct SagaFailure { pub timed_out: bool, } -/// The command currently awaited. +/// The command (or fan-out) currently awaited. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SagaCursor { pub step: u32, pub phase: SagaPhase, pub attempt: u32, pub deadline: SystemTime, + /// Members dispatched by a fan-out action (`0` for a single command). + pub fan_out: u32, + /// Fan-out member a compensation undoes (`None` for single steps). + pub member: Option, +} + +/// Settlement of the fan-out step currently awaited. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SagaFanOut { + pub members: u32, + pub succeeded: BTreeSet, + pub rejected: BTreeMap, + pub timed_out: BTreeSet, +} + +impl SagaFanOut { + fn new(members: u32) -> Self { + Self { + members, + ..Self::default() + } + } + + /// Whether `member` has replied or timed out. + pub fn is_settled(&self, member: u32) -> bool { + self.succeeded.contains(&member) + || self.rejected.contains_key(&member) + || self.timed_out.contains(&member) + } + + /// Whether every member has replied or timed out. + pub fn all_settled(&self) -> bool { + (self.succeeded.len() + self.rejected.len() + self.timed_out.len()) as u32 >= self.members + } + + /// Members that applied or may have (timed out): what compensation undoes. + pub fn maybe_applied(&self) -> BTreeSet { + self.succeeded.union(&self.timed_out).copied().collect() + } } /// Persisted saga events. Event names are `saga.`; payloads are the @@ -84,10 +123,12 @@ enum SagaEvent { }, CompensationSucceeded { step: u32, + member: Option, message_id: String, }, CompensationFailed { step: u32, + member: Option, message_id: String, reason: String, }, @@ -96,6 +137,22 @@ enum SagaEvent { Stuck { reason: String, }, + MemberSucceeded { + step: u32, + member: u32, + message_id: String, + output: Vec, + }, + MemberFailed { + step: u32, + member: u32, + message_id: String, + reason: String, + timed_out: bool, + }, + FanOutSettled { + step: u32, + }, } impl SagaEvent { @@ -110,6 +167,9 @@ impl SagaEvent { SagaEvent::Completed => "saga.completed", SagaEvent::Compensated => "saga.compensated", SagaEvent::Stuck { .. } => "saga.stuck", + SagaEvent::MemberSucceeded { .. } => "saga.member_succeeded", + SagaEvent::MemberFailed { .. } => "saga.member_failed", + SagaEvent::FanOutSettled { .. } => "saga.fan_out_settled", } } } @@ -162,6 +222,10 @@ pub struct Saga { /// Steps that took (or may have taken) effect, oldest first — the /// compensation stack. applied: Vec, + /// For fan-out steps on the stack: the members still to compensate. + applied_members: BTreeMap>, + /// Settlement of the fan-out action currently awaited. + fan_out: Option, failure: Option, processed: BTreeSet, _definition: PhantomData D>, @@ -175,6 +239,8 @@ impl Default for Saga { status: SagaStatus::NotStarted, cursor: None, applied: Vec::new(), + applied_members: BTreeMap::new(), + fan_out: None, failure: None, processed: BTreeSet::new(), _definition: PhantomData, @@ -190,6 +256,8 @@ impl fmt::Debug for Saga { .field("status", &self.status) .field("cursor", &self.cursor) .field("applied", &self.applied) + .field("applied_members", &self.applied_members) + .field("fan_out", &self.fan_out) .field("failure", &self.failure) .finish() } @@ -252,6 +320,16 @@ impl Saga { .collect() } + /// Settlement of the fan-out action currently awaited, if any. + pub fn fan_out(&self) -> Option<&SagaFanOut> { + self.fan_out.as_ref() + } + + /// Members of fan-out step `step` still on the compensation stack. + pub fn applied_members(&self, step: &str) -> Option<&BTreeSet> { + step_index::(step).and_then(|index| self.applied_members.get(&index)) + } + /// Whether `message_id` has already been processed. pub fn has_processed(&self, message_id: &str) -> bool { self.processed.contains(message_id) @@ -327,6 +405,14 @@ impl Saga { let current_attempt = message.attempt == cursor.attempt; let mut messages = Vec::new(); let id = message.message_id.clone(); + if cursor.phase == SagaPhase::Action && cursor.fan_out > 0 { + return self.handle_member(step, &cursor, message, now); + } + if message.member != cursor.member { + // A reply for another member's compensation, or a fan-out reply + // delivered to a single-command step. + return Ok(SagaTransition::unchanged(SagaOutcome::Ignored)); + } match (cursor.phase, &message.signal) { (SagaPhase::Action, SagaSignal::Succeeded { output }) => { self.record(SagaEvent::StepSucceeded { @@ -357,6 +443,7 @@ impl Saga { (SagaPhase::Compensation, SagaSignal::Succeeded { .. }) => { self.record(SagaEvent::CompensationSucceeded { step, + member: cursor.member, message_id: id, })?; self.compensate_next(now, &mut messages)?; @@ -377,6 +464,78 @@ impl Saga { }) } + /// A reply or timer for one member of the fan-out action in flight. + fn handle_member( + &mut self, + step: u32, + cursor: &SagaCursor, + message: &SagaMessage, + now: SystemTime, + ) -> Result { + let Some(member) = message.member else { + return Err(SagaError::Malformed(format!( + "reply to fan-out step `{}` has no member", + message.step + ))); + }; + if member >= cursor.fan_out { + return Err(SagaError::Malformed(format!( + "fan-out step `{}` has no member {member}", + message.step + ))); + } + if message.attempt != cursor.attempt + || self + .fan_out + .as_ref() + .is_none_or(|fan_out| fan_out.is_settled(member)) + { + // A late reply after this member's timeout, or a superseded timer. + return Ok(SagaTransition::unchanged(SagaOutcome::Ignored)); + } + let message_id = message.message_id.clone(); + match &message.signal { + SagaSignal::Succeeded { output } => self.record(SagaEvent::MemberSucceeded { + step, + member, + message_id, + output: output.clone(), + })?, + SagaSignal::Failed { reason } => self.record(SagaEvent::MemberFailed { + step, + member, + message_id, + reason: reason.clone(), + timed_out: false, + })?, + SagaSignal::TimedOut => self.record(SagaEvent::MemberFailed { + step, + member, + message_id, + reason: format!("member {member} of step `{}` timed out", message.step), + timed_out: true, + })?, + } + let mut messages = Vec::new(); + let settled = self + .fan_out + .as_ref() + .filter(|fan_out| fan_out.all_settled()) + .cloned(); + if let Some(fan_out) = settled { + self.record(SagaEvent::FanOutSettled { step })?; + if fan_out.succeeded.len() as u32 == fan_out.members { + self.advance_from(step as usize + 1, now, &mut messages)?; + } else { + self.compensate_next(now, &mut messages)?; + } + } + Ok(SagaTransition { + outcome: SagaOutcome::Applied, + messages, + }) + } + // --------------------------------------------------------------------- // Transitions // --------------------------------------------------------------------- @@ -391,11 +550,25 @@ impl Saga { let Some(step) = D::steps().get(next) else { return self.record(SagaEvent::Completed); }; - let command = D::action(step.name, self.id(), &self.data)?; - self.dispatch(next as u32, SagaPhase::Action, 1, command, now, messages) + let (commands, fan_out) = D::action(step.name, self.id(), &self.data)?.into_commands()?; + let commands = if fan_out { + commands + .into_iter() + .enumerate() + .map(|(member, command)| (Some(member as u32), command)) + .collect() + } else { + commands + .into_iter() + .map(|command| (None, command)) + .collect() + }; + self.dispatch(next as u32, SagaPhase::Action, 1, commands, now, messages) } /// Dispatch the next compensation from the stack, or finish compensating. + /// Fan-out steps compensate their applied members one at a time, highest + /// index first. fn compensate_next( &mut self, now: SystemTime, @@ -403,8 +576,32 @@ impl Saga { ) -> Result<(), SagaError> { for &index in self.applied.clone().iter().rev() { let step = &D::steps()[index as usize]; - if let Some(command) = D::compensation(step.name, self.id(), &self.data)? { - return self.dispatch(index, SagaPhase::Compensation, 1, command, now, messages); + if let Some(members) = self.applied_members.get(&index).cloned() { + for &member in members.iter().rev() { + if let Some(command) = + D::compensation(step.name, member as usize, self.id(), &self.data)? + { + return self.dispatch( + index, + SagaPhase::Compensation, + 1, + vec![(Some(member), command)], + now, + messages, + ); + } + } + continue; + } + if let Some(command) = D::compensation(step.name, 0, self.id(), &self.data)? { + return self.dispatch( + index, + SagaPhase::Compensation, + 1, + vec![(None, command)], + now, + messages, + ); } } self.record(SagaEvent::Compensated) @@ -421,6 +618,7 @@ impl Saga { ) -> Result<(), SagaError> { self.record(SagaEvent::CompensationFailed { step, + member: cursor.member, message_id, reason: reason.clone(), })?; @@ -428,26 +626,29 @@ impl Saga { return self.record(SagaEvent::Stuck { reason }); } let name = D::steps()[step as usize].name; - let command = D::compensation(name, self.id(), &self.data)?.ok_or_else(|| { + let member = cursor.member.unwrap_or(0) as usize; + let command = D::compensation(name, member, self.id(), &self.data)?.ok_or_else(|| { SagaError::Definition(format!("compensation for `{name}` disappeared")) })?; self.dispatch( step, SagaPhase::Compensation, cursor.attempt + 1, - command, + vec![(cursor.member, command)], now, messages, ) } - /// Record a dispatch and emit the command plus its durable timeout timer. + /// Record a dispatch and emit each command plus its durable timeout timer. + /// Fan-out members (and member compensations) carry their index in the + /// command id (`…:{attempt}:{member}`) and in `saga-member` metadata. fn dispatch( &mut self, step: u32, phase: SagaPhase, attempt: u32, - command: super::SagaCommand, + commands: Vec<(Option, super::SagaCommand)>, now: SystemTime, messages: &mut Vec, ) -> Result<(), SagaError> { @@ -459,7 +660,7 @@ impl Saga { let deadline = now .checked_add(timeout) .ok_or_else(|| SagaError::Definition("saga step timeout overflows".into()))?; - let command_id = format!( + let base_id = format!( "{}:{}:{}:{}:{}", D::SAGA_TYPE, self.id(), @@ -476,27 +677,49 @@ impl Saga { (META_ATTEMPT, attempt.to_string()), (META_REPLY_TO, D::REPLY_TO.to_string()), ]; + let fan_out = match phase { + SagaPhase::Action if commands.iter().any(|(member, _)| member.is_some()) => { + commands.len() as u32 + } + _ => 0, + }; + let member = match phase { + SagaPhase::Compensation => commands.first().and_then(|(member, _)| *member), + SagaPhase::Action => None, + }; - let mut outbound = OutboxMessage::create_to( - command_id.clone(), - command.command_type, - command.destination, - command.payload, - )?; - let mut timer = OutboxMessage::create_to( - format!("{command_id}:timeout"), - SAGA_TIMEOUT, - D::REPLY_TO, - Vec::new(), - )? - .deliver_at(deadline); - for message in [&mut outbound, &mut timer] { - for (key, value) in &metadata { - message.set_meta(*key, value.clone()); + let mut outgoing = Vec::with_capacity(commands.len() * 2); + for (member, command) in commands { + let command_id = match member { + Some(member) => format!("{base_id}:{member}"), + None => base_id.clone(), + }; + let mut outbound = OutboxMessage::create_to( + command_id.clone(), + command.command_type, + command.destination, + command.payload, + )?; + let mut timer = OutboxMessage::create_to( + format!("{command_id}:timeout"), + SAGA_TIMEOUT, + D::REPLY_TO, + Vec::new(), + )? + .deliver_at(deadline); + for message in [&mut outbound, &mut timer] { + for (key, value) in &metadata { + message.set_meta(*key, value.clone()); + } + if let Some(member) = member { + message.set_meta(META_MEMBER, member.to_string()); + } + message.set_correlation_id(saga_id.clone()); } - message.set_correlation_id(saga_id.clone()); + timer.set_meta(META_COMMAND_ID, command_id); + outgoing.push(outbound); + outgoing.push(timer); } - timer.set_meta(META_COMMAND_ID, command_id); self.record(SagaEvent::Dispatched { cursor: SagaCursor { @@ -504,10 +727,11 @@ impl Saga { phase, attempt, deadline, + fan_out, + member, }, })?; - messages.push(outbound); - messages.push(timer); + messages.extend(outgoing); Ok(()) } @@ -533,8 +757,18 @@ impl Saga { self.status = SagaStatus::Compensating; // Steps above this one had no compensation; drop them. if let Some(position) = self.applied.iter().rposition(|s| *s == cursor.step) { - self.applied.truncate(position + 1); + for dropped in self.applied.drain(position + 1..) { + self.applied_members.remove(&dropped); + } + } + // Likewise members above this one. + if let (Some(member), Some(members)) = + (cursor.member, self.applied_members.get_mut(&cursor.step)) + { + members.retain(|candidate| *candidate <= member); } + } else if cursor.fan_out > 0 { + self.fan_out = Some(SagaFanOut::new(cursor.fan_out)); } self.cursor = Some(cursor); } @@ -544,7 +778,7 @@ impl Saga { output, } => { if let Some(definition) = D::steps().get(step as usize) { - D::on_step_succeeded(definition.name, &mut self.data, &output); + D::on_step_succeeded(definition.name, 0, &mut self.data, &output); } self.applied.push(step); self.processed.insert(message_id); @@ -572,9 +806,23 @@ impl Saga { self.status = SagaStatus::Compensating; self.cursor = None; } - SagaEvent::CompensationSucceeded { step, message_id } => { - if let Some(position) = self.applied.iter().rposition(|s| *s == step) { - self.applied.truncate(position); + SagaEvent::CompensationSucceeded { + step, + member, + message_id, + } => { + let step_done = match (member, self.applied_members.get_mut(&step)) { + (Some(member), Some(members)) => { + members.remove(&member); + members.is_empty() + } + _ => true, + }; + if step_done { + self.applied_members.remove(&step); + if let Some(position) = self.applied.iter().rposition(|s| *s == step) { + self.applied.truncate(position); + } } self.processed.insert(message_id); self.cursor = None; @@ -589,6 +837,7 @@ impl Saga { SagaEvent::Compensated => { self.status = SagaStatus::Compensated; self.applied.clear(); + self.applied_members.clear(); self.cursor = None; } SagaEvent::Stuck { reason } => { @@ -598,6 +847,63 @@ impl Saga { } self.cursor = None; } + SagaEvent::MemberSucceeded { + step, + member, + message_id, + output, + } => { + if let Some(definition) = D::steps().get(step as usize) { + D::on_step_succeeded(definition.name, member as usize, &mut self.data, &output); + } + if let Some(fan_out) = &mut self.fan_out { + fan_out.succeeded.insert(member); + } + self.processed.insert(message_id); + } + SagaEvent::MemberFailed { + member, + message_id, + reason, + timed_out, + .. + } => { + if let Some(fan_out) = &mut self.fan_out { + if timed_out { + fan_out.timed_out.insert(member); + } else { + fan_out.rejected.insert(member, reason); + } + } + self.processed.insert(message_id); + } + SagaEvent::FanOutSettled { step } => { + let fan_out = self.fan_out.take().unwrap_or_default(); + let maybe_applied = fan_out.maybe_applied(); + if !maybe_applied.is_empty() { + self.applied.push(step); + self.applied_members.insert(step, maybe_applied); + } + if fan_out.succeeded.len() as u32 != fan_out.members { + let (reason, timed_out) = match fan_out.rejected.iter().next() { + Some((member, reason)) => (format!("member {member}: {reason}"), false), + None => ( + format!("{} member(s) timed out", fan_out.timed_out.len()), + true, + ), + }; + self.failure = Some(SagaFailure { + step: D::steps() + .get(step as usize) + .map(|s| s.name.to_string()) + .unwrap_or_default(), + reason, + timed_out, + }); + self.status = SagaStatus::Compensating; + } + self.cursor = None; + } } } } diff --git a/tests/saga_transfer/main.rs b/tests/saga_transfer/main.rs index fbaf4c399..a9ef7fa8e 100644 --- a/tests/saga_transfer/main.rs +++ b/tests/saga_transfer/main.rs @@ -26,8 +26,8 @@ use distributed::saga::SagaCursor; use distributed::{ sourced, AggregateBuilder, AggregateRepository, ClaimOutboxMessages, Entity, InMemoryOutboxStore, InMemoryRepository, ManualClock, OutboxClaimRef, OutboxMessage, - OutboxStore, Saga, SagaCommand, SagaDefinition, SagaError, SagaMessage, SagaOutcome, SagaReply, - SagaRunner, SagaStatus, SagaStep, + OutboxStore, Saga, SagaAction, SagaCommand, SagaDefinition, SagaError, SagaMessage, + SagaOutcome, SagaReply, SagaRunner, SagaStatus, SagaStep, }; // =========================================================================== @@ -127,7 +127,7 @@ impl SagaDefinition for Transfer { STEPS } - fn action(step: &str, saga_id: &str, data: &TransferData) -> Result { + fn action(step: &str, saga_id: &str, data: &TransferData) -> Result { let (cell, kind, wallet) = match step { "debit" => (SOURCE_CELL, "wallet.debit", &data.from), "credit" => (TARGET_CELL, "wallet.credit", &data.to), @@ -142,10 +142,12 @@ impl SagaDefinition for Transfer { amount: data.amount, }, ) + .map(Into::into) } fn compensation( step: &str, + _member: usize, saga_id: &str, data: &TransferData, ) -> Result, SagaError> { @@ -166,7 +168,7 @@ impl SagaDefinition for Transfer { .map(Some) } - fn on_step_succeeded(step: &str, data: &mut TransferData, output: &[u8]) { + fn on_step_succeeded(step: &str, _member: usize, data: &mut TransferData, output: &[u8]) { if step == "credit" { data.credit_receipt = Some(String::from_utf8_lossy(output).into_owned()); } From 290b9444f5f02e427d0223574e1f863e2afc7144 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 10:05:24 -0500 Subject: [PATCH 09/40] ci: build every domain crate for wasm32 in a required job scripts/check-wasm32-domains.sh builds each domain crate's library for wasm32-unknown-unknown (e2e-ui todo/chat domains, the Mob Pit crawler, and game_services/crates/*-domain and kernel once they exist) plus distributed and distributed_game without features. A negative fixture with Tokio's net feature must fail, so the check provably bites. blob-domain is excluded with its reason: it mounts SQL read models. The integration-wasm32-domains workflow runs on PRs and gates the main release tag. [[tasks/gr-p30-wasm32-domain-check]] --- .github/workflows/README.md | 1 + .../workflows/integration-wasm32-domains.yaml | 33 ++++++++ .github/workflows/on-pr-quality.yaml | 3 + .../on-push-main-version-and-tag.yaml | 5 +- Cargo.toml | 2 +- scripts/check-wasm32-domains.sh | 79 +++++++++++++++++++ .../fixtures/wasm32-native-domain/Cargo.toml | 20 +++++ .../fixtures/wasm32-native-domain/src/lib.rs | 3 + 8 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/integration-wasm32-domains.yaml create mode 100755 scripts/check-wasm32-domains.sh create mode 100644 tests/fixtures/wasm32-native-domain/Cargo.toml create mode 100644 tests/fixtures/wasm32-native-domain/src/lib.rs diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 8e49cd5e1..d2136caf6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -14,6 +14,7 @@ existing unbounded-tech quality provider plus the integration/* jobs below — | [`integration-*.yaml`](./) | yes | **This repo:** broker / DB / CLI / observability / GraphQL identity+OIDC | | [`integration-e2e-ui.yaml`](./integration-e2e-ui.yaml) | yes | **This repo:** `tests/e2e-ui` offline suite, compose Playwright e2e, and kind local-GitOps Playwright e2e | | [`integration-celld.yaml`](./integration-celld.yaml) | yes | **This repo:** `tests/e2e-celld` workspace tests, celld storage and cell-alarm proofs (`celld dev`), live celld+NATS | +| [`integration-wasm32-domains.yaml`](./integration-wasm32-domains.yaml) | yes | **This repo:** every domain crate builds for `wasm32-unknown-unknown` (`scripts/check-wasm32-domains.sh`), and a native-only fixture must fail | | [`integration-js.yaml`](./integration-js.yaml) | yes | **This repo:** install, typecheck, test, build, and packed-consumer smoke test for `js/` | | [`on-pr-quality.yaml`](./on-pr-quality.yaml) | entry | **This repo** PR gate (not the consumer quality contract) | | [`on-push-main-version-and-tag.yaml`](./on-push-main-version-and-tag.yaml) | entry | **This repo** main → **vnext** tag | diff --git a/.github/workflows/integration-wasm32-domains.yaml b/.github/workflows/integration-wasm32-domains.yaml new file mode 100644 index 000000000..22659e636 --- /dev/null +++ b/.github/workflows/integration-wasm32-domains.yaml @@ -0,0 +1,33 @@ +name: wasm32 domain crates + +# Reusable workflow: referenced via `uses: ./.github/workflows/integration-wasm32-domains.yaml`. +# Domain crates run inside celld cells (wasm32 Durable Objects), so each must +# build for wasm32-unknown-unknown without Tokio sockets, native SQLite or +# other native-only dependencies (game-services decision D4). The script also +# proves the check bites: tests/fixtures/wasm32-native-domain must fail. +# +# Local parity: scripts/check-wasm32-domains.sh +on: + workflow_call: + +jobs: + wasm32-domains: + name: Domain crates build for wasm32-unknown-unknown + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + with: + shared-key: wasm32-domains + cache-directories: target/wasm32-domains + - name: Build every domain crate for wasm32 (and reject the native fixture) + run: scripts/check-wasm32-domains.sh diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index 5d3906655..ecc421b3f 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -82,3 +82,6 @@ jobs: celld: uses: ./.github/workflows/integration-celld.yaml + + wasm32-domains: + uses: ./.github/workflows/integration-wasm32-domains.yaml diff --git a/.github/workflows/on-push-main-version-and-tag.yaml b/.github/workflows/on-push-main-version-and-tag.yaml index a04f3a99c..303b92ed4 100644 --- a/.github/workflows/on-push-main-version-and-tag.yaml +++ b/.github/workflows/on-push-main-version-and-tag.yaml @@ -52,11 +52,14 @@ jobs: celld: uses: ./.github/workflows/integration-celld.yaml + wasm32-domains: + uses: ./.github/workflows/integration-wasm32-domains.yaml + # This uses commit logs and tags from git to determine the next version number and create a tag for the release. # Some commits such as chore: will not trigger a version bump and tag; this is by design. version-and-tag: name: Version and Tag - needs: [quality, all-features, postgres, nats, rabbitmq, kafka, distributed-cli, observability, graphql, e2e-ui, js-client, celld] + needs: [quality, all-features, postgres, nats, rabbitmq, kafka, distributed-cli, observability, graphql, e2e-ui, js-client, celld, wasm32-domains] uses: unbounded-tech/workflow-vnext-tag/.github/workflows/workflow.yaml@v1.22.2 secrets: DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} diff --git a/Cargo.toml b/Cargo.toml index 2c1e670d7..44d730698 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = ["distributed_macros", "distributed_cli", "distributed_game", "distributed_game_macros"] -exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "examples/game/server"] +exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"] resolver = "2" [workspace.package] diff --git a/scripts/check-wasm32-domains.sh b/scripts/check-wasm32-domains.sh new file mode 100755 index 000000000..bd35cf117 --- /dev/null +++ b/scripts/check-wasm32-domains.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# Domain crates must compile to wasm32-unknown-unknown (game-services D4): +# celld runs them as Durable Objects, so no Tokio runtime, sockets, native +# SQLite or other native-only dependency may reach a domain crate. +# +# Builds every domain crate's library for wasm32 with the dependency features +# it declares (domain crates depend on `distributed` with +# `default-features = false`), then proves the check bites: the negative +# fixture tests/fixtures/wasm32-native-domain (Tokio `net`) must fail. +# +# scripts/check-wasm32-domains.sh # check every domain crate +# scripts/check-wasm32-domains.sh --list # print the crates it checks +# +# A domain crate is any of: +# tests/e2e-ui/crates/*-domain, examples/game/server/crawler, +# game_services/crates/*-domain, game_services/crates/kernel +# plus the framework crates domains depend on: distributed (no features) and +# distributed_game (no features; the FactCommand derive). +# +# Excluded, with the reason (a crate here never runs in a cell): +# tests/e2e-ui/crates/blob-domain e2e-ui blob game host: its `domain` +# feature mounts SQL read models (e2e-readmodels, workspace distributed +# with sqlx/tokio). Its pure board core builds for wasm via `--features wasm`. +set -eu + +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" +target=wasm32-unknown-unknown +cargo=${CARGO:-cargo} +target_dir=${CARGO_TARGET_DIR:-$root/target/wasm32-domains} + +manifests() { + echo Cargo.toml + echo distributed_game/Cargo.toml + for manifest in tests/e2e-ui/crates/*-domain/Cargo.toml \ + examples/game/server/crawler/Cargo.toml \ + game_services/crates/*-domain/Cargo.toml \ + game_services/crates/kernel/Cargo.toml; do + case "$manifest" in + tests/e2e-ui/crates/blob-domain/Cargo.toml) continue ;; + esac + [ -f "$manifest" ] && echo "$manifest" + done + return 0 +} + +if [ "${1:-}" = "--list" ]; then + manifests + exit 0 +fi + +failed=0 +for manifest in $(manifests); do + printf '==> wasm32 %s\n' "$manifest" + if [ "$manifest" = Cargo.toml ] || [ "$manifest" = distributed_game/Cargo.toml ]; then + features="--no-default-features" + else + features="" + fi + # shellcheck disable=SC2086 # $features is intentionally word-split. + if ! "$cargo" build --quiet --lib --target "$target" --manifest-path "$manifest" \ + --target-dir "$target_dir" $features; then + printf 'FAIL: %s does not build for %s\n' "$manifest" "$target" >&2 + failed=1 + fi +done + +fixture=tests/fixtures/wasm32-native-domain/Cargo.toml +printf '==> negative fixture %s (must fail)\n' "$fixture" +if "$cargo" build --quiet --lib --target "$target" --manifest-path "$fixture" \ + --target-dir "$target_dir" >/dev/null 2>&1; then + printf 'FAIL: %s built for %s; the check no longer detects native-only dependencies\n' \ + "$fixture" "$target" >&2 + failed=1 +else + printf 'ok: native-only dependency rejected\n' +fi + +exit "$failed" diff --git a/tests/fixtures/wasm32-native-domain/Cargo.toml b/tests/fixtures/wasm32-native-domain/Cargo.toml new file mode 100644 index 000000000..07e361a3d --- /dev/null +++ b/tests/fixtures/wasm32-native-domain/Cargo.toml @@ -0,0 +1,20 @@ +# Negative fixture for scripts/check-wasm32-domains.sh: a "domain" crate that +# pulls a native-only dependency (Tokio's `net`). Building it for +# wasm32-unknown-unknown must fail, which proves the domain check catches +# native-only dependencies. Never depend on this crate. +[workspace] +members = ["."] +resolver = "2" + +[package] +name = "wasm32-native-domain" +version = "0.0.0" +edition = "2021" +publish = false + +[lib] +path = "src/lib.rs" + +[dependencies] +distributed = { path = "../../..", default-features = false } +tokio = { version = "1", default-features = false, features = ["net"] } diff --git a/tests/fixtures/wasm32-native-domain/src/lib.rs b/tests/fixtures/wasm32-native-domain/src/lib.rs new file mode 100644 index 000000000..4c1238332 --- /dev/null +++ b/tests/fixtures/wasm32-native-domain/src/lib.rs @@ -0,0 +1,3 @@ +//! Must not build for wasm32-unknown-unknown (see Cargo.toml). + +pub use tokio::net::TcpListener; From 8898386202a00f09037c0153d485383e193ab08a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 10:38:09 -0500 Subject: [PATCH 10/40] feat(graphql): leaderboard roots, @live boards and TS client LeaderboardSurface declares a Redis/in-memory board (or board family) with per-role grants. GraphqlEngineBuilder::leaderboard serves its roots: {name}Leaderboard(first), {name}LeaderboardAround(radius), {name}MyRank and, only with rank_any_member, {name}Rank(member). Viewer roots bind the member from the verified session claim, never an argument. Top, around and my-rank are also @live subscriptions, refreshed through GraphqlEngine::leaderboard_change_listener registered on the LeaderboardProjector (applied changes and promotions; duplicates are silent). In protocol mode results carry an incomplete snapshot and live frames use snapshot mode. The declaration feeds the SDL, the authorization fingerprint, the client manifest (optional leaderboards section, omitted when empty so other fingerprints are unchanged) and the CLI's new leaderboards.ts. CI runs the leaderboard conformance suite and the Redis-backed GraphQL test against the game-demos Redis service. [[tasks/gr-p29-leaderboard-graphql]] [[tasks/gr-p9-redis]] --- .github/workflows/integration-game-demos.yaml | 7 +- .../client_compiler/manifest/leaderboards.rs | 147 +++++ .../src/client_compiler/manifest/mod.rs | 2 + .../manifest/model_validation.rs | 8 +- .../src/client_compiler/manifest/parse.rs | 14 +- .../src/client_compiler/manifest/types.rs | 1 + .../client_compiler/render/leaderboards.rs | 147 +++++ .../src/client_compiler/render/mod.rs | 1 + .../src/client_compiler/render/project.rs | 19 +- distributed_cli/src/client_compiler/tests.rs | 145 +++++ .../tests/fixtures/generated-leaderboards.ts | 50 ++ docs/live-query-delivery.md | 30 + src/graphql/client_manifest/build.rs | 54 +- src/graphql/client_manifest/mod.rs | 38 +- src/graphql/client_manifest/types.rs | 45 ++ src/graphql/engine/auth.rs | 34 + src/graphql/engine/builder.rs | 49 +- src/graphql/engine/core.rs | 6 + src/graphql/leaderboard_roots.rs | 465 ++++++++++++++ src/graphql/mod.rs | 2 + src/graphql/projection_delta/vector_tests.rs | 1 + src/graphql/schema.rs | 23 +- src/graphql/sdl.rs | 62 +- src/graphql/surface/application.rs | 12 + src/graphql/surface/build.rs | 1 + src/graphql/surface/leaderboard.rs | 585 ++++++++++++++++++ src/graphql/surface/mod.rs | 7 + src/graphql/surface/types.rs | 54 +- src/leaderboard/mod.rs | 13 +- src/leaderboard/projector.rs | 63 +- tests/graphql_leaderboard/main.rs | 482 +++++++++++++++ tests/redis_leaderboard/main.rs | 95 +++ 32 files changed, 2624 insertions(+), 38 deletions(-) create mode 100644 distributed_cli/src/client_compiler/manifest/leaderboards.rs create mode 100644 distributed_cli/src/client_compiler/render/leaderboards.rs create mode 100644 distributed_cli/tests/fixtures/generated-leaderboards.ts create mode 100644 src/graphql/leaderboard_roots.rs create mode 100644 src/graphql/surface/leaderboard.rs create mode 100644 tests/graphql_leaderboard/main.rs diff --git a/.github/workflows/integration-game-demos.yaml b/.github/workflows/integration-game-demos.yaml index 316b4d6ef..7c9aec875 100644 --- a/.github/workflows/integration-game-demos.yaml +++ b/.github/workflows/integration-game-demos.yaml @@ -17,7 +17,8 @@ jobs: runs-on: ubuntu-latest env: CARGO_TERM_COLOR: always - # The shared ticket rate limiter's Lua script runs against a real Redis. + # The shared ticket rate limiter's Lua script and the leaderboard store + # (conformance suite, projector, GraphQL boards) run against a real Redis. services: redis: image: redis:7-alpine @@ -48,6 +49,10 @@ jobs: run: npm ci --no-audit --no-fund && npx playwright install --with-deps chromium - name: Run examples/game smoke run: examples/game/run.sh smoke + - name: Leaderboard conformance and GraphQL boards against real Redis + env: + REDIS_URL: redis://127.0.0.1:6379 + run: cargo test --features redis,graphql,sqlite --test redis_leaderboard -- --nocapture - name: Ticket rate limiter against real Redis working-directory: js env: diff --git a/distributed_cli/src/client_compiler/manifest/leaderboards.rs b/distributed_cli/src/client_compiler/manifest/leaderboards.rs new file mode 100644 index 000000000..2591ddd6d --- /dev/null +++ b/distributed_cli/src/client_compiler/manifest/leaderboards.rs @@ -0,0 +1,147 @@ +//! Leaderboard section of the client manifest: server-ranked board queries +//! (`top`, `around_me`, `my_rank`, `rank`) generated into `leaderboards.ts`. +//! They are not read-model roots and never enter the replica cache. + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +use super::*; + +/// The only row type the framework emits for leaderboards. +pub(crate) const LEADERBOARD_ENTRY_TYPENAME: &str = "LeaderboardEntry"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManifestLeaderboard { + pub(crate) name: String, + pub(crate) entry_typename: String, + pub(crate) order: String, + pub(crate) boards: String, + pub(crate) default_first: u32, + pub(crate) max_first: u32, + pub(crate) max_radius: u32, + pub(crate) roots: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManifestLeaderboardRoot { + pub(crate) name: String, + pub(crate) operation: String, + pub(crate) list: bool, + pub(crate) live: bool, + pub(crate) arguments: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManifestLeaderboardArgument { + pub(crate) name: String, + pub(crate) type_name: String, + pub(crate) nullable: bool, +} + +fn invalid(message: impl Into) -> ClientCompileError { + ClientCompileError::manifest("client.manifest.leaderboard", message) +} + +/// Validate the leaderboard section against itself and the read-model roots. +pub(crate) fn validate_leaderboards( + leaderboards: &[ManifestLeaderboard], + roots: &std::collections::BTreeMap<(RootOperation, String), ManifestRoot>, + typenames: &BTreeSet, +) -> Result<(), ClientCompileError> { + if !leaderboards.is_empty() && typenames.contains(LEADERBOARD_ENTRY_TYPENAME) { + return Err(invalid(format!( + "model typename `{LEADERBOARD_ENTRY_TYPENAME}` collides with the leaderboard entry type" + ))); + } + let mut names = BTreeSet::new(); + let mut root_names = roots + .keys() + .map(|(_, name)| name.clone()) + .collect::>(); + for leaderboard in leaderboards { + if !is_graphql_name(&leaderboard.name) || !names.insert(leaderboard.name.clone()) { + return Err(invalid(format!( + "invalid or duplicate leaderboard `{}`", + leaderboard.name + ))); + } + if leaderboard.entry_typename != LEADERBOARD_ENTRY_TYPENAME { + return Err(invalid(format!( + "leaderboard `{}` entry type must be `{LEADERBOARD_ENTRY_TYPENAME}`", + leaderboard.name + ))); + } + if !matches!(leaderboard.order.as_str(), "high_first" | "low_first") + || !matches!(leaderboard.boards.as_str(), "single" | "family") + { + return Err(invalid(format!( + "leaderboard `{}` has an unknown order or board kind", + leaderboard.name + ))); + } + if leaderboard.default_first == 0 + || leaderboard.default_first > leaderboard.max_first + || leaderboard.max_radius == 0 + { + return Err(invalid(format!( + "leaderboard `{}` has inconsistent limits", + leaderboard.name + ))); + } + if leaderboard.roots.is_empty() { + return Err(invalid(format!( + "leaderboard `{}` exposes no roots", + leaderboard.name + ))); + } + let family = leaderboard.boards == "family"; + for root in &leaderboard.roots { + if !is_graphql_name(&root.name) || !root_names.insert(root.name.clone()) { + return Err(invalid(format!( + "leaderboard root `{}` is invalid or collides with another root", + root.name + ))); + } + let (list, live, extra): (bool, bool, Option<(&str, &str, bool)>) = + match root.operation.as_str() { + "top" => (true, root.live, Some(("first", "Int", true))), + "around_me" => (true, root.live, Some(("radius", "Int", true))), + "my_rank" => (false, root.live, None), + "rank" => (false, false, Some(("member", "String", false))), + other => { + return Err(invalid(format!( + "leaderboard root `{}` has unknown operation `{other}`", + root.name + ))) + } + }; + let mut expected = Vec::new(); + if family { + expected.push(("board", "String", false)); + } + expected.extend(extra); + let actual = root + .arguments + .iter() + .map(|argument| { + ( + argument.name.as_str(), + argument.type_name.as_str(), + argument.nullable, + ) + }) + .collect::>(); + if root.list != list || root.live != live || actual != expected { + return Err(invalid(format!( + "leaderboard root `{}` does not match its `{}` contract", + root.name, root.operation + ))); + } + } + } + Ok(()) +} diff --git a/distributed_cli/src/client_compiler/manifest/mod.rs b/distributed_cli/src/client_compiler/manifest/mod.rs index 5eac2d648..c29095b25 100644 --- a/distributed_cli/src/client_compiler/manifest/mod.rs +++ b/distributed_cli/src/client_compiler/manifest/mod.rs @@ -1,4 +1,5 @@ mod commands; +mod leaderboards; mod model_validation; mod parse; mod projections; @@ -9,6 +10,7 @@ mod util; pub(crate) use super::{is_graphql_name, ClientCompileError, ClientSurfaceSelector}; pub(crate) use commands::*; +pub(crate) use leaderboards::*; pub(crate) use model_validation::*; pub(crate) use projections::*; pub(crate) use projectors::*; diff --git a/distributed_cli/src/client_compiler/manifest/model_validation.rs b/distributed_cli/src/client_compiler/manifest/model_validation.rs index 7c5a58f97..4c34f4847 100644 --- a/distributed_cli/src/client_compiler/manifest/model_validation.rs +++ b/distributed_cli/src/client_compiler/manifest/model_validation.rs @@ -273,10 +273,12 @@ pub(crate) fn validate_derived_capabilities( capabilities: &ManifestCapabilities, roots: &BTreeMap<(RootOperation, String), ManifestRoot>, commands: &[ManifestCommand], + live_leaderboards: bool, ) -> Result<(), ClientCompileError> { - let has_live_roots = roots - .keys() - .any(|(operation, _)| *operation == RootOperation::Subscription); + let has_live_roots = live_leaderboards + || roots + .keys() + .any(|(operation, _)| *operation == RootOperation::Subscription); if capabilities.live_queries != has_live_roots { return Err(ClientCompileError::manifest( "client.manifest.live_capability", diff --git a/distributed_cli/src/client_compiler/manifest/parse.rs b/distributed_cli/src/client_compiler/manifest/parse.rs index c9651b271..ba0c5ad7a 100644 --- a/distributed_cli/src/client_compiler/manifest/parse.rs +++ b/distributed_cli/src/client_compiler/manifest/parse.rs @@ -27,6 +27,9 @@ struct ManifestWire { projectors: Vec, projection_programs: Vec, projection_bindings: Vec, + /// Absent on manifests without leaderboards (fingerprints unchanged). + #[serde(default)] + leaderboards: Vec, } #[derive(Serialize)] @@ -45,6 +48,8 @@ struct ManifestSchemaMaterial<'a> { projectors: &'a [ManifestProjector], projection_programs: &'a [ManifestProjectionProgram], projection_bindings: &'a [ManifestProjectionBinding], + #[serde(skip_serializing_if = "<[ManifestLeaderboard]>::is_empty")] + leaderboards: &'a [ManifestLeaderboard], } impl ClientManifest { @@ -205,7 +210,12 @@ impl ClientManifest { &projectors, )?); let trusted_presets = derive_trusted_preset_descriptors(&models, &commands)?; - validate_derived_capabilities(&wire.capabilities, &roots, &commands)?; + validate_leaderboards(&wire.leaderboards, &roots, &typenames)?; + let live_leaderboards = wire + .leaderboards + .iter() + .any(|leaderboard| leaderboard.roots.iter().any(|root| root.live)); + validate_derived_capabilities(&wire.capabilities, &roots, &commands, live_leaderboards)?; if wire.schema_fingerprint != computed_schema_fingerprint { return Err(ClientCompileError::manifest( "client.manifest.schema_fingerprint", @@ -233,6 +243,7 @@ impl ClientManifest { projectors, projection_programs, projection_bindings, + leaderboards: wire.leaderboards, }) } @@ -257,6 +268,7 @@ fn schema_fingerprint(wire: &ManifestWire) -> Result projectors: &wire.projectors, projection_programs: &wire.projection_programs, projection_bindings: &wire.projection_bindings, + leaderboards: &wire.leaderboards, }; serde_json::to_vec(&material) .map(|bytes| hash_bytes(&bytes)) diff --git a/distributed_cli/src/client_compiler/manifest/types.rs b/distributed_cli/src/client_compiler/manifest/types.rs index a8c9b64b3..1238bf828 100644 --- a/distributed_cli/src/client_compiler/manifest/types.rs +++ b/distributed_cli/src/client_compiler/manifest/types.rs @@ -25,6 +25,7 @@ pub(crate) struct ClientManifest { pub(crate) projectors: Vec, pub(crate) projection_programs: Vec, pub(crate) projection_bindings: Vec, + pub(crate) leaderboards: Vec, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/distributed_cli/src/client_compiler/render/leaderboards.rs b/distributed_cli/src/client_compiler/render/leaderboards.rs new file mode 100644 index 000000000..86950c5d1 --- /dev/null +++ b/distributed_cli/src/client_compiler/render/leaderboards.rs @@ -0,0 +1,147 @@ +//! `leaderboards.ts`: typed documents for the manifest's leaderboard roots. +//! +//! Leaderboard results are ranked by the server (Redis sorted sets) and are +//! not normalized read-model rows, so they are plain GraphQL operations with +//! exact documents, variable and data types, and an `@live` subscription +//! document where the surface has one. They never enter the replica cache. + +use super::common::json_string; +use crate::client_compiler::manifest::{ + ClientManifest, ManifestLeaderboard, ManifestLeaderboardRoot, LEADERBOARD_ENTRY_TYPENAME, +}; +use crate::client_compiler::ClientCompileError; + +/// `None` when the manifest has no leaderboards. +pub(super) fn render_leaderboards( + manifest: &ClientManifest, +) -> Result, ClientCompileError> { + if manifest.leaderboards.is_empty() { + return Ok(None); + } + let mut out = vec![ + "/** GENERATED by distributed client. Server-ranked leaderboard queries; not replica-cached. */".to_string(), + format!( + "export interface {LEADERBOARD_ENTRY_TYPENAME} {{\n\ + \t/** Zero-based rank in the board's order. */\n\ + \treadonly rank: number;\n\ + \treadonly member: string;\n\ + \treadonly score: number;\n\ + }}" + ), + ]; + let mut index = Vec::new(); + for leaderboard in &manifest.leaderboards { + for root in &leaderboard.roots { + let (block, export) = render_root(manifest, leaderboard, root)?; + out.push(block); + index.push(format!("\t{}: {export},", json_string(&root.name)?)); + } + } + out.push(format!( + "export const LEADERBOARD_OPERATIONS = {{\n{}\n}} as const;", + index.join("\n") + )); + Ok(Some(format!("{}\n", out.join("\n\n")))) +} + +fn pascal(name: &str) -> String { + let mut chars = name.chars(); + match chars.next() { + Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), + None => String::new(), + } +} + +fn render_root( + manifest: &ClientManifest, + leaderboard: &ManifestLeaderboard, + root: &ManifestLeaderboardRoot, +) -> Result<(String, String), ClientCompileError> { + let operation = pascal(&root.name); + let variables = root + .arguments + .iter() + .map(|argument| { + let ty = if argument.type_name == "Int" { + "number" + } else { + "string" + }; + if argument.nullable { + format!("readonly {}?: {ty} | null;", argument.name) + } else { + format!("readonly {}: {ty};", argument.name) + } + }) + .collect::>(); + let definitions = root + .arguments + .iter() + .map(|argument| { + format!( + "${}: {}{}", + argument.name, + argument.type_name, + if argument.nullable { "" } else { "!" } + ) + }) + .collect::>(); + let (definitions, call) = if root.arguments.is_empty() { + (String::new(), String::new()) + } else { + ( + format!("({})", definitions.join(", ")), + format!( + "({})", + root.arguments + .iter() + .map(|argument| format!("{0}: ${0}", argument.name)) + .collect::>() + .join(", ") + ), + ) + }; + let selection = format!("{}{call} {{ rank member score }}", root.name); + let document = format!("query {operation}{definitions} {{ {selection} }}"); + let live = root + .live + .then(|| format!("subscription {operation}_Live{definitions} {{ {selection} }}")); + let data_type = if root.list { + format!("readonly {LEADERBOARD_ENTRY_TYPENAME}[]") + } else { + format!("{LEADERBOARD_ENTRY_TYPENAME} | null") + }; + let export = format!("{operation}Leaderboard"); + let block = format!( + "export type {operation}Variables = {{{}}};\n\ + export type {operation}Data = {{ readonly {}: {data_type} }};\n\ + export const {export} = {{\n\ + \tleaderboard: {},\n\ + \toperation: {},\n\ + \torder: {},\n\ + \tlimits: {{ defaultFirst: {}, maxFirst: {}, maxRadius: {} }},\n\ + \tdocument: {},\n\ + \tliveDocument: {},\n\ + \tschemaHash: {},\n\ + }} as const;", + if variables.is_empty() { + String::new() + } else { + format!(" {} ", variables.join(" ")) + }, + root.name, + json_string(&leaderboard.name)?, + json_string(&root.operation)?, + json_string(&leaderboard.order)?, + leaderboard.default_first, + leaderboard.max_first, + leaderboard.max_radius, + json_string(&document)?, + match &live { + Some(live) => json_string(live)?, + None => "null".to_string(), + }, + json_string(&manifest.schema_fingerprint)?, + ); + Ok((block, export)) +} diff --git a/distributed_cli/src/client_compiler/render/mod.rs b/distributed_cli/src/client_compiler/render/mod.rs index 598f61b70..bacfa4e9b 100644 --- a/distributed_cli/src/client_compiler/render/mod.rs +++ b/distributed_cli/src/client_compiler/render/mod.rs @@ -1,6 +1,7 @@ mod artifact; mod commands; mod common; +mod leaderboards; mod operation; mod project; mod typescript; diff --git a/distributed_cli/src/client_compiler/render/project.rs b/distributed_cli/src/client_compiler/render/project.rs index a070313e4..94f110325 100644 --- a/distributed_cli/src/client_compiler/render/project.rs +++ b/distributed_cli/src/client_compiler/render/project.rs @@ -54,6 +54,14 @@ pub(crate) fn render_project( contents: pures, }); } + let leaderboards = super::leaderboards::render_leaderboards(manifest)?; + let has_leaderboards = leaderboards.is_some(); + if let Some(leaderboards) = leaderboards { + files.push(GeneratedClientFile { + path: "leaderboards.ts".into(), + contents: leaderboards, + }); + } files.push(GeneratedClientFile { path: "protocol.ts".into(), contents: render_protocol(manifest)?, @@ -72,7 +80,7 @@ pub(crate) fn render_project( }); files.push(GeneratedClientFile { path: "index.ts".into(), - contents: render_index(&operations, has_pures), + contents: render_index(&operations, has_pures, has_leaderboards), }); files.push(GeneratedClientFile { path: "manifest.json".into(), @@ -285,7 +293,11 @@ fn render_islands( )) } -fn render_index(operations: &[CompiledOperation], has_pures: bool) -> String { +fn render_index( + operations: &[CompiledOperation], + has_pures: bool, + has_leaderboards: bool, +) -> String { let mut lines = vec![ "/** GENERATED public entrypoint. */".to_string(), "export * from './commands.js';".into(), @@ -295,6 +307,9 @@ fn render_index(operations: &[CompiledOperation], has_pures: bool) -> String { if has_pures { lines.push("export * from './pures.js';".into()); } + if has_leaderboards { + lines.push("export * from './leaderboards.js';".into()); + } for operation in operations { let module = operation .module_path diff --git a/distributed_cli/src/client_compiler/tests.rs b/distributed_cli/src/client_compiler/tests.rs index 407f832f5..ed51fa260 100644 --- a/distributed_cli/src/client_compiler/tests.rs +++ b/distributed_cli/src/client_compiler/tests.rs @@ -4482,3 +4482,148 @@ fn generated_lazy_commands_defer_definitions_and_preserve_command_signatures() { include_str!("../../tests/fixtures/generated-lazy-commands.ts") ); } + +fn leaderboard_manifest() -> JsonValue { + let mut value = manifest(); + value["leaderboards"] = json!([ + { + "name": "xp", + "entry_typename": "LeaderboardEntry", + "order": "high_first", + "boards": "single", + "default_first": 3, + "max_first": 5, + "max_radius": 2, + "roots": [ + {"name": "xpLeaderboard", "operation": "top", "list": true, "live": true, + "arguments": [{"name": "first", "type_name": "Int", "nullable": true}]}, + {"name": "xpMyRank", "operation": "my_rank", "list": false, "live": true, + "arguments": []} + ] + }, + { + "name": "season", + "entry_typename": "LeaderboardEntry", + "order": "low_first", + "boards": "family", + "default_first": 10, + "max_first": 100, + "max_radius": 25, + "roots": [ + {"name": "seasonRank", "operation": "rank", "list": false, "live": false, + "arguments": [ + {"name": "board", "type_name": "String", "nullable": false}, + {"name": "member", "type_name": "String", "nullable": false} + ]} + ] + } + ]); + super::manifest::refresh_schema_fingerprint(&mut value); + value +} + +const LEADERBOARD_PAGE: &str = r#" + query Todos($limit: Int!) { + todos(limit: $limit) { title } + } +"#; + +#[test] +fn leaderboard_roots_generate_typed_documents() { + let project = compile_client(input_with_manifest( + leaderboard_manifest(), + LEADERBOARD_PAGE, + )) + .unwrap(); + let generated = file(&project, "leaderboards.ts"); + assert!(generated.contains("export interface LeaderboardEntry {")); + assert!(generated + .contains("export type XpLeaderboardVariables = { readonly first?: number | null; };")); + assert!(generated.contains( + "export type XpLeaderboardData = { readonly xpLeaderboard: readonly LeaderboardEntry[] };" + )); + assert!(generated.contains( + r#"document: "query XpLeaderboard($first: Int) { xpLeaderboard(first: $first) { rank member score } }","# + )); + assert!(generated.contains( + r#"liveDocument: "subscription XpLeaderboard_Live($first: Int) { xpLeaderboard(first: $first) { rank member score } }","# + )); + // The viewer's rank takes no member argument: the server binds it from + // the verified session. + assert!(generated.contains("export type XpMyRankVariables = {};")); + assert!(generated.contains(r#"document: "query XpMyRank { xpMyRank { rank member score } }","#)); + assert!(generated.contains( + "export type SeasonRankVariables = { readonly board: string; readonly member: string; };" + )); + assert!(generated.contains("liveDocument: null,")); + assert!(generated.contains("\"xpLeaderboard\": XpLeaderboardLeaderboard,")); + assert!(file(&project, "index.ts").contains("export * from './leaderboards.js';")); + assert_eq!( + generated, + include_str!("../../tests/fixtures/generated-leaderboards.ts"), + "the checked-in TypeScript consumer fixture must remain byte-exact" + ); +} + +#[test] +fn manifests_without_leaderboards_generate_no_module() { + let project = compile_client(input(LEADERBOARD_PAGE)).unwrap(); + assert!(project + .files + .iter() + .all(|file| file.path != "leaderboards.ts")); + assert!(!file(&project, "index.ts").contains("leaderboards")); +} + +type ManifestMutation = Box; + +#[test] +fn malformed_leaderboard_sections_are_refused() { + let cases: Vec<(&str, ManifestMutation)> = vec![ + ( + "a member argument on my_rank", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][0]["roots"][1]["arguments"] = + json!([{"name": "member", "type_name": "String", "nullable": false}]); + }), + ), + ( + "a live rank root", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][1]["roots"][0]["live"] = json!(true); + }), + ), + ( + "an unknown operation", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][0]["roots"][0]["operation"] = json!("everyone"); + }), + ), + ( + "a root colliding with a read-model root", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][0]["roots"][0]["name"] = json!("todos"); + }), + ), + ( + "a family root without board", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][1]["roots"][0]["arguments"] = + json!([{"name": "member", "type_name": "String", "nullable": false}]); + }), + ), + ]; + for (case, mutate) in cases { + let mut value = leaderboard_manifest(); + mutate(&mut value); + super::manifest::refresh_schema_fingerprint(&mut value); + let error = compile_client(input_with_manifest(value, LEADERBOARD_PAGE)).expect_err(case); + assert_eq!(error.code, "client.manifest.leaderboard", "{case}: {error}"); + } + + // A stale fingerprint is still caught. + let mut tampered = leaderboard_manifest(); + tampered["leaderboards"][0]["max_first"] = json!(50); + let error = compile_client(input_with_manifest(tampered, LEADERBOARD_PAGE)).unwrap_err(); + assert_eq!(error.code, "client.manifest.schema_fingerprint"); +} diff --git a/distributed_cli/tests/fixtures/generated-leaderboards.ts b/distributed_cli/tests/fixtures/generated-leaderboards.ts new file mode 100644 index 000000000..15c6b5e23 --- /dev/null +++ b/distributed_cli/tests/fixtures/generated-leaderboards.ts @@ -0,0 +1,50 @@ +/** GENERATED by distributed client. Server-ranked leaderboard queries; not replica-cached. */ + +export interface LeaderboardEntry { + /** Zero-based rank in the board's order. */ + readonly rank: number; + readonly member: string; + readonly score: number; +} + +export type XpLeaderboardVariables = { readonly first?: number | null; }; +export type XpLeaderboardData = { readonly xpLeaderboard: readonly LeaderboardEntry[] }; +export const XpLeaderboardLeaderboard = { + leaderboard: "xp", + operation: "top", + order: "high_first", + limits: { defaultFirst: 3, maxFirst: 5, maxRadius: 2 }, + document: "query XpLeaderboard($first: Int) { xpLeaderboard(first: $first) { rank member score } }", + liveDocument: "subscription XpLeaderboard_Live($first: Int) { xpLeaderboard(first: $first) { rank member score } }", + schemaHash: "sha256:ef236f93542aea73780b227b1318849fa1688f015459a9b4f4c5c3a9c3752d80", +} as const; + +export type XpMyRankVariables = {}; +export type XpMyRankData = { readonly xpMyRank: LeaderboardEntry | null }; +export const XpMyRankLeaderboard = { + leaderboard: "xp", + operation: "my_rank", + order: "high_first", + limits: { defaultFirst: 3, maxFirst: 5, maxRadius: 2 }, + document: "query XpMyRank { xpMyRank { rank member score } }", + liveDocument: "subscription XpMyRank_Live { xpMyRank { rank member score } }", + schemaHash: "sha256:ef236f93542aea73780b227b1318849fa1688f015459a9b4f4c5c3a9c3752d80", +} as const; + +export type SeasonRankVariables = { readonly board: string; readonly member: string; }; +export type SeasonRankData = { readonly seasonRank: LeaderboardEntry | null }; +export const SeasonRankLeaderboard = { + leaderboard: "season", + operation: "rank", + order: "low_first", + limits: { defaultFirst: 10, maxFirst: 100, maxRadius: 25 }, + document: "query SeasonRank($board: String!, $member: String!) { seasonRank(board: $board, member: $member) { rank member score } }", + liveDocument: null, + schemaHash: "sha256:ef236f93542aea73780b227b1318849fa1688f015459a9b4f4c5c3a9c3752d80", +} as const; + +export const LEADERBOARD_OPERATIONS = { + "xpLeaderboard": XpLeaderboardLeaderboard, + "xpMyRank": XpMyRankLeaderboard, + "seasonRank": SeasonRankLeaderboard, +} as const; diff --git a/docs/live-query-delivery.md b/docs/live-query-delivery.md index aa373488d..41b3528da 100644 --- a/docs/live-query-delivery.md +++ b/docs/live-query-delivery.md @@ -102,3 +102,33 @@ projection stopped or provide a server ordering signal. This is a breaking v5 protocol change: `mode` replaces the ambiguous `supported` boolean. Upgrade server and generated-client runtime together. Old or unknown wire forms fail closed; applications do not need a polling or reload workaround. + +## Leaderboard roots + +Sorted-set leaderboards (`distributed::leaderboard`, Redis with the `redis` +feature) are exposed through the same surface by +`GraphqlEngineBuilder::leaderboard(LeaderboardSurface, store)`. For a surface +named `xp`, a role's grant (`LeaderboardGrant`) selects the roots: +`xpLeaderboard(first)`, `xpLeaderboardAround(radius)`, `xpMyRank` and, only +with `rank_any_member`, `xpRank(member)`. A board family +(`LeaderboardSurface::family`) adds `board: String!` to each. The viewer +roots take the member from the verified session claim (default `x-user-id`, +`LeaderboardSurface::member_claim`), never from an argument, and fail with +`FORBIDDEN` when the claim is absent. The same declaration feeds the static +SDL, the client manifest (`leaderboards`, omitted when empty so other +manifests and fingerprints are unchanged) and the generated +`leaderboards.ts` (typed documents and `@live` subscription documents). + +`xpLeaderboard`, `xpLeaderboardAround` and `xpMyRank` are also subscription +roots. They re-execute when the board's change key +(`leaderboard:{board}`) is published, which +`GraphqlEngine::leaderboard_change_listener` does when registered on the +`LeaderboardProjector` that writes the board (applied changes and rebuild +promotions; duplicates are silent). Pushes are hash-gated like SQL live +queries, and a failed execution is terminal. With the causal protocol on, +a leaderboard result carries an incomplete snapshot +(`recordsComplete: false`, `indexesComparable: false`) and live frames use +`snapshot` mode with `reset: true`: leaderboard rows are never normalized +into the replica cache or resumed from cursors. The listener is in-process; +a projector in another process must forward its board changes to the +engine (for example over the bus) to drive `@live` there. diff --git a/src/graphql/client_manifest/build.rs b/src/graphql/client_manifest/build.rs index 8e9de235a..e2f82cbeb 100644 --- a/src/graphql/client_manifest/build.rs +++ b/src/graphql/client_manifest/build.rs @@ -437,8 +437,12 @@ pub(super) fn client_manifest_from_surface_with_execution( let scalar_codecs = supported_scalar_codecs(); let record_evidence = query_footprint_has_record_evidence(surface); + let leaderboards = client_leaderboards(surface); + let live_leaderboards = leaderboards + .iter() + .any(|leaderboard| leaderboard.roots.iter().any(|root| root.live)); let capabilities = ClientCapabilities { - live_queries: !surface.subscription_fields.is_empty(), + live_queries: !surface.subscription_fields.is_empty() || live_leaderboards, record_revisions: record_evidence, tombstones: record_evidence, causal_receipts: !commands.is_empty(), @@ -467,6 +471,8 @@ pub(super) fn client_manifest_from_surface_with_execution( projectors: &'a [ClientProjector], projection_programs: &'a [ClientProjectionProgram], projection_bindings: &'a [ClientProjectionBinding], + #[serde(skip_serializing_if = "<[ClientLeaderboard]>::is_empty")] + leaderboards: &'a [ClientLeaderboard], } let schema_material = SchemaMaterial { manifest_version: DISTRIBUTED_CLIENT_MANIFEST_VERSION, @@ -483,6 +489,7 @@ pub(super) fn client_manifest_from_surface_with_execution( projectors: &projectors, projection_programs: &projection_programs, projection_bindings: &projection_bindings, + leaderboards: &leaderboards, }; let schema_fingerprint = hash_json(&schema_material)?; @@ -503,6 +510,7 @@ pub(super) fn client_manifest_from_surface_with_execution( projectors, projection_programs, projection_bindings, + leaderboards, }; // Validate the one exact scope-wide descriptor union while the complete // role-selected manifest is still available. The serialized manifest need @@ -510,3 +518,47 @@ pub(super) fn client_manifest_from_surface_with_execution( trusted_preset_descriptors(&manifest)?; Ok(manifest) } + +/// Leaderboard roots of the selected surface (same names and arguments as +/// the runtime schema and SDL). +fn client_leaderboards(surface: &Surface) -> Vec { + let mut out = surface + .leaderboards() + .iter() + .map(|leaderboard| ClientLeaderboard { + name: leaderboard.name().to_string(), + entry_typename: crate::graphql::surface::LEADERBOARD_ENTRY_TYPE.to_string(), + order: crate::graphql::surface::leaderboard_order_str(leaderboard.order()).to_string(), + boards: match leaderboard.boards() { + crate::graphql::surface::LeaderboardBoards::Single { .. } => "single", + crate::graphql::surface::LeaderboardBoards::Family { .. } => "family", + } + .to_string(), + default_first: leaderboard.default_first(), + max_first: leaderboard.max_first(), + max_radius: leaderboard.max_radius(), + roots: leaderboard + .roots(leaderboard.selected_grant()) + .into_iter() + .map(|root| ClientLeaderboardRoot { + live: surface.subscriptions && root.operation.is_live(), + list: root.operation.is_list(), + operation: root.operation.as_str().to_string(), + arguments: root + .arguments + .iter() + .map(|argument| ClientLeaderboardArgument { + name: argument.name.to_string(), + type_name: argument.type_name.to_string(), + nullable: argument.nullable, + }) + .collect(), + name: root.name, + }) + .collect(), + }) + .filter(|leaderboard| !leaderboard.roots.is_empty()) + .collect::>(); + out.sort_by(|a, b| a.name.cmp(&b.name)); + out +} diff --git a/src/graphql/client_manifest/mod.rs b/src/graphql/client_manifest/mod.rs index 9a5f8db20..3de2e7d68 100644 --- a/src/graphql/client_manifest/mod.rs +++ b/src/graphql/client_manifest/mod.rs @@ -59,25 +59,25 @@ pub use types::{ ClientAggregateSemantics, ClientArgument, ClientArgumentKind, ClientCapabilities, ClientCommand, ClientCommandExtensionSlots, ClientCommandPureArg, ClientCommandPureReduce, ClientCommandShape, ClientField, ClientFilterField, ClientFilterInput, - ClientFilterInputRelationship, ClientFilterSemantics, ClientKeyField, ClientModel, - ClientOrderSemantics, ClientPaginationSemantics, ClientProjectionArm, - ClientProjectionAssignment, ClientProjectionBinding, ClientProjectionBindingState, - ClientProjectionEnvelopeField, ClientProjectionEventRef, ClientProjectionExecutionClass, - ClientProjectionExpression, ClientProjectionFallback, ClientProjectionField, - ClientProjectionInvalidation, ClientProjectionKeyField, ClientProjectionMutationKind, - ClientProjectionObjectField, ClientProjectionOperation, ClientProjectionPartition, - ClientProjectionPlacement, ClientProjectionPreviewSource, ClientProjectionProgram, - ClientProjectionRelationshipEffect, ClientProjectionRelationshipEffectKind, - ClientProjectionScalarTransform, ClientProjectionTopologyIdentity, ClientProjectionValue, - ClientProjectionValueField, ClientProjectionValueType, ClientProjector, - ClientProtocolOperation, ClientProtocolOperations, ClientRelationship, - ClientRelationshipAggregate, ClientRelationshipKind, ClientRelationshipMaintenance, ClientRoot, - ClientRootKind, ClientRootOperation, ClientRowPolicy, ClientTrustedPresetDescriptor, - ClientTypeDef, ClientTypeField, CommandConfirmationsExtension, CommandConsistencyExtension, - CommandDirectProjectionExtension, CommandEffectsExtension, CommandInputDefaultsExtension, - CommandProjectionArmRef, CommandProjectionExtension, CommandProjectionPreviewOccurrence, - CommandProjectionPreviewValue, DistributedClientManifest, ModelNormalization, - RelationshipKeyMapping, ScalarCodec, + ClientFilterInputRelationship, ClientFilterSemantics, ClientKeyField, ClientLeaderboard, + ClientLeaderboardArgument, ClientLeaderboardRoot, ClientModel, ClientOrderSemantics, + ClientPaginationSemantics, ClientProjectionArm, ClientProjectionAssignment, + ClientProjectionBinding, ClientProjectionBindingState, ClientProjectionEnvelopeField, + ClientProjectionEventRef, ClientProjectionExecutionClass, ClientProjectionExpression, + ClientProjectionFallback, ClientProjectionField, ClientProjectionInvalidation, + ClientProjectionKeyField, ClientProjectionMutationKind, ClientProjectionObjectField, + ClientProjectionOperation, ClientProjectionPartition, ClientProjectionPlacement, + ClientProjectionPreviewSource, ClientProjectionProgram, ClientProjectionRelationshipEffect, + ClientProjectionRelationshipEffectKind, ClientProjectionScalarTransform, + ClientProjectionTopologyIdentity, ClientProjectionValue, ClientProjectionValueField, + ClientProjectionValueType, ClientProjector, ClientProtocolOperation, ClientProtocolOperations, + ClientRelationship, ClientRelationshipAggregate, ClientRelationshipKind, + ClientRelationshipMaintenance, ClientRoot, ClientRootKind, ClientRootOperation, + ClientRowPolicy, ClientTrustedPresetDescriptor, ClientTypeDef, ClientTypeField, + CommandConfirmationsExtension, CommandConsistencyExtension, CommandDirectProjectionExtension, + CommandEffectsExtension, CommandInputDefaultsExtension, CommandProjectionArmRef, + CommandProjectionExtension, CommandProjectionPreviewOccurrence, CommandProjectionPreviewValue, + DistributedClientManifest, ModelNormalization, RelationshipKeyMapping, ScalarCodec, }; pub(crate) use validation::trusted_preset_descriptors; diff --git a/src/graphql/client_manifest/types.rs b/src/graphql/client_manifest/types.rs index 63c957688..2504819a1 100644 --- a/src/graphql/client_manifest/types.rs +++ b/src/graphql/client_manifest/types.rs @@ -145,6 +145,51 @@ pub struct DistributedClientManifest { pub projectors: Vec, pub projection_programs: Vec, pub projection_bindings: Vec, + /// Leaderboard queries. Omitted when empty, so a manifest without + /// leaderboards (and its fingerprint) is unchanged; a compiler that does + /// not know this field refuses a manifest that has one. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub leaderboards: Vec, +} + +/// One leaderboard surface on the client contract. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClientLeaderboard { + /// GraphQL base name (`xp` → `xpLeaderboard`, …). + pub name: String, + /// Object type of each row (`LeaderboardEntry`: `rank`, `member`, `score`). + pub entry_typename: String, + /// `high_first` or `low_first`. + pub order: String, + /// `single` (no `board` argument) or `family` (`board: String!`). + pub boards: String, + pub default_first: u32, + pub max_first: u32, + pub max_radius: u32, + pub roots: Vec, +} + +/// One leaderboard root the selected roles may run. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClientLeaderboardRoot { + pub name: String, + /// `top`, `around_me`, `my_rank` or `rank`. + pub operation: String, + /// `true`: `[LeaderboardEntry!]!`; `false`: nullable `LeaderboardEntry`. + pub list: bool, + /// Whether a same-named subscription root exists (`@live`). + pub live: bool, + pub arguments: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClientLeaderboardArgument { + pub name: String, + pub type_name: String, + pub nullable: bool, } /// Framework-owned operations generated alongside application operations. diff --git a/src/graphql/engine/auth.rs b/src/graphql/engine/auth.rs index cd448a27f..d6864f035 100644 --- a/src/graphql/engine/auth.rs +++ b/src/graphql/engine/auth.rs @@ -1,8 +1,32 @@ use super::*; +/// Leaderboard grants of `roles`, as fingerprint material, plus their member +/// claims. Empty when no leaderboard is granted, so existing fingerprints are +/// unchanged. +fn leaderboard_material( + leaderboards: &[crate::graphql::surface::LeaderboardSurface], + roles: &[&str], + claim_keys: &mut BTreeSet, +) -> Vec { + let mut material = Vec::new(); + for leaderboard in leaderboards { + for role in roles { + if let Some(selected) = leaderboard.for_role(role) { + claim_keys.insert(selected.get_member_claim().to_string()); + material.push(serde_json::json!({ + "role": role, + "leaderboard": selected.canonical_value(), + })); + } + } + } + material +} + pub(crate) fn role_authorization_info( role: &str, permissions: &BTreeMap<(String, String), RoleModelPerm>, + leaderboards: &[crate::graphql::surface::LeaderboardSurface], ) -> Result<(String, Vec), GraphqlBuildError> { #[derive(Serialize)] struct PermissionMaterial<'a> { @@ -20,6 +44,8 @@ pub(crate) fn role_authorization_info( version: u32, role: &'a str, permissions: Vec>, + #[serde(skip_serializing_if = "Vec::is_empty")] + leaderboards: Vec, } let mut claim_keys = BTreeSet::from([ROLE_KEY.to_string(), USER_ID_KEY.to_string()]); @@ -45,11 +71,13 @@ pub(crate) fn role_authorization_info( aggregations: entry.permission.aggregations, }); } + let leaderboards = leaderboard_material(leaderboards, &[role], &mut claim_keys); let canonical = serde_json::to_vec(&RoleAuthorizationMaterial { domain: "distributed.graphql.authorization-surface", version: 1, role, permissions: role_permissions, + leaderboards, }) .map_err(|_| { GraphqlBuildError(format!( @@ -105,6 +133,7 @@ pub(crate) fn role_authorization_info_for_roles( privilege_key: &str, schema_roles: &[String], permissions: &BTreeMap<(String, String), RoleModelPerm>, + leaderboards: &[crate::graphql::surface::LeaderboardSurface], ) -> Result<(String, Vec), GraphqlBuildError> { #[derive(Serialize)] struct MultiPrivilegeMaterial<'a> { @@ -113,6 +142,8 @@ pub(crate) fn role_authorization_info_for_roles( privilege_key: &'a str, schema_roles: &'a [String], models: Vec<&'a str>, + #[serde(skip_serializing_if = "Vec::is_empty")] + leaderboards: Vec, } let mut claim_keys = BTreeSet::from([USER_ID_KEY.to_string()]); let mut models = BTreeSet::new(); @@ -126,12 +157,15 @@ pub(crate) fn role_authorization_info_for_roles( } } let models: Vec<&str> = models.into_iter().collect(); + let roles = schema_roles.iter().map(String::as_str).collect::>(); + let leaderboards = leaderboard_material(leaderboards, &roles, &mut claim_keys); let canonical = serde_json::to_vec(&MultiPrivilegeMaterial { domain: "distributed.graphql.authorization-surface.multi", version: 1, privilege_key, schema_roles, models, + leaderboards, }) .map_err(|_| { GraphqlBuildError(format!( diff --git a/src/graphql/engine/builder.rs b/src/graphql/engine/builder.rs index aeca975c9..f19408217 100644 --- a/src/graphql/engine/builder.rs +++ b/src/graphql/engine/builder.rs @@ -39,6 +39,7 @@ impl GraphqlEngineBuilder { // DevHeaders keeps ambient header tests/green; public scaffolds set OidcBearer (D6). identity: IdentityConfig::dev_headers(), read_stores: BTreeMap::new(), + leaderboards: Vec::new(), } } @@ -597,6 +598,21 @@ impl GraphqlEngineBuilder { self.identity = config; self } + /// Expose a sorted-set leaderboard through GraphQL (roots, `@live` + /// subscriptions, the client manifest and the generated TS client); see + /// [`LeaderboardSurface`](crate::graphql::surface::LeaderboardSurface). + /// Feed `@live` by registering + /// [`GraphqlEngine::leaderboard_change_listener`] on the projector that + /// writes `store`. + pub fn leaderboard( + mut self, + surface: crate::graphql::surface::LeaderboardSurface, + store: Arc, + ) -> Self { + self.leaderboards.push((surface, store)); + self + } + pub fn change_stream(mut self, rx: tokio::sync::broadcast::Receiver) -> Self { self.change_rx = Some(rx); self @@ -652,6 +668,18 @@ impl GraphqlEngineBuilder { // Validate permissions. let declared_roles = self.roles.clone(); + for (leaderboard, _) in &self.leaderboards { + for role in leaderboard.grants().keys() { + if let Some(roles) = &declared_roles { + if !roles.contains(role) && role != &self.anonymous_role { + return Err(GraphqlBuildError(format!( + "leaderboard `{}` grants undeclared role `{role}`", + leaderboard.name() + ))); + } + } + } + } let anonymous = self.anonymous_role.clone(); let perm_keys: Vec<_> = self.permissions.keys().cloned().collect(); for (model, role) in &perm_keys { @@ -771,6 +799,8 @@ impl GraphqlEngineBuilder { .map_err(GraphqlBuildError)? .with_service_binding(self.command_binding.clone()) .with_projection_owners(self.projectors.clone()) + .map_err(GraphqlBuildError)? + .with_leaderboards(self.leaderboards.iter().map(|(surface, _)| surface.clone())) .map_err(GraphqlBuildError)?, ); let query_protocol = if self.protocol_token_key.is_some() { @@ -791,6 +821,11 @@ impl GraphqlEngineBuilder { .transpose()?; let mut roles: BTreeSet = self.permissions.keys().map(|(_, r)| r.clone()).collect(); + roles.extend( + self.leaderboards + .iter() + .flat_map(|(leaderboard, _)| leaderboard.grants().keys().cloned()), + ); if let Some(declared) = &declared_roles { roles.extend(declared.iter().cloned()); } @@ -835,7 +870,7 @@ impl GraphqlEngineBuilder { .as_deref() .expect("protocol configuration validated a service ID"); let (authorization_fingerprint, claim_keys) = - role_authorization_info(role, &self.permissions)?; + role_authorization_info(role, &self.permissions, &full_surface.leaderboards)?; let export = DistributedClientSurfaceExport::from_selected_with_execution( service_id, Arc::clone(&role_surface), @@ -950,7 +985,11 @@ impl GraphqlEngineBuilder { }; let (authorization_fingerprint, claim_keys) = if registration.schema_roles.len() == 1 { - role_authorization_info(®istration.schema_roles[0], &self.permissions)? + role_authorization_info( + ®istration.schema_roles[0], + &self.permissions, + &full_surface.leaderboards, + )? } else { // Multi-privilege: fingerprint the application surface grant // intersection under the synthetic privilege key. @@ -958,6 +997,7 @@ impl GraphqlEngineBuilder { &privilege_key, ®istration.schema_roles, &self.permissions, + &full_surface.leaderboards, )? }; // Ensure the privilege key has a role surface for projection @@ -1079,6 +1119,11 @@ impl GraphqlEngineBuilder { query_protocol, read_stores: read_store_kinds, cell_getters, + leaderboard_stores: self + .leaderboards + .into_iter() + .map(|(surface, store)| (surface.name().to_string(), store)) + .collect(), }); Ok(GraphqlEngine { inner }) diff --git a/src/graphql/engine/core.rs b/src/graphql/engine/core.rs index f90815d07..f8b3f14e3 100644 --- a/src/graphql/engine/core.rs +++ b/src/graphql/engine/core.rs @@ -274,6 +274,8 @@ pub(crate) struct EngineInner { pub(crate) read_stores: BTreeMap, pub(crate) cell_getters: BTreeMap>, + /// Leaderboard stores by surface name ([`GraphqlEngineBuilder::leaderboard`]). + pub(crate) leaderboard_stores: BTreeMap>, } pub struct GraphqlEngine { @@ -325,4 +327,8 @@ pub struct GraphqlEngineBuilder { pub(crate) pending_errors: Vec, pub(crate) identity: IdentityConfig, pub(crate) read_stores: BTreeMap, + pub(crate) leaderboards: Vec<( + crate::graphql::surface::LeaderboardSurface, + Arc, + )>, } diff --git a/src/graphql/leaderboard_roots.rs b/src/graphql/leaderboard_roots.rs new file mode 100644 index 000000000..e36bc143a --- /dev/null +++ b/src/graphql/leaderboard_roots.rs @@ -0,0 +1,465 @@ +//! Runtime GraphQL roots for [`LeaderboardSurface`]s: queries, `@live` +//! subscriptions and the change listener that drives them. +//! +//! Authorization is structural like every other root: a role's schema holds +//! only the operations its grant allows, and the viewer-relative operations +//! (`…LeaderboardAround`, `…MyRank`) read the member from the verified +//! session claim, never from an argument. + +use std::sync::Arc; +use std::time::Duration; + +use async_graphql::dynamic::{ + Field, FieldFuture, InputValue, Object, ResolverContext, Subscription, SubscriptionField, + SubscriptionFieldFuture, TypeRef, +}; +use async_graphql::Value; +use futures_util::Stream; +use tokio::sync::{broadcast, mpsc}; + +use super::engine::{EngineInner, ExecutionAuthority, GraphqlEngine}; +use super::protocol::{ + DistributedLiveMetadata, DistributedLiveMode, DistributedQuerySnapshot, + ProtocolResponseAccumulator, +}; +use super::schema::client_error; +use super::surface::{ + leaderboard_change_key, LeaderboardOperation, LeaderboardRoot, LeaderboardSurface, Surface, + LEADERBOARD_ENTRY_TYPE, +}; +use crate::leaderboard::{Board, LeaderboardChangeListener, LeaderboardEntry, LeaderboardStore}; +use crate::microsvc::Session; +use crate::read_model::ReadModelChange; + +/// Default `radius` of an around-me query (capped by the surface). +const DEFAULT_RADIUS: u32 = 5; + +/// The `LeaderboardEntry` object type. +pub(crate) fn entry_object() -> Object { + let field = |name: &'static str, ty: &'static str| { + Field::new(name, TypeRef::named_nn(ty), move |ctx| { + FieldFuture::new(async move { + Ok(ctx.parent_value.as_value().and_then(|value| match value { + Value::Object(object) => object.get(name).cloned(), + _ => None, + })) + }) + }) + }; + Object::new(LEADERBOARD_ENTRY_TYPE) + .field(field("rank", TypeRef::INT)) + .field(field("member", TypeRef::STRING)) + .field(field("score", TypeRef::FLOAT)) +} + +fn result_type(operation: LeaderboardOperation) -> TypeRef { + if operation.is_list() { + TypeRef::named_nn_list_nn(LEADERBOARD_ENTRY_TYPE) + } else { + TypeRef::named(LEADERBOARD_ENTRY_TYPE) + } +} + +fn argument_type(argument: &super::surface::LeaderboardArgument) -> TypeRef { + if argument.nullable { + TypeRef::named(argument.type_name) + } else { + TypeRef::named_nn(argument.type_name) + } +} + +/// Add every granted leaderboard query root of `surface`; returns whether +/// the entry type is needed. +pub(crate) fn add_query_roots(mut query: Object, surface: &Surface) -> (Object, bool) { + let mut any = false; + for leaderboard in &surface.leaderboards { + for root in leaderboard.roots(leaderboard.selected_grant()) { + any = true; + let name = leaderboard.name().to_string(); + let operation = root.operation; + let mut field = Field::new(root.name.clone(), result_type(operation), move |ctx| { + let name = name.clone(); + FieldFuture::new(async move { resolve(&ctx, &name, operation).await }) + }); + for argument in &root.arguments { + field = field.argument(InputValue::new(argument.name, argument_type(argument))); + } + query = query.field(field); + } + } + (query, any) +} + +/// Add the `@live` subscription twins; returns how many were added. +pub(crate) fn add_subscription_roots( + mut subscription: Subscription, + surface: &Surface, +) -> (Subscription, usize) { + let mut added = 0; + for leaderboard in &surface.leaderboards { + for root in leaderboard.roots(leaderboard.selected_grant()) { + if !root.operation.is_live() { + continue; + } + added += 1; + let name = leaderboard.name().to_string(); + let operation = root.operation; + let mut field = + SubscriptionField::new(root.name.clone(), result_type(operation), move |ctx| { + let name = name.clone(); + let request = Request::from_context(&ctx, &name, operation); + SubscriptionFieldFuture::new(async move { + let request = request?; + Ok(live_stream(request)) + }) + }); + for argument in &root.arguments { + field = field.argument(InputValue::new(argument.name, argument_type(argument))); + } + subscription = subscription.field(field); + } + } + (subscription, added) +} + +/// One authorized leaderboard read, with everything owned. +struct Request { + inner: Arc, + store: Arc, + surface: LeaderboardSurface, + root: String, + operation: LeaderboardOperation, + board: Board, + member: Option, + count: usize, + protocol: Option, +} + +impl Request { + fn from_context( + ctx: &ResolverContext<'_>, + name: &str, + operation: LeaderboardOperation, + ) -> Result { + let inner = ctx + .data_opt::>() + .cloned() + .ok_or_else(|| async_graphql::Error::new("GraphqlEngine not in request data"))?; + let session = ctx + .data_opt::() + .cloned() + .unwrap_or_else(Session::new); + let role = privilege_role(ctx.data_opt::(), &session, &inner); + // Re-check the grant against the role surface the request runs under. + let surface = inner + .role_surfaces + .get(&role) + .and_then(|surface| { + surface + .leaderboards + .iter() + .find(|leaderboard| leaderboard.name() == name) + .cloned() + }) + .ok_or_else(|| client_error("FORBIDDEN", "leaderboard is not granted"))?; + let root = surface + .roots(surface.selected_grant()) + .into_iter() + .find(|root: &LeaderboardRoot| root.operation == operation) + .ok_or_else(|| client_error("FORBIDDEN", "leaderboard operation is not granted"))?; + let store = inner + .leaderboard_stores + .get(name) + .cloned() + .ok_or_else(|| client_error("INTERNAL", "leaderboard store is not configured"))?; + let board_key = string_arg(ctx, "board")?; + let board = surface + .board(board_key.as_deref()) + .map_err(|error| client_error("BAD_REQUEST", error))?; + let member = match operation { + LeaderboardOperation::AroundMe | LeaderboardOperation::MyRank => Some( + session + .get(surface.get_member_claim()) + .filter(|member| !member.is_empty()) + .map(str::to_string) + .ok_or_else(|| { + client_error("FORBIDDEN", "the session has no leaderboard member claim") + })?, + ), + LeaderboardOperation::Rank => Some( + string_arg(ctx, "member")? + .filter(|member| !member.is_empty()) + .ok_or_else(|| client_error("BAD_REQUEST", "member is required"))?, + ), + LeaderboardOperation::Top => None, + }; + let count = match operation { + LeaderboardOperation::Top => bounded_arg( + ctx, + "first", + surface.default_first(), + 1, + surface.max_first(), + )?, + LeaderboardOperation::AroundMe => bounded_arg( + ctx, + "radius", + DEFAULT_RADIUS.min(surface.max_radius()), + 0, + surface.max_radius(), + )?, + _ => 0, + }; + Ok(Self { + protocol: ctx.data_opt::().cloned(), + inner, + store, + surface, + root: root.name, + operation, + board, + member, + count: count as usize, + }) + } + + async fn execute(&self) -> Result { + let unavailable = |_| client_error("INTERNAL", "leaderboard store unavailable"); + let entries = match self.operation { + LeaderboardOperation::Top => self + .store + .top(&self.board, self.count) + .await + .map_err(unavailable)?, + LeaderboardOperation::AroundMe => self + .store + .around( + &self.board, + self.member.as_deref().unwrap_or_default(), + self.count, + ) + .await + .map_err(unavailable)?, + LeaderboardOperation::MyRank | LeaderboardOperation::Rank => { + let member = self.member.as_deref().unwrap_or_default(); + let rank = self + .store + .rank(&self.board, member) + .await + .map_err(unavailable)?; + let score = self + .store + .score(&self.board, member) + .await + .map_err(unavailable)?; + return Ok(match (rank, score) { + (Some(rank), Some(score)) => entry_value(&LeaderboardEntry { + rank, + member: member.to_string(), + score, + }), + _ => Value::Null, + }); + } + }; + Ok(Value::List(entries.iter().map(entry_value).collect())) + } + + /// A leaderboard result carries no record or index evidence: the + /// protocol envelope marks it incomplete and, live, as snapshot mode. + fn record_protocol(&self, live: bool) -> Result<(), async_graphql::Error> { + let Some(protocol) = &self.protocol else { + return Ok(()); + }; + let encoding = |_| client_error("INTERNAL", "query evidence encoding failed"); + let scope_token = protocol + .bind_query_snapshot_scope(&serde_json::json!({ + "leaderboard": self.surface.name(), + "root": self.root, + "board": self.board.name(), + "member": self.member, + "count": self.count, + })) + .map_err(encoding)?; + let snapshot = DistributedQuerySnapshot { + scope_token, + records_complete: false, + indexes_comparable: false, + records: Vec::new(), + indexes: Vec::new(), + observations: Vec::new(), + }; + let live = live.then_some(DistributedLiveMetadata { + mode: DistributedLiveMode::Snapshot, + reset: true, + cursors: Vec::new(), + }); + protocol + .record_query_metadata(snapshot, live) + .map_err(encoding) + } +} + +fn entry_value(entry: &LeaderboardEntry) -> Value { + let mut object = async_graphql::indexmap::IndexMap::new(); + object.insert( + async_graphql::Name::new("rank"), + Value::from(i64::try_from(entry.rank).unwrap_or(i64::MAX)), + ); + object.insert( + async_graphql::Name::new("member"), + Value::from(entry.member.clone()), + ); + object.insert(async_graphql::Name::new("score"), Value::from(entry.score)); + Value::Object(object) +} + +fn string_arg( + ctx: &ResolverContext<'_>, + name: &str, +) -> Result, async_graphql::Error> { + match ctx.args.get(name) { + None => Ok(None), + Some(value) if value.is_null() => Ok(None), + Some(value) => value + .string() + .map(|value| Some(value.to_string())) + .map_err(|_| client_error("BAD_REQUEST", format!("`{name}` must be a string"))), + } +} + +fn bounded_arg( + ctx: &ResolverContext<'_>, + name: &str, + default: u32, + min: u32, + max: u32, +) -> Result { + let value = match ctx.args.get(name) { + None => return Ok(default), + Some(value) if value.is_null() => return Ok(default), + Some(value) => value + .i64() + .map_err(|_| client_error("BAD_REQUEST", format!("`{name}` must be an integer")))?, + }; + if value < i64::from(min) || value > i64::from(max) { + return Err(client_error( + "BAD_REQUEST", + format!("`{name}` must be within {min}..={max}"), + )); + } + Ok(value as u32) +} + +fn privilege_role( + authority: Option<&ExecutionAuthority>, + session: &Session, + inner: &EngineInner, +) -> String { + if let Some(authority) = authority { + return authority.privilege_role.clone(); + } + session + .roles() + .first() + .map(|role| (*role).to_string()) + .unwrap_or_else(|| inner.anonymous_role.clone()) +} + +async fn resolve( + ctx: &ResolverContext<'_>, + name: &str, + operation: LeaderboardOperation, +) -> Result, async_graphql::Error> { + let request = Request::from_context(ctx, name, operation)?; + let value = request.execute().await?; + request.record_protocol(false)?; + Ok(match value { + Value::Null => None, + value => Some(value), + }) +} + +type LiveItem = Result; + +struct LeaderboardLiveStream { + rx: mpsc::Receiver, +} + +impl Stream for LeaderboardLiveStream { + type Item = LiveItem; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.rx.poll_recv(cx) + } +} + +/// Initial result, then a re-execution (debounced, hash-gated) whenever the +/// board's change key is published. A failed execution is terminal. +fn live_stream(request: Request) -> LeaderboardLiveStream { + let mut changes = request.inner.change_hub.subscribe(); + let key = leaderboard_change_key(request.board.name()); + let (tx, rx) = mpsc::channel::(8); + tokio::spawn(async move { + let mut last_hash = None; + loop { + let executed = tokio::select! { + _ = tx.closed() => return, + executed = request.execute() => executed, + }; + match executed { + Ok(value) => { + let hash = super::subscribe::response_hash(&value); + if last_hash != Some(hash) { + if let Err(error) = request.record_protocol(true) { + let _ = tx.send(Err(error)).await; + return; + } + last_hash = Some(hash); + if tx.send(Ok(value)).await.is_err() { + return; + } + } + } + Err(error) => { + let _ = tx.send(Err(error)).await; + return; + } + } + // Wait for a change to this board (an empty set means "all"). + loop { + let change = tokio::select! { + _ = tx.closed() => return, + change = changes.recv() => change, + }; + match change { + Ok(change) if change.tables.is_empty() || change.tables.contains(&key) => break, + Ok(_) => continue, + Err(broadcast::error::RecvError::Lagged(_)) => break, + Err(broadcast::error::RecvError::Closed) => return, + } + } + tokio::select! { + _ = tx.closed() => return, + _ = tokio::time::sleep(Duration::from_millis(100)) => {} + } + while let Ok(_) | Err(broadcast::error::TryRecvError::Lagged(_)) = changes.try_recv() {} + } + }); + LeaderboardLiveStream { rx } +} + +impl GraphqlEngine { + /// A [`LeaderboardChangeListener`] that refreshes `@live` leaderboard + /// subscriptions of this engine: register it on the projector writing + /// the boards (`LeaderboardProjector::with_change_listener`). + pub fn leaderboard_change_listener(&self) -> LeaderboardChangeListener { + let hub = self.inner.change_hub.clone(); + Arc::new(move |board: &Board| { + hub.publish(ReadModelChange::new([leaderboard_change_key(board.name())])); + }) + } +} diff --git a/src/graphql/mod.rs b/src/graphql/mod.rs index 46565a410..bd2edf605 100644 --- a/src/graphql/mod.rs +++ b/src/graphql/mod.rs @@ -58,6 +58,8 @@ pub mod http; #[cfg(feature = "graphql")] pub mod identity; #[cfg(feature = "graphql")] +mod leaderboard_roots; +#[cfg(feature = "graphql")] pub mod protocol; #[cfg(feature = "graphql")] pub(crate) mod query_protocol; diff --git a/src/graphql/projection_delta/vector_tests.rs b/src/graphql/projection_delta/vector_tests.rs index 023b7a323..599c4c97d 100644 --- a/src/graphql/projection_delta/vector_tests.rs +++ b/src/graphql/projection_delta/vector_tests.rs @@ -1025,6 +1025,7 @@ fn replay_manifest() -> DistributedClientManifest { projectors: vec![], projection_programs: vec![], projection_bindings: vec![], + leaderboards: vec![], } } diff --git a/src/graphql/schema.rs b/src/graphql/schema.rs index ca5b16627..5bb70e351 100644 --- a/src/graphql/schema.rs +++ b/src/graphql/schema.rs @@ -143,6 +143,17 @@ pub fn build_role_schema( } } + // Leaderboard roots granted to this role (resolved by their stores). + let (with_leaderboards, leaderboard_entry) = + super::leaderboard_roots::add_query_roots(query, role_surface); + query = with_leaderboards; + if leaderboard_entry { + registered_objects.insert( + super::surface::LEADERBOARD_ENTRY_TYPE.to_string(), + super::leaderboard_roots::entry_object(), + ); + } + // Comparison input types come from this exact role Surface instance. // Only register when IR lists ops — never emit empty input objects. for scalar in &scalars_needed { @@ -178,7 +189,7 @@ pub fn build_role_schema( // Empty-role Query must still define ≥1 field (async-graphql requirement). // Spec: empty role → FORBIDDEN with extensions.code (no query surface). - if role_surface.query_fields.is_empty() && !has_causal_commands { + if role_surface.query_fields.is_empty() && !leaderboard_entry && !has_causal_commands { query = query.field(Field::new( "_empty", TypeRef::named_nn(TypeRef::BOOLEAN), @@ -322,6 +333,13 @@ pub fn build_role_schema( has_subscription = true; } + if role_surface.subscriptions { + let (with_leaderboards, added) = + super::leaderboard_roots::add_subscription_roots(subscription, role_surface); + subscription = with_leaderboards; + has_subscription |= added > 0; + } + let mut builder = if let Some(m) = mutation { if has_subscription { Schema::build( @@ -918,7 +936,7 @@ fn sanitize_compile_error(e: &str) -> String { } } -fn client_error(code: &str, message: impl Into) -> async_graphql::Error { +pub(crate) fn client_error(code: &str, message: impl Into) -> async_graphql::Error { use async_graphql::ErrorExtensions; let code = code.to_string(); async_graphql::Error::new(message.into()).extend_with(move |_, ext| { @@ -1255,6 +1273,7 @@ mod causal_command_schema_tests { projectors: Vec::new(), projectors_attached: false, service_binding: None, + leaderboards: Vec::new(), } } diff --git a/src/graphql/sdl.rs b/src/graphql/sdl.rs index 0bcdc2903..4177a459d 100644 --- a/src/graphql/sdl.rs +++ b/src/graphql/sdl.rs @@ -118,6 +118,32 @@ pub fn graphql_sdl_for_role( graphql_sdl_from_surface(&role_surface) } +fn leaderboard_root_sdl(root: &super::surface::LeaderboardRoot) -> String { + let arguments = if root.arguments.is_empty() { + String::new() + } else { + format!( + "({})", + root.arguments + .iter() + .map(|argument| format!( + "{}: {}{}", + argument.name, + argument.type_name, + if argument.nullable { "" } else { "!" } + )) + .collect::>() + .join(", ") + ) + }; + let result = if root.operation.is_list() { + format!("[{}!]!", super::surface::LEADERBOARD_ENTRY_TYPE) + } else { + super::surface::LEADERBOARD_ENTRY_TYPE.to_string() + }; + format!(" {}{arguments}: {result}\n", root.name) +} + /// Internal renderer over an already IR-filtered set of read models. fn graphql_sdl_from_read_models(surface: &super::surface::Surface) -> Result { let has_causal_commands = !surface.commands.is_empty(); @@ -188,6 +214,20 @@ fn graphql_sdl_from_read_models(surface: &super::surface::Surface) -> Result>(); + if !leaderboard_roots.is_empty() { + claim_name(&mut type_names, super::surface::LEADERBOARD_ENTRY_TYPE)?; + } + for root in &leaderboard_roots { + claim_name(&mut query_fields, &root.name)?; + if surface.subscriptions && root.operation.is_live() { + claim_name(&mut subscription_fields, &root.name)?; + } + } let mut out = String::new(); @@ -228,6 +268,13 @@ fn graphql_sdl_from_read_models(surface: &super::surface::Surface) -> Result Result

) -> Self { - Self { store, projection } + Self { + store, + projection, + listeners: Vec::new(), + } + } + + /// Notify `listener` whenever a live board's scores change. + pub fn with_change_listener(mut self, listener: LeaderboardChangeListener) -> Self { + self.listeners.push(listener); + self + } + + fn notify(&self, board: &Board) { + for listener in &self.listeners { + listener(board); + } } /// The store. @@ -148,7 +172,12 @@ where event.occurred_at_unix_ms(), ); match self.store.apply(&board, &change.submission, &key).await? { - ApplyOutcome::Applied { .. } => report.applied += 1, + ApplyOutcome::Applied { .. } => { + report.applied += 1; + if !staging { + self.notify(&board); + } + } ApplyOutcome::Duplicate { .. } => report.duplicates += 1, } } @@ -266,6 +295,7 @@ where pub async fn commit(self) -> Result { for board in self.boards.values() { self.projector.store.promote(board).await?; + self.projector.notify(board); } Ok(self.report) } @@ -501,4 +531,33 @@ mod tests { Some(2_000.0) ); } + + #[tokio::test] + async fn listeners_hear_applied_changes_and_promotions_but_not_duplicates() { + let heard = Arc::new(std::sync::Mutex::new(Vec::::new())); + let log = heard.clone(); + let projector = projector().with_change_listener(Arc::new(move |board: &Board| { + log.lock().unwrap().push(board.name().to_string()); + })); + projector.apply_event(&slain(1, "ann", 10)).await.unwrap(); + projector.apply_event(&slain(1, "ann", 10)).await.unwrap(); + assert_eq!( + *heard.lock().unwrap(), + ["xp", "best-kill"], + "one per applied change; the duplicate is silent" + ); + let boards = [Board::high_first("xp").unwrap()]; + projector + .rebuild(&boards, [slain(1, "ann", 10), slain(2, "bob", 5)]) + .await + .unwrap(); + let heard = heard.lock().unwrap(); + assert_eq!( + heard.len(), + 4, + "one per promoted board, none for staging writes" + ); + assert!(heard[2..].contains(&"xp".to_string())); + assert!(heard[2..].contains(&"best-kill".to_string())); + } } diff --git a/tests/graphql_leaderboard/main.rs b/tests/graphql_leaderboard/main.rs new file mode 100644 index 000000000..2a53fa6c5 --- /dev/null +++ b/tests/graphql_leaderboard/main.rs @@ -0,0 +1,482 @@ +//! Leaderboards through the generated GraphQL surface: role-structured roots, +//! viewer-bound `myRank`, `@live` top-N driven by the projector's change +//! listener, the client manifest section and the static SDL. + +#![cfg(all(feature = "graphql", feature = "sqlite"))] + +use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; + +use async_graphql::Request; +use distributed::graphql::client_manifest::{ + DistributedClientManifest, DistributedClientSurfaceExport, +}; +use distributed::graphql::sdl::graphql_sdl_from_surface; +use distributed::graphql::surface::{ + build_surface, surface_for_role, LeaderboardGrant, LeaderboardSurface, SurfaceDialect, + SurfaceOptions, +}; +use distributed::graphql::GraphqlEngine; +use distributed::leaderboard::{ + Board, InMemoryLeaderboard, LeaderboardChange, LeaderboardError, LeaderboardProjector, + LeaderboardStore, ScorePolicy, ScoreSubmission, +}; +use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; +use distributed::{DomainEventOccurrence, ExternalEventSource}; +use futures_util::StreamExt; + +fn xp_board() -> Board { + Board::high_first("mobpit.xp").unwrap() +} + +fn xp_surface() -> LeaderboardSurface { + LeaderboardSurface::single("xp", &xp_board()) + .limits(3, 5, 2) + .grant("user", LeaderboardGrant::read()) + .grant("moderator", LeaderboardGrant::read().rank_any_member()) +} + +fn season_surface() -> LeaderboardSurface { + LeaderboardSurface::family( + "season", + "season.xp", + distributed::leaderboard::ScoreOrder::HighFirst, + ) + .grant("user", LeaderboardGrant::top_only()) +} + +async fn engine(store: Arc) -> GraphqlEngine { + let repo = distributed::SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .unwrap(); + let catalog = distributed::ReadModelCatalog::new("leaderboards"); + GraphqlEngine::from_schema_catalog(&catalog, repo.pool().clone()) + .unwrap() + .roles(&["user", "moderator"]) + .leaderboard(xp_surface(), store.clone()) + .leaderboard(season_surface(), store) + .build() + .expect("build engine") +} + +fn session(role: &str, user: Option<&str>) -> Session { + let mut session = Session::new(); + session.set(ROLE_KEY, role); + if let Some(user) = user { + session.set(USER_ID_KEY, user); + } + session +} + +async fn seed(store: &InMemoryLeaderboard) { + for (member, score) in [("ann", 50.0), ("bob", 40.0), ("cat", 30.0), ("dan", 20.0)] { + store + .submit( + &xp_board(), + &ScoreSubmission::new(member, score, ScorePolicy::Replace), + ) + .await + .unwrap(); + } +} + +async fn run(engine: &GraphqlEngine, session: &Session, query: &str) -> serde_json::Value { + let response = engine.execute(session, Request::new(query)).await; + serde_json::to_value(&response).unwrap() +} + +fn error_code(response: &serde_json::Value) -> Option<&str> { + response["errors"][0]["extensions"]["code"].as_str() +} + +#[tokio::test] +async fn top_around_and_my_rank_read_the_board_for_the_viewer() { + let store = Arc::new(InMemoryLeaderboard::new()); + seed(&store).await; + let engine = engine(store).await; + let cat = session("user", Some("cat")); + + let top = run(&engine, &cat, "{ xpLeaderboard { rank member score } }").await; + assert_eq!( + top["data"]["xpLeaderboard"], + serde_json::json!([ + { "rank": 0, "member": "ann", "score": 50.0 }, + { "rank": 1, "member": "bob", "score": 40.0 }, + { "rank": 2, "member": "cat", "score": 30.0 }, + ]), + "default first = 3" + ); + let around = run( + &engine, + &cat, + "{ xpLeaderboardAround(radius: 1) { member } }", + ) + .await; + assert_eq!( + around["data"]["xpLeaderboardAround"], + serde_json::json!([{ "member": "bob" }, { "member": "cat" }, { "member": "dan" }]) + ); + let mine = run(&engine, &cat, "{ xpMyRank { rank member score } }").await; + assert_eq!( + mine["data"]["xpMyRank"], + serde_json::json!({ "rank": 2, "member": "cat", "score": 30.0 }) + ); + let unranked = run( + &engine, + &session("user", Some("zed")), + "{ xpMyRank { rank } }", + ) + .await; + assert_eq!(unranked["data"]["xpMyRank"], serde_json::Value::Null); + + let season = run( + &engine, + &cat, + r#"{ seasonLeaderboard(board: "s1") { member } }"#, + ) + .await; + assert_eq!(season["data"]["seasonLeaderboard"], serde_json::json!([])); +} + +#[tokio::test] +async fn my_rank_is_bound_to_the_session_and_ranking_others_needs_the_grant() { + let store = Arc::new(InMemoryLeaderboard::new()); + seed(&store).await; + let engine = engine(store).await; + let cat = session("user", Some("cat")); + + // A user cannot name another member: no argument, no `xpRank` root. + let spoofed = run(&engine, &cat, r#"{ xpMyRank(member: "ann") { rank } }"#).await; + assert!(spoofed["errors"].is_array(), "{spoofed}"); + let other = run(&engine, &cat, r#"{ xpRank(member: "ann") { rank } }"#).await; + assert!(other["errors"].is_array(), "{other}"); + // Around-me is the viewer's neighbourhood only. + let around = run( + &engine, + &cat, + r#"{ xpLeaderboardAround(member: "ann") { rank } }"#, + ) + .await; + assert!(around["errors"].is_array(), "{around}"); + + // A moderator granted `rank_any_member` may rank anyone. + let moderator = session("moderator", Some("mod-1")); + let ranked = run( + &engine, + &moderator, + r#"{ xpRank(member: "ann") { rank score } }"#, + ) + .await; + assert_eq!( + ranked["data"]["xpRank"], + serde_json::json!({ "rank": 0, "score": 50.0 }) + ); + + // Without the member claim the viewer-relative roots fail closed. + let anonymous = run(&engine, &session("user", None), "{ xpMyRank { rank } }").await; + assert_eq!(error_code(&anonymous), Some("FORBIDDEN"), "{anonymous}"); +} + +#[tokio::test] +async fn arguments_are_validated() { + let store = Arc::new(InMemoryLeaderboard::new()); + let engine = engine(store).await; + let cat = session("user", Some("cat")); + for query in [ + "{ xpLeaderboard(first: 6) { rank } }", + "{ xpLeaderboard(first: 0) { rank } }", + "{ xpLeaderboardAround(radius: 3) { rank } }", + r#"{ seasonLeaderboard(board: "not a key") { rank } }"#, + ] { + let response = run(&engine, &cat, query).await; + assert_eq!( + error_code(&response), + Some("BAD_REQUEST"), + "{query}: {response}" + ); + } + let missing = run(&engine, &cat, "{ seasonLeaderboard { rank } }").await; + assert!( + missing["errors"].is_array(), + "family boards require `board`" + ); +} + +#[derive(serde::Serialize, serde::Deserialize, distributed::DomainEvent)] +#[domain_event(name = "test.xp_gained", version = 1)] +struct XpGained { + player: String, + xp: u32, +} + +fn gained(position: u64, player: &str, xp: u32) -> DomainEventOccurrence { + DomainEventOccurrence::capture_external( + ExternalEventSource { + producer: "zone.sim".into(), + stream: "zone-1".into(), + position, + key: "xp".into(), + }, + UNIX_EPOCH + Duration::from_millis(position), + Default::default(), + &XpGained { + player: player.into(), + xp, + }, + ) + .unwrap() +} + +#[tokio::test] +async fn live_top_n_updates_after_an_applied_score_change() { + let store = Arc::new(InMemoryLeaderboard::new()); + seed(&store).await; + let engine = engine(store.clone()).await; + let projection = + |event: &DomainEventOccurrence| -> Result, LeaderboardError> { + let body: XpGained = event + .decode_body() + .map_err(|error| LeaderboardError::invalid(error.to_string()))?; + Ok(vec![LeaderboardChange::new( + xp_board(), + ScoreSubmission::new(body.player, f64::from(body.xp), ScorePolicy::Increment), + )]) + }; + let projector = LeaderboardProjector::new(store.clone(), Arc::new(projection)) + .with_change_listener(engine.leaderboard_change_listener()); + + let mut stream = engine.execute_stream( + &session("user", Some("dan")), + Request::new("subscription { xpLeaderboard(first: 2) { member score } }"), + ); + let first = tokio::time::timeout(Duration::from_secs(5), stream.next()) + .await + .unwrap() + .unwrap(); + assert_eq!( + serde_json::to_value(&first.data).unwrap()["xpLeaderboard"], + serde_json::json!([{ "member": "ann", "score": 50.0 }, { "member": "bob", "score": 40.0 }]) + ); + + // A change below the top two re-executes but does not push (hash gate). + projector.apply_event(&gained(1, "dan", 5)).await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(400), stream.next()) + .await + .is_err(), + "no push when the top-N is unchanged" + ); + + projector.apply_event(&gained(2, "dan", 100)).await.unwrap(); + let pushed = tokio::time::timeout(Duration::from_secs(5), stream.next()) + .await + .unwrap() + .unwrap(); + assert_eq!( + serde_json::to_value(&pushed.data).unwrap()["xpLeaderboard"], + serde_json::json!([{ "member": "dan", "score": 125.0 }, { "member": "ann", "score": 50.0 }]) + ); + + // A redelivered (duplicate) event changes nothing and pushes nothing. + projector.apply_event(&gained(2, "dan", 100)).await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(400), stream.next()) + .await + .is_err() + ); +} + +fn surfaces() -> distributed::graphql::surface::Surface { + let options = SurfaceOptions { + dialect: SurfaceDialect::Sqlite, + aggregates: false, + subscriptions: true, + default_limit: 100, + max_limit: 1000, + }; + build_surface(&[], &options) + .unwrap() + .with_leaderboards([xp_surface(), season_surface()]) + .unwrap() +} + +fn role_manifest(role: &str) -> DistributedClientManifest { + let selected = surface_for_role(&surfaces(), role, &Default::default()).unwrap(); + DistributedClientSurfaceExport::from_selected("leaderboards", selected) + .unwrap() + .manifest() + .unwrap() + .clone() +} + +#[test] +fn board_queries_are_in_the_client_manifest_per_role() { + let user = role_manifest("user"); + assert!(user.capabilities.live_queries); + let names = user + .leaderboards + .iter() + .flat_map(|leaderboard| leaderboard.roots.iter().map(|root| root.name.as_str())) + .collect::>(); + assert_eq!( + names, + [ + "seasonLeaderboard", + "xpLeaderboard", + "xpLeaderboardAround", + "xpMyRank" + ] + ); + let xp = user.leaderboards.iter().find(|l| l.name == "xp").unwrap(); + assert_eq!(xp.entry_typename, "LeaderboardEntry"); + assert_eq!(xp.order, "high_first"); + assert_eq!((xp.default_first, xp.max_first, xp.max_radius), (3, 5, 2)); + let my_rank = xp.roots.iter().find(|r| r.name == "xpMyRank").unwrap(); + assert!( + my_rank.arguments.is_empty(), + "the member is never an argument" + ); + assert!(my_rank.live && !my_rank.list); + + let moderator = role_manifest("moderator"); + let rank = moderator.leaderboards[0] + .roots + .iter() + .find(|root| root.name == "xpRank") + .unwrap(); + assert!(!rank.live); + assert_eq!(rank.arguments[0].name, "member"); + + let guest = role_manifest("guest"); + assert!(guest.leaderboards.is_empty()); + let json = serde_json::to_value(&guest).unwrap(); + assert!(json.get("leaderboards").is_none(), "absent when empty"); + + // Round-trips through the strict manifest schema. + let encoded = serde_json::to_string(&user).unwrap(); + let decoded: DistributedClientManifest = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, user); + assert_ne!(user.schema_fingerprint, guest.schema_fingerprint); +} + +#[test] +fn sdl_declares_the_entry_type_and_role_roots() { + let user = surface_for_role(&surfaces(), "user", &Default::default()).unwrap(); + let sdl = graphql_sdl_from_surface(&user).unwrap(); + assert!(sdl + .contains("type LeaderboardEntry {\n rank: Int!\n member: String!\n score: Float!\n}")); + assert!(sdl.contains(" xpLeaderboard(first: Int): [LeaderboardEntry!]!\n")); + assert!(sdl.contains(" xpMyRank: LeaderboardEntry\n")); + assert!(sdl.contains(" seasonLeaderboard(board: String!, first: Int): [LeaderboardEntry!]!\n")); + assert!(!sdl.contains("xpRank")); + assert!(sdl.contains("type Subscription {")); +} + +/// With the causal client protocol on (as Mob Pit runs it), leaderboard +/// results carry an incomplete snapshot and, live, `snapshot` mode with reset. +#[tokio::test] +async fn protocol_envelopes_mark_leaderboards_as_snapshot_only() { + let store = Arc::new(InMemoryLeaderboard::new()); + seed(&store).await; + let repo = distributed::SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .unwrap(); + let catalog = distributed::ReadModelCatalog::new("leaderboards"); + let engine = GraphqlEngine::from_schema_catalog(&catalog, repo.pool().clone()) + .unwrap() + .roles(&["user", "moderator"]) + .protocol_token_key([7; 32]) + .leaderboard(xp_surface(), store.clone()) + .build() + .expect("protocol engine"); + let cat = session("user", Some("cat")); + + let query = engine + .execute(&cat, Request::new("{ xpMyRank { rank } }")) + .await; + assert!(query.errors.is_empty(), "{:?}", query.errors); + let query = serde_json::to_value(&query).unwrap(); + let snapshot = &query["extensions"]["distributed"]["snapshot"]; + assert_eq!(snapshot["recordsComplete"], false, "{query}"); + assert_eq!(snapshot["indexesComparable"], false, "{query}"); + + let mut stream = engine.execute_stream( + &cat, + Request::new("subscription { xpLeaderboard(first: 1) { member } }"), + ); + let frame = tokio::time::timeout(Duration::from_secs(5), stream.next()) + .await + .unwrap() + .unwrap(); + assert!(frame.errors.is_empty(), "{:?}", frame.errors); + let frame = serde_json::to_value(&frame).unwrap(); + let live = &frame["extensions"]["distributed"]["live"]; + assert_eq!(live["mode"], "snapshot", "{frame}"); + assert_eq!(live["reset"], true, "{frame}"); +} + +/// The framework's manifest compiles with the CLI client compiler: the CLI +/// recomputes the same schema fingerprint and generates `leaderboards.ts`. +#[test] +fn the_cli_compiles_a_framework_leaderboard_manifest_into_the_ts_client() { + use distributed::graphql::surface::RoleGrant; + use distributed_cli::{ + compile_client, ClientCompileInput, ClientDocument, ClientSurfaceSelector, + }; + + let items = distributed::TableSchema { + model_name: "ItemView".into(), + table_name: "items".into(), + columns: vec![ + distributed::TableColumn { + primary_key: true, + ..distributed::TableColumn::new("id", "id", distributed::ColumnType::Text) + }, + distributed::TableColumn::new("name", "name", distributed::ColumnType::Text), + ], + primary_key: distributed::PrimaryKey::new(["id"]), + version_column: Some("_sourced_version".into()), + foreign_keys: Vec::new(), + indexes: Vec::new(), + relationships: Vec::new(), + kind: distributed::TableKind::ReadModel, + }; + let options = SurfaceOptions { + dialect: SurfaceDialect::Sqlite, + aggregates: false, + subscriptions: true, + default_limit: 100, + max_limit: 1000, + }; + let surface = build_surface(&[items], &options) + .unwrap() + .with_leaderboards([xp_surface()]) + .unwrap(); + let grants = + std::collections::BTreeMap::from([("ItemView".to_string(), RoleGrant::all_columns())]); + let selected = surface_for_role(&surface, "user", &grants).unwrap(); + let manifest = DistributedClientSurfaceExport::from_selected("leaderboards", selected) + .unwrap() + .manifest_json_pretty() + .unwrap(); + let project = compile_client(ClientCompileInput::new( + serde_json::from_str(&manifest).unwrap(), + ClientSurfaceSelector::role("user"), + vec![ClientDocument::new( + "src/routes/items.graphql", + "query Items { items { name } }", + )], + )) + .expect("the CLI accepts the framework manifest"); + let leaderboards = project + .files + .iter() + .find(|file| file.path == "leaderboards.ts") + .expect("leaderboards.ts"); + assert!(leaderboards + .contents + .contains("export const XpMyRankLeaderboard = {")); + assert!(leaderboards + .contents + .contains(r#"document: "query XpLeaderboardAround($radius: Int) { xpLeaderboardAround(radius: $radius) { rank member score } }","#)); +} diff --git a/tests/redis_leaderboard/main.rs b/tests/redis_leaderboard/main.rs index 3078932cc..b514e6814 100644 --- a/tests/redis_leaderboard/main.rs +++ b/tests/redis_leaderboard/main.rs @@ -206,3 +206,98 @@ async fn unreachable_redis_fails_fast_as_retryable() { .unwrap_err(); assert!(err.is_retryable()); } + +/// The same Redis boards served through GraphQL: roots read the sorted sets +/// and an applied projector change refreshes a `@live` top-N. +#[cfg(all(feature = "graphql", feature = "sqlite"))] +#[tokio::test] +async fn redis_boards_serve_graphql_queries_and_live_updates() { + use distributed::graphql::surface::{LeaderboardGrant, LeaderboardSurface}; + use distributed::graphql::GraphqlEngine; + use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; + use futures_util::StreamExt; + + let Some(url) = redis_url() else { return }; + let store = Arc::new( + RedisLeaderboard::connect(&url) + .unwrap() + .with_prefix(unique("gql")), + ); + let board = Board::high_first("gold").unwrap(); + for (member, score) in [("ann", 30.0), ("bob", 20.0)] { + store + .submit( + &board, + &ScoreSubmission::new(member, score, ScorePolicy::Replace), + ) + .await + .unwrap(); + } + let repo = distributed::SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .unwrap(); + let engine = GraphqlEngine::from_schema_catalog( + &distributed::ReadModelCatalog::new("gold-service"), + repo.pool().clone(), + ) + .unwrap() + .roles(&["user"]) + .leaderboard( + LeaderboardSurface::single("gold", &board).grant("user", LeaderboardGrant::read()), + store.clone(), + ) + .build() + .unwrap(); + let mut session = Session::new(); + session.set(ROLE_KEY, "user"); + session.set(USER_ID_KEY, "bob"); + + let mine = engine + .execute( + &session, + async_graphql::Request::new("{ goldMyRank { rank score } }"), + ) + .await; + assert!(mine.errors.is_empty(), "{:?}", mine.errors); + assert_eq!( + serde_json::to_value(&mine.data).unwrap()["goldMyRank"], + serde_json::json!({ "rank": 1, "score": 20.0 }) + ); + + let projector = LeaderboardProjector::new( + store.clone(), + Arc::new( + |event: &DomainEventOccurrence| -> Result, LeaderboardError> { + let body: GoldLooted = event + .decode_body() + .map_err(|error| LeaderboardError::invalid(error.to_string()))?; + Ok(vec![LeaderboardChange::new( + Board::high_first("gold").unwrap(), + ScoreSubmission::new(body.player, f64::from(body.gold), ScorePolicy::Increment), + )]) + }, + ), + ) + .with_change_listener(engine.leaderboard_change_listener()); + let mut live = engine.execute_stream( + &session, + async_graphql::Request::new("subscription { goldLeaderboard(first: 1) { member } }"), + ); + let first = tokio::time::timeout(Duration::from_secs(5), live.next()) + .await + .unwrap() + .unwrap(); + assert_eq!( + serde_json::to_value(&first.data).unwrap()["goldLeaderboard"], + serde_json::json!([{ "member": "ann" }]) + ); + projector.apply_event(&looted(1, "bob", 50)).await.unwrap(); + let pushed = tokio::time::timeout(Duration::from_secs(5), live.next()) + .await + .unwrap() + .unwrap(); + assert_eq!( + serde_json::to_value(&pushed.data).unwrap()["goldLeaderboard"], + serde_json::json!([{ "member": "bob" }]) + ); +} From 6f2f879d726653e6e9b96f383bebc4011cd5519f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 10:54:07 -0500 Subject: [PATCH 11/40] feat(game): CelldFactSink routes zone facts to celld cells One sink per zone server sends each fact to POST {CELLD_URL}/{kind}/{shard}/{command}, routed by the fact's command (by principal, by an input field, or lifecycle_route: zone.* facts go to the account's presence cell, never to gameplay cells). Command ids come from command_id_for(service, fact key), so a retry after a timeout replays the cell ledger receipt. Principal and FactStamp travel as x-user-id and x-game-* headers, only with the internal secret; cells rebuild the session with cell::fact_session_from_headers. celld unreachable, 5xx or in-progress is Retryable; with the new RetryPolicy::until_settled facts stay queued (no local store) and the zone sees Backpressure at capacity. Unrouted or rejected facts are Permanent. The in-process CellFactSink is now native-only. tests/celld_facts proves it on celld dev 0.4.0 and 0.6.2 with three cell kinds, replay, the internal secret, and a celld crash/restart. [[tasks/gr-p24-zone-celld-fact-routing]] --- .github/workflows/integration-celld.yaml | 8 + .gitignore | 3 + Cargo.toml | 2 +- distributed_game/Cargo.toml | 9 + distributed_game/README.md | 33 +- distributed_game/src/cell.rs | 38 +- distributed_game/src/celld.rs | 462 +++++++++++++++++ distributed_game/src/facts.rs | 13 + distributed_game/src/lib.rs | 2 + distributed_game/tests/celld_facts/main.rs | 574 +++++++++++++++++++++ tests/celld/fact-worker/Cargo.toml | 24 + tests/celld/fact-worker/src/lib.rs | 455 ++++++++++++++++ tests/celld/fact-worker/wrangler.jsonc | 24 + 13 files changed, 1642 insertions(+), 5 deletions(-) create mode 100644 distributed_game/src/celld.rs create mode 100644 distributed_game/tests/celld_facts/main.rs create mode 100644 tests/celld/fact-worker/Cargo.toml create mode 100644 tests/celld/fact-worker/src/lib.rs create mode 100644 tests/celld/fact-worker/wrangler.jsonc diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index 74ae5b51e..c42757276 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -42,6 +42,7 @@ jobs: workspaces: | tests/celld/worker -> tests/celld/worker/target tests/celld/alarm-worker -> tests/celld/alarm-worker/target + tests/celld/fact-worker -> tests/celld/fact-worker/target shared-key: celld-storage - uses: actions/setup-node@v4 with: @@ -63,6 +64,13 @@ jobs: run: worker-build --release - name: Prove re-armable cell alarms (atomic arm, cadence, watchdog, restart) run: node tests/celld/alarms.mjs + - name: Build the zone-fact cells Worker + working-directory: tests/celld/fact-worker + run: worker-build --release + - name: Route zone facts to celld cells (CelldFactSink, outage and restart) + env: + CELLD_FACTS_REQUIRED: "1" + run: cargo test -p distributed_game --features celld,runtime --test celld_facts - name: Retain storage proof results and logs if: always() uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index fb273dae0..aa754a628 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,9 @@ tests/celld/relay-worker/build/ tests/celld/alarm-worker/target/ tests/celld/alarm-worker/build/ tests/celld/alarm-worker/.celld/ +tests/celld/fact-worker/target/ +tests/celld/fact-worker/build/ +tests/celld/fact-worker/.celld/ # Build output of CLI test fixture crates /distributed_cli/tests/fixtures/*/target/ diff --git a/Cargo.toml b/Cargo.toml index 44d730698..1aeb3e497 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = ["distributed_macros", "distributed_cli", "distributed_game", "distributed_game_macros"] -exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"] +exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/celld/fact-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"] resolver = "2" [workspace.package] diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index 8b2e35f47..aa45f84c9 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -16,6 +16,9 @@ default = [] runtime = ["dep:tokio"] # Adapter that commits facts into a `distributed` AggregateCell. cell = ["dep:distributed", "dep:sha2"] +# `celld::CelldFactSink`: a zone server's facts sent to celld cells over the +# internal HTTP path (routes by command; stable command ids; no local store). +celld = ["cell", "dep:reqwest"] # Per-zone Prometheus metrics (dependency-free text exposition). metrics = [] # Load-test bots (library + `game-bots` binary). @@ -40,6 +43,7 @@ base64 = { version = "0.23", optional = true } tokio-tungstenite = { version = "0.30", optional = true } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"], optional = true } rusqlite = { version = "0.39", features = ["bundled"], optional = true } +reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true } [dev-dependencies] trybuild = "=1.0.120" @@ -68,6 +72,11 @@ name = "net_e2e" path = "tests/net_e2e/main.rs" required-features = ["net"] +[[test]] +name = "celld_facts" +path = "tests/celld_facts/main.rs" +required-features = ["celld", "runtime"] + [[test]] name = "inbox" path = "tests/inbox/main.rs" diff --git a/distributed_game/README.md b/distributed_game/README.md index 5a14c9493..8a6eb532f 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -865,7 +865,8 @@ trait and its ABI (`run_abi`, `call_bytes`). |---|---|---| | *(default)* | Zone/host runtime, facts queue, protocol, schema, replication, authoring derives, game surface, predicted steps (compiles to `wasm32-unknown-unknown`, where `#[predict]` exports its steps) | `serde`, `serde_json`, `distributed_game_macros` | | `runtime` | `spawn_fact_worker` on Tokio | `tokio` (rt, time, sync) | -| `cell` | `cell::CellFactSink` → `AggregateCell::dispatch_idempotent`, `stamp_event_record` | `distributed`, `sha2` | +| `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record` | `distributed`, `sha2` | +| `celld` | `celld::CelldFactSink`: facts → celld cells over the internal HTTP path, routed by command | `cell` + `reqwest` | | `metrics` | `metrics::GameMetrics` per-zone Prometheus text | – | | `net` | `net::GameServer` (axum WebSocket), HMAC tickets, `GET /metrics` (implies `metrics`), handoff, admin peer routes, `WorldCoordinator` | `axum`, `hmac`, `sha2`, `base64`, `tokio` | | `bots` | `bots` load-test client library + `game-bots` binary | `net` + `tokio-tungstenite` | @@ -909,6 +910,36 @@ double-crediting. A custom sink needs only crate-root imports: (`BoxFuture` is also re-exported from `world` and `net` for `ZoneOwnership` / `BarrierParticipant` impls). +**Facts to celld cells.** Durable aggregates live in celld cells; a zone +server sends its facts there with one `celld::CelldFactSink` (feature +`celld`), routing each fact by its command to a cell kind and shard: + +```rust,ignore +let sink = CelldFactSink::new(celld_url, InternalHttpSecret::new(secret)?, "zone-server", "zone:pit-1")? + .route(CelldFactRoute::by_principal::("crawler")) // Crawler:{principal} + .route(CelldFactRoute::by_input_field::("match", "match_id")) + .lifecycle_route("presence"); // zone.* → Presence:{account} +let (queue, feed) = fact_channel(capacity, RetryPolicy::until_settled(base, max)); +spawn_fact_worker(feed.with_max_concurrent_commits(n), sink); +``` + +Each fact is `POST {CELLD_URL}/{kind}/{shard}/{command}` with +`{commandId, input}`; the command id is `cell::command_id_for(service, +key)`, so a retry after a timeout replays the cell's ledger receipt +(`replayed`) instead of committing twice. The principal travels as +`x-user-id` and the command-ledger principal partition, the stamp as +`x-game-*` headers, and every request carries +`DISTRIBUTED_INTERNAL_SECRET`; the cell rebuilds the trusted session with +`cell::fact_session_from_headers` after checking the secret. Player +lifecycle facts route to presence, never to gameplay cells. celld slow, +unreachable or answering 5xx/`COMMAND_IN_PROGRESS` is `Retryable`: with +`RetryPolicy::until_settled` the fact stays in the queue (no local durable +fallback), the zone keeps ticking, a full queue answers `Backpressure`, +and after celld returns the backlog drains in per-principal order. An +unrouted command, a rejected command or a bad shard is `Permanent`. +`tests/celld_facts` proves this on `celld dev` against +`tests/celld/fact-worker`, including a crash and restart of celld. + **Throughput.** One worker per zone commits facts in emission order, one at a time (a slow commit delays every later fact). Set `ServerConfig::max_concurrent_commits` (or diff --git a/distributed_game/src/cell.rs b/distributed_game/src/cell.rs index f962400ff..d12ba304e 100644 --- a/distributed_game/src/cell.rs +++ b/distributed_game/src/cell.rs @@ -17,16 +17,25 @@ //! `InProgress` and `Internal` errors are retryable; everything else is //! permanent. +#[cfg(not(target_arch = "wasm32"))] use std::sync::Arc; +#[cfg(not(target_arch = "wasm32"))] use distributed::cell_host::{AggregateCell, CellCommandIdentity, CellDispatchError}; use distributed::microsvc::Session; -use distributed::{Aggregate, USER_ID_KEY}; +#[cfg(not(target_arch = "wasm32"))] +use distributed::Aggregate; +use distributed::USER_ID_KEY; use sha2::{Digest, Sha256}; -use crate::facts::{BoxFuture, Fact, FactError, FactReceipt, FactSink}; +use crate::facts::Fact; +#[cfg(not(target_arch = "wasm32"))] +use crate::facts::{BoxFuture, FactError, FactReceipt, FactSink}; -/// Fact sink backed by an aggregate cell. +/// Fact sink backed by an aggregate cell hosted in the zone server's own +/// process (native only: on wasm32 a cell is a Durable Object, reached over +/// the network with [`crate::celld::CelldFactSink`]). +#[cfg(not(target_arch = "wasm32"))] pub struct CellFactSink where A: Aggregate + Send + Sync + 'static, @@ -36,6 +45,7 @@ where system_principal: String, } +#[cfg(not(target_arch = "wasm32"))] impl CellFactSink where A: Aggregate + Send + Sync + 'static, @@ -87,6 +97,26 @@ pub fn trusted_session(principal: &str, fact: &Fact) -> Session { session } +/// The trusted session of a fact received by a celld cell over the internal +/// path ([`crate::celld::CelldFactSink`]): `x-user-id` plus the `x-game-*` +/// stamp headers. `header` reads one request header. Call it only after the +/// request authenticated with `DISTRIBUTED_INTERNAL_SECRET`; returns `None` +/// when the principal or a stamp field is missing (not a fact request). +pub fn fact_session_from_headers(header: impl Fn(&str) -> Option) -> Option { + let principal = header(USER_ID_KEY).filter(|value| !value.is_empty())?; + let mut vars = std::collections::HashMap::new(); + for (name, _) in crate::FactStamp::default().session_vars() { + vars.insert(name.clone(), header(&name)?); + } + let stamp = crate::FactStamp::from_session_vars(&vars)?; + let mut session = Session::new(); + session.set(USER_ID_KEY, principal); + for (key, value) in stamp.session_vars() { + session.set(key, value); + } + Some(session) +} + /// The [`FactStamp`](crate::FactStamp) a [`trusted_session`] carries /// (None for a session that did not come from a fact). Handlers use it to /// order facts ([`FactStamp::emission_order`](crate::FactStamp::emission_order)) @@ -95,6 +125,7 @@ pub fn session_stamp(session: &Session) -> Option { crate::FactStamp::from_session_vars(session.variables()) } +#[cfg(not(target_arch = "wasm32"))] fn classify(error: CellDispatchError) -> FactError { match error { CellDispatchError::InProgress | CellDispatchError::Internal(_) => { @@ -104,6 +135,7 @@ fn classify(error: CellDispatchError) -> FactError { } } +#[cfg(not(target_arch = "wasm32"))] impl FactSink for CellFactSink where A: Aggregate + Send + Sync + 'static, diff --git a/distributed_game/src/celld.rs b/distributed_game/src/celld.rs new file mode 100644 index 000000000..3884b5077 --- /dev/null +++ b/distributed_game/src/celld.rs @@ -0,0 +1,462 @@ +//! [`CelldFactSink`]: a zone server's facts, sent over the network to the +//! durable aggregates in celld cells (game-services decision D3). +//! +//! One sink per zone server routes every fact by its command +//! ([`FactCommand::COMMAND`](crate::FactCommand::COMMAND)) to a registered +//! [`CelldFactRoute`]: the cell kind and a shard function of the fact, giving +//! `POST {CELLD_URL}/{kind}/{shard}/{command}` (for example +//! `Crawler:{character}` or `Match:{match_id}`). Player lifecycle facts +//! (`zone.*`, [`crate::lifecycle`]) route to the presence cell of the +//! player's account, never to a gameplay cell +//! ([`CelldFactSink::lifecycle_route`]). +//! +//! - **Exactly one commit per key.** The command id is +//! [`command_id_for`]`(service_id, fact key)`, so a retry after a timeout +//! replays through the cell's command ledger (`replayed` receipt) instead +//! of committing twice. +//! - **Trusted identity over the internal path only.** Every request carries +//! `DISTRIBUTED_INTERNAL_SECRET`; the principal (bound by the host from the +//! verified ticket) travels as `x-user-id` and the command-ledger principal +//! partition, and the [`FactStamp`](crate::FactStamp) as `x-game-*` +//! headers. The cell rebuilds the session with +//! [`fact_session_from_headers`](crate::cell::fact_session_from_headers). +//! - **No local durable fallback.** celld slow or unreachable, a 5xx, or a +//! command still in progress is [`FactError::Retryable`]: the fact stays in +//! the zone's [`FactQueue`](crate::FactQueue) backlog and the worker backs +//! off (use [`RetryPolicy::until_settled`](crate::RetryPolicy::until_settled) +//! so an outage never drops a fact); a full queue surfaces to the zone as +//! [`EmitError::Backpressure`](crate::EmitError::Backpressure). An unrouted +//! command and a rejected command are [`FactError::Permanent`]. + +use std::sync::Arc; +use std::time::Duration; + +use distributed::cell_host::{ + InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, + CELL_SERVICE_ID_HEADER, +}; +use distributed::USER_ID_KEY; +use serde_json::{json, Value}; + +use crate::cell::command_id_for; +use crate::facts::{BoxFuture, Fact, FactCommand, FactError, FactReceipt, FactSink}; +use crate::lifecycle::LIFECYCLE_COMMANDS; + +/// Default per-request timeout. +pub const DEFAULT_CELLD_FACT_TIMEOUT: Duration = Duration::from_secs(10); + +type ShardFn = Arc Option + Send + Sync>; + +/// Where facts of one command go: a celld cell kind and a shard function. +#[derive(Clone)] +pub struct CelldFactRoute { + command: String, + kind: String, + shard: ShardFn, +} + +impl CelldFactRoute { + /// Route `command` to `{kind}/{shard(fact, principal)}`. `principal` is the + /// fact's verified principal, or the sink's system principal for zone + /// facts. Return `None` when the fact cannot be routed (settles + /// `Permanent`). + pub fn new( + command: impl Into, + kind: impl Into, + shard: impl Fn(&Fact, &str) -> Option + Send + Sync + 'static, + ) -> Self { + Self { + command: command.into(), + kind: kind.into(), + shard: Arc::new(shard), + } + } + + /// Route the typed fact `C` to the cell of its principal (one cell per + /// player: `PlayerProgress:{character}`, `Crawler:{principal}`). + pub fn by_principal(kind: impl Into) -> Self { + Self::new(C::COMMAND, kind, |_, principal| Some(principal.to_string())) + } + + /// Route the typed fact `C` by a field of its input (one cell per match: + /// `Match:{match_id}` with `field = "match_id"`). + pub fn by_input_field(kind: impl Into, field: &'static str) -> Self { + Self::new(C::COMMAND, kind, move |fact, _| { + match fact.input().get(field)? { + Value::String(value) => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + _ => None, + } + }) + } + + /// The routed command. + pub fn command(&self) -> &str { + &self.command + } + + /// The cell kind (URL segment). + pub fn kind(&self) -> &str { + &self.kind + } +} + +/// Fact sink posting each fact to its celld cell over the internal path. +#[derive(Clone)] +pub struct CelldFactSink { + client: reqwest::Client, + base: String, + secret: InternalHttpSecret, + service_id: String, + system_principal: String, + routes: Vec, +} + +impl CelldFactSink { + /// A sink for `celld_url` under command-ledger `service_id`; zone facts + /// without a player use `system_principal`. + pub fn new( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + system_principal: impl Into, + ) -> Result { + Self::with_timeout( + celld_url, + secret, + service_id, + system_principal, + DEFAULT_CELLD_FACT_TIMEOUT, + ) + } + + /// As [`new`](Self::new) with an explicit per-request timeout. + pub fn with_timeout( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + system_principal: impl Into, + timeout: Duration, + ) -> Result { + let base = celld_url.into().trim_end_matches('/').to_string(); + if !(base.starts_with("http://") || base.starts_with("https://")) { + return Err(format!("celld URL must be http(s): {base}")); + } + let service_id = service_id.into(); + let system_principal = system_principal.into(); + if service_id.trim().is_empty() || system_principal.trim().is_empty() { + return Err("celld fact sink needs a service id and a system principal".into()); + } + let client = reqwest::Client::builder() + .timeout(timeout) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("celld fact client: {error}"))?; + Ok(Self { + client, + base, + secret, + service_id, + system_principal, + routes: Vec::new(), + }) + } + + /// Add a route. A later route for the same command replaces the earlier. + pub fn route(mut self, route: CelldFactRoute) -> Self { + self.routes + .retain(|existing| existing.command != route.command); + self.routes.push(route); + self + } + + /// Route every player lifecycle fact (`zone.*`) to the presence cell + /// `{kind}/{principal}` (the player's account), not to gameplay cells. + pub fn lifecycle_route(mut self, kind: impl Into) -> Self { + let kind = kind.into(); + for command in LIFECYCLE_COMMANDS { + self = self.route(CelldFactRoute::new( + command, + kind.clone(), + |_, principal| Some(principal.to_string()), + )); + } + self + } + + /// The route a fact would take, if any. + pub fn route_for(&self, command: &str) -> Option<&CelldFactRoute> { + self.routes.iter().find(|route| route.command == command) + } + + /// The request a fact becomes: `(url, command id)`. + pub fn target(&self, fact: &Fact) -> Result<(String, String), FactError> { + let route = self.route_for(fact.command()).ok_or_else(|| { + FactError::Permanent(format!( + "no celld route for fact command `{}`", + fact.command() + )) + })?; + let principal = fact.principal().unwrap_or(&self.system_principal); + let shard = (route.shard)(fact, principal) + .filter(|shard| valid_segment(shard)) + .ok_or_else(|| { + FactError::Permanent(format!( + "fact `{}` has no valid `{}` shard", + fact.key(), + route.kind + )) + })?; + Ok(( + format!( + "{}/{}/{}/{}", + self.base, + encode_segment(&route.kind), + encode_segment(&shard), + encode_segment(fact.command()) + ), + command_id_for(&self.service_id, fact.key()), + )) + } +} + +/// Path segments are non-empty, short, and printable. +fn valid_segment(segment: &str) -> bool { + !segment.is_empty() + && segment.len() <= 256 + && segment.bytes().all(|byte| (0x21..=0x7e).contains(&byte)) +} + +fn encode_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b':') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +fn classify(status: u16, body: &Value) -> FactError { + let code = body.get("code").and_then(Value::as_str).unwrap_or(""); + let message = body + .get("error") + .and_then(Value::as_str) + .unwrap_or("celld rejected the fact"); + let detail = format!("HTTP {status} {code}: {message}"); + match (status, code) { + (_, "COMMAND_IN_PROGRESS") | (_, "INTERNAL") => FactError::Retryable(detail), + (408 | 429, _) => FactError::Retryable(detail), + (500..=599, _) => FactError::Retryable(detail), + _ => FactError::Permanent(detail), + } +} + +impl FactSink for CelldFactSink { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (url, command_id) = self.target(fact)?; + let principal = fact.principal().unwrap_or(&self.system_principal); + let mut request = self + .client + .post(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .header(CELL_SERVICE_ID_HEADER, &self.service_id) + .header(CELL_PRINCIPAL_PARTITION_HEADER, principal) + .header(USER_ID_KEY, principal) + .json(&json!({ "commandId": command_id, "input": fact.input() })); + for (name, value) in fact.stamp().session_vars() { + request = request.header(name, value); + } + let response = request.send().await.map_err(|error| { + FactError::Retryable(format!("celld unreachable at {url}: {error}")) + })?; + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if !(200..300).contains(&status) { + return Err(classify(status, &body)); + } + let replayed = body + .pointer("/receipt/replayed") + .and_then(Value::as_bool) + .unwrap_or(false); + Ok(FactReceipt { + payload: body.get("payload").cloned().unwrap_or(Value::Null), + replayed, + }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::stamp::FactStamp; + + const SECRET: &str = "test-only-internal-secret-change-me-2026"; + + #[derive(serde::Serialize)] + struct Kill { + mob: String, + } + impl FactCommand for Kill { + const COMMAND: &'static str = "crawler.record_kill"; + } + #[derive(serde::Serialize)] + struct Result_ { + match_id: String, + } + impl FactCommand for Result_ { + const COMMAND: &'static str = "match.record_result"; + } + + fn sink() -> CelldFactSink { + CelldFactSink::new( + "http://127.0.0.1:9/", + InternalHttpSecret::new(SECRET).unwrap(), + "zone-server", + "zone:pit-1", + ) + .unwrap() + .route(CelldFactRoute::by_principal::("crawler")) + .route(CelldFactRoute::by_input_field::( + "match", "match_id", + )) + .lifecycle_route("presence") + } + + fn fact(command: &str, input: Value, principal: Option<&str>) -> Fact { + Fact::trusted( + format!("{command}:pit-1:1:0"), + command, + input, + principal.map(str::to_string), + FactStamp::default(), + ) + } + + #[test] + fn routes_by_command_to_kind_and_shard_with_a_stable_command_id() { + let sink = sink(); + let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); + let (url, id) = sink.target(&kill).unwrap(); + assert_eq!(url, "http://127.0.0.1:9/crawler/alice/crawler.record_kill"); + assert_eq!(id, command_id_for("zone-server", kill.key())); + assert_eq!(sink.target(&kill).unwrap().1, id, "stable across retries"); + + let result = fact("match.record_result", json!({ "match_id": "m/7" }), None); + let (url, _) = sink.target(&result).unwrap(); + assert_eq!( + url, "http://127.0.0.1:9/match/m%2F7/match.record_result", + "a shard cannot add path segments" + ); + let spaced = fact("match.record_result", json!({ "match_id": "m 7" }), None); + assert!(matches!(sink.target(&spaced), Err(FactError::Permanent(_)))); + + for command in LIFECYCLE_COMMANDS { + let lifecycle = fact(command, json!({ "zone": "pit-1" }), Some("alice")); + let (url, _) = sink.target(&lifecycle).unwrap(); + assert_eq!(url, format!("http://127.0.0.1:9/presence/alice/{command}")); + } + } + + #[test] + fn unknown_commands_and_unroutable_facts_are_permanent() { + let sink = sink(); + let unknown = fact("guild.whatever", json!({}), Some("alice")); + assert!(matches!( + sink.target(&unknown), + Err(FactError::Permanent(_)) + )); + let missing = fact("match.record_result", json!({}), None); + assert!(matches!( + sink.target(&missing), + Err(FactError::Permanent(_)) + )); + } + + #[test] + fn classifies_cell_responses() { + assert!(matches!( + classify(409, &json!({ "code": "COMMAND_IN_PROGRESS" })), + FactError::Retryable(_) + )); + assert!(matches!( + classify(503, &Value::Null), + FactError::Retryable(_) + )); + assert!(matches!( + classify(500, &json!({ "code": "INTERNAL" })), + FactError::Retryable(_) + )); + assert!(matches!( + classify(400, &json!({ "code": "BAD_REQUEST" })), + FactError::Permanent(_) + )); + assert!(matches!( + classify(401, &json!({ "code": "UNAUTHORIZED" })), + FactError::Permanent(_) + )); + assert!(matches!( + classify(422, &json!({ "code": "REJECTED" })), + FactError::Permanent(_) + )); + } + + #[tokio::test] + async fn an_unreachable_celld_is_retryable() { + let sink = sink(); + let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); + assert!(matches!( + sink.commit(&kill).await, + Err(FactError::Retryable(_)) + )); + } + + #[test] + fn the_cell_rebuilds_the_trusted_fact_session_from_the_headers() { + let stamp = FactStamp { + zone_id: "pit-1".into(), + tick: 42, + content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_000, + activation: 3, + config_version: "cfg-2".into(), + }; + let fact = Fact::trusted( + "k", + "crawler.record_kill", + json!({}), + Some("alice".into()), + stamp.clone(), + ); + // Exactly the headers `commit` sends (besides the internal ones). + let mut headers = std::collections::HashMap::new(); + headers.insert(USER_ID_KEY.to_string(), "alice".to_string()); + for (name, value) in fact.stamp().session_vars() { + headers.insert(name, value); + } + let session = + crate::cell::fact_session_from_headers(|name| headers.get(name).cloned()).unwrap(); + assert_eq!(session.user_id(), Some("alice")); + assert_eq!(crate::cell::session_stamp(&session), Some(stamp)); + headers.remove("x-game-tick"); + assert!( + crate::cell::fact_session_from_headers(|name| headers.get(name).cloned()).is_none() + ); + headers.insert("x-game-tick".into(), "42".into()); + headers.remove(USER_ID_KEY); + assert!( + crate::cell::fact_session_from_headers(|name| headers.get(name).cloned()).is_none() + ); + } + + #[test] + fn rejects_bad_configuration() { + let secret = || InternalHttpSecret::new(SECRET).unwrap(); + assert!(CelldFactSink::new("ftp://x", secret(), "s", "p").is_err()); + assert!(CelldFactSink::new("http://x", secret(), " ", "p").is_err()); + } +} diff --git a/distributed_game/src/facts.rs b/distributed_game/src/facts.rs index 844759133..baf00eb59 100644 --- a/distributed_game/src/facts.rs +++ b/distributed_game/src/facts.rs @@ -260,6 +260,19 @@ impl Default for RetryPolicy { } impl RetryPolicy { + /// Retry retryable failures until the fact settles (backing off up to + /// `max_backoff`): a network sink whose target is down keeps the fact + /// pending in the queue, so the zone sees backpressure at capacity + /// instead of losing durable outcomes. Permanent failures still settle + /// immediately. + pub fn until_settled(base_backoff: Duration, max_backoff: Duration) -> Self { + Self { + max_attempts: u32::MAX, + base_backoff, + max_backoff, + } + } + /// Backoff before attempt `attempt + 1` (attempt >= 1). pub fn backoff(&self, attempt: u32) -> Duration { let factor = 1u32 << attempt.saturating_sub(1).min(20); diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index c9b2f6598..ea85e92e2 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -54,6 +54,8 @@ pub mod authoring; pub mod bots; #[cfg(feature = "cell")] pub mod cell; +#[cfg(feature = "celld")] +pub mod celld; pub mod claims; pub mod clock; pub mod codec; diff --git a/distributed_game/tests/celld_facts/main.rs b/distributed_game/tests/celld_facts/main.rs new file mode 100644 index 000000000..3e4b44e1e --- /dev/null +++ b/distributed_game/tests/celld_facts/main.rs @@ -0,0 +1,574 @@ +//! Zone facts over the network into celld cells (`CelldFactSink`). +//! +//! Owns an isolated `celld dev` process (no Docker) serving +//! `tests/celld/fact-worker` (Kills, Match and Presence cells), and kills only +//! the processes it starts. Build the worker first: +//! +//! (cd tests/celld/fact-worker && worker-build --release) +//! cargo test -p distributed_game --features celld,runtime --test celld_facts +//! +//! `CELLD` selects the binary (default `celld` on PATH); `celld dev` needs +//! esbuild (`CELLD_ESBUILD`). Without a built worker the test skips, unless +//! `CELLD_FACTS_REQUIRED=1` (CI), which turns that into a failure. + +use std::io::Write as _; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use distributed::cell_host::InternalHttpSecret; +use distributed_game::celld::{CelldFactRoute, CelldFactSink}; +use distributed_game::lifecycle::{JoinVia, LeftReason, PlayerJoined, PlayerLeft}; +use distributed_game::{ + fact_channel, spawn_fact_worker, EmitError, Fact, FactCommand, FactError, FactOutcome, + FactQueue, FactSettlement, FactSink, FactStamp, RetryPolicy, +}; +use serde_json::{json, Value}; + +const SECRET: &str = "test-only-internal-secret-change-me-2026"; +const SERVICE: &str = "zone-server-test"; + +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "kills.record")] +struct Kill { + mob: String, +} + +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "match.record_result")] +struct MatchResult { + match_id: String, + winner_slot: u32, +} + +fn worker_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../tests/celld/fact-worker") +} + +struct Celld { + project: PathBuf, + port: u16, + child: Option, + generation: u32, +} + +impl Celld { + fn start_new() -> Option { + let build = worker_dir().join("build"); + if !build.join("worker/shim.mjs").exists() { + assert!( + std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), + "CELLD_FACTS_REQUIRED is set but the fact worker is not built" + ); + eprintln!("skipping celld fact routing: build tests/celld/fact-worker first"); + return None; + } + let project = std::env::temp_dir().join(format!( + "distributed-celld-facts-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + copy_dir(&build, &project.join("build")); + std::fs::copy( + worker_dir().join("wrangler.jsonc"), + project.join("wrangler.jsonc"), + ) + .unwrap(); + let port = TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let mut celld = Self { + project, + port, + child: None, + generation: 0, + }; + if let Err(reason) = celld.start() { + assert!( + std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), + "celld dev is required but did not start: {reason}" + ); + eprintln!("skipping celld fact routing: {reason}"); + return None; + } + Some(celld) + } + + fn base(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Start `celld dev`; `Err` when it is unavailable or exits early (no + /// celld on PATH, or no esbuild for its bundler). + fn start(&mut self) -> Result<(), String> { + self.generation += 1; + let binary = std::env::var("CELLD").unwrap_or_else(|_| "celld".into()); + let help = Command::new(&binary) + .args(["dev", "--help"]) + .output() + .map_err(|error| format!("cannot run `{binary}`: {error}"))?; + let mut args = vec![ + "dev".to_string(), + self.project.display().to_string(), + "--host".into(), + "127.0.0.1".into(), + "--port".into(), + self.port.to_string(), + "--logs".into(), + ]; + if String::from_utf8_lossy(&help.stdout).contains("--no-watch") { + args.push("--no-watch".into()); + } + let log_path = self.project.join(format!("celld-{}.log", self.generation)); + let log = std::fs::File::create(&log_path).unwrap(); + let child = Command::new(&binary) + .args(&args) + .env("RUST_LOG", "warn") + .stdin(Stdio::null()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .map_err(|error| format!("cannot start `{binary} dev`: {error}"))?; + self.child = Some(child); + let deadline = Instant::now() + Duration::from_secs(90); + while Instant::now() < deadline { + if std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() + && blocking_get(&format!("{}/health", self.base())).is_some_and(|s| s == 200) + { + return Ok(()); + } + let exited = self + .child + .as_mut() + .and_then(|child| child.try_wait().ok().flatten()); + if let Some(status) = exited { + self.child = None; + let log = std::fs::read_to_string(&log_path).unwrap_or_default(); + return Err(format!( + "celld dev exited ({status}): {}", + log.lines().rev().take(3).collect::>().join(" | ") + )); + } + std::thread::sleep(Duration::from_millis(200)); + } + Err(format!( + "celld dev did not become ready; logs in {}", + self.project.display() + )) + } + + /// Kill the launcher this test started and every descendant. + fn crash(&mut self) { + let Some(mut child) = self.child.take() else { + return; + }; + let root = child.id(); + let _ = Command::new("kill") + .args(["-STOP", &root.to_string()]) + .status(); + let table = Command::new("ps") + .args(["-axo", "pid=,ppid="]) + .output() + .unwrap(); + let pairs = String::from_utf8_lossy(&table.stdout) + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + Some(( + parts.next()?.parse::().ok()?, + parts.next()?.parse::().ok()?, + )) + }) + .collect::>(); + let mut doomed = vec![root]; + let mut index = 0; + while index < doomed.len() { + let parent = doomed[index]; + doomed.extend( + pairs + .iter() + .filter(|(_, ppid)| *ppid == parent) + .map(|(pid, _)| *pid), + ); + index += 1; + } + for pid in doomed.iter().rev() { + let _ = Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status(); + } + let _ = child.wait(); + // The port is released once the processes are gone. + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline + && std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() + { + std::thread::sleep(Duration::from_millis(100)); + } + } +} + +impl Drop for Celld { + fn drop(&mut self) { + self.crash(); + if !std::thread::panicking() { + let _ = std::fs::remove_dir_all(&self.project); + } else { + let _ = writeln!( + std::io::stderr(), + "celld artifacts kept in {}", + self.project.display() + ); + } + } +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), target).unwrap(); + } + } +} + +fn blocking_get(url: &str) -> Option { + let url = url.strip_prefix("http://")?; + let (host, path) = url.split_once('/')?; + let mut stream = std::net::TcpStream::connect(host).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok()?; + write!( + stream, + "GET /{path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n" + ) + .ok()?; + let mut response = String::new(); + std::io::Read::read_to_string(&mut stream, &mut response).ok()?; + response.split_whitespace().nth(1)?.parse().ok() +} + +fn sink(base: &str, timeout: Duration) -> CelldFactSink { + CelldFactSink::with_timeout( + base, + InternalHttpSecret::new(SECRET).unwrap(), + SERVICE, + "zone:pit-1", + timeout, + ) + .unwrap() + .route(CelldFactRoute::by_principal::("kills")) + .route(CelldFactRoute::by_input_field::( + "match", "match_id", + )) + .lifecycle_route("presence") +} + +fn stamp(tick: u64) -> FactStamp { + FactStamp { + zone_id: "pit-1".into(), + tick, + content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_000 + tick, + activation: 1, + config_version: "cfg-1".into(), + } +} + +fn fact(key: &str, input: &C, principal: Option<&str>, tick: u64) -> Fact { + Fact::trusted( + key, + C::COMMAND, + serde_json::to_value(input).unwrap(), + principal.map(str::to_string), + stamp(tick), + ) +} + +async fn state(base: &str, kind: &str, shard: &str) -> Value { + reqwest::Client::new() + .get(format!("{base}/{kind}/{shard}")) + .header("x-distributed-internal-secret", SECRET) + .send() + .await + .unwrap() + .json() + .await + .unwrap() +} + +/// Drain settlements until `count` arrive (or time out). +async fn settle(queue: &mut FactQueue, count: usize, within: Duration) -> Vec { + let deadline = Instant::now() + within; + let mut settled = Vec::new(); + while settled.len() < count && Instant::now() < deadline { + settled.extend(queue.drain()); + tokio::time::sleep(Duration::from_millis(50)).await; + } + settled +} + +fn committed(settlement: &FactSettlement) -> bool { + matches!(settlement.outcome, FactOutcome::Committed(_)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn zone_facts_route_to_three_cell_kinds_and_survive_a_celld_outage() { + let Some(mut celld) = Celld::start_new() else { + return; + }; + let base = celld.base(); + let policy = RetryPolicy::until_settled(Duration::from_millis(100), Duration::from_millis(500)); + let (mut queue, feed) = fact_channel(4, policy); + let worker = spawn_fact_worker( + feed.with_max_concurrent_commits(2), + sink(&base, Duration::from_secs(10)), + ); + + // 1. One zone, one sink, three aggregate kinds (lifecycle -> presence). + let joined = PlayerJoined { + zone: "pit-1".into(), + via: JoinVia::Ticket, + from_zone: None, + }; + queue + .try_send(fact( + "zone.player_joined:pit-1:1:p0:0", + &joined, + Some("alice"), + 1, + )) + .unwrap(); + queue + .try_send(fact( + "kills.record:pit-1:1:k1", + &Kill { mob: "m1".into() }, + Some("alice"), + 2, + )) + .unwrap(); + queue + .try_send(fact( + "kills.record:pit-1:1:k2", + &Kill { mob: "m2".into() }, + Some("bob"), + 3, + )) + .unwrap(); + queue + .try_send(fact( + "match.record_result:pit-1:1:m-7", + &MatchResult { + match_id: "m-7".into(), + winner_slot: 2, + }, + None, + 4, + )) + .unwrap(); + let settled = settle(&mut queue, 4, Duration::from_secs(30)).await; + assert_eq!(settled.len(), 4, "{settled:?}"); + assert!(settled.iter().all(committed), "{settled:?}"); + let left = PlayerLeft { + zone: "pit-1".into(), + reason: LeftReason::Left, + }; + queue + .try_send(fact( + "zone.player_left:pit-1:1:p0:1", + &left, + Some("alice"), + 5, + )) + .unwrap(); + assert!(settle(&mut queue, 1, Duration::from_secs(30)) + .await + .iter() + .all(committed)); + + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1"]) + ); + assert_eq!( + state(&base, "kills", "bob").await["state"]["mobs"], + json!(["m2"]) + ); + assert_eq!( + state(&base, "match", "m-7").await["state"]["results"], + json!([2]) + ); + assert_eq!( + state(&base, "presence", "alice").await["state"]["transitions"], + json!(["joined:pit-1:ticket", "left:pit-1:left"]), + "lifecycle facts go to presence, not to the player's kill cell" + ); + + // 2. The same key twice (a retry after a lost response) commits once and + // replays the ledger receipt. + let direct = sink(&base, Duration::from_secs(10)); + let again = fact( + "kills.record:pit-1:1:k1", + &Kill { mob: "m1".into() }, + Some("alice"), + 2, + ); + let replay = direct.commit(&again).await.unwrap(); + assert!(replay.replayed); + assert_eq!(state(&base, "kills", "alice").await["events"], 1); + // Forced timeout after commit: the first attempt gives up before the + // response; the retry finds the commit in the ledger. + let hasty = sink(&base, Duration::from_millis(1)); + let late = fact( + "kills.record:pit-1:1:k3", + &Kill { mob: "m3".into() }, + Some("alice"), + 6, + ); + assert!(matches!( + hasty.commit(&late).await, + Err(FactError::Retryable(_)) + )); + tokio::time::sleep(Duration::from_millis(500)).await; + let receipt = direct.commit(&late).await.unwrap(); + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1", "m3"]), + "exactly one commit for the key (replayed: {})", + receipt.replayed + ); + + // 3. An unknown command settles Permanent without a network call. + queue + .try_send(Fact::trusted( + "guild.promote:pit-1:1:x", + "guild.promote", + json!({}), + Some("alice".into()), + stamp(7), + )) + .unwrap(); + let unknown = settle(&mut queue, 1, Duration::from_secs(10)).await; + assert!( + matches!( + &unknown[0].outcome, + FactOutcome::Failed { + error: FactError::Permanent(_), + attempts: 1 + } + ), + "{unknown:?}" + ); + + // 4. The cell worker rejects requests without the internal secret. + let forged = reqwest::Client::new() + .post(format!("{base}/kills/alice/kills.record")) + .header("x-distributed-service-id", SERVICE) + .header("x-distributed-principal-partition", "alice") + .header("x-user-id", "alice") + .json(&json!({ "commandId": "0190a000-0000-7000-8000-000000000001", "input": { "mob": "x" } })) + .send() + .await + .unwrap(); + assert_eq!(forged.status(), 401); + let forged_secret = reqwest::Client::new() + .post(format!("{base}/kills/alice/kills.record")) + .header("x-distributed-internal-secret", "forged-secret-for-red-team-request-0000") + .json(&json!({ "commandId": "0190a000-0000-7000-8000-000000000002", "input": { "mob": "x" } })) + .send() + .await + .unwrap(); + assert_eq!(forged_secret.status(), 401); + // With the secret but without the fact headers, handlers refuse. + let not_a_fact = reqwest::Client::new() + .post(format!("{base}/kills/alice/kills.record")) + .header("x-distributed-internal-secret", SECRET) + .header("x-distributed-service-id", SERVICE) + .header("x-distributed-principal-partition", "alice") + .header("x-user-id", "alice") + .json(&json!({ "commandId": "0190a000-0000-7000-8000-000000000003", "input": { "mob": "x" } })) + .send() + .await + .unwrap(); + assert!( + not_a_fact.status().is_client_error(), + "{}", + not_a_fact.status() + ); + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1", "m3"]) + ); + + // 5. celld down: facts stay pending, the zone is never blocked, the + // queue reports Backpressure at capacity. + celld.crash(); + let outage = [ + ("alice", "a1", 10), + ("bob", "b1", 11), + ("alice", "a2", 12), + ("bob", "b2", 13), + ]; + for (principal, mob, tick) in outage { + let started = Instant::now(); + queue + .try_send(fact( + &format!("kills.record:pit-1:1:{mob}"), + &Kill { mob: mob.into() }, + Some(principal), + tick, + )) + .unwrap(); + assert!( + started.elapsed() < Duration::from_millis(50), + "emitting never waits on celld" + ); + } + assert_eq!( + queue.try_send(fact( + "kills.record:pit-1:1:a3", + &Kill { mob: "a3".into() }, + Some("alice"), + 14 + )), + Err(EmitError::Backpressure) + ); + assert!( + settle(&mut queue, 1, Duration::from_secs(3)) + .await + .is_empty(), + "nothing settles while celld is down" + ); + assert_eq!(queue.in_flight(), 4); + + // celld back: the backlog drains, each principal in emission order. + celld.start().expect("celld restarts"); + let drained = settle(&mut queue, 4, Duration::from_secs(60)).await; + assert_eq!(drained.len(), 4, "{drained:?}"); + assert!(drained.iter().all(committed), "{drained:?}"); + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1", "m3", "a1", "a2"]) + ); + assert_eq!( + state(&base, "kills", "bob").await["state"]["mobs"], + json!(["m2", "b1", "b2"]) + ); + let counts = drained + .iter() + .map(|settlement| settlement.key.clone()) + .collect::>(); + assert_eq!(counts.len(), 4); + + drop(queue); + let _ = tokio::time::timeout(Duration::from_secs(5), worker).await; +} diff --git a/tests/celld/fact-worker/Cargo.toml b/tests/celld/fact-worker/Cargo.toml new file mode 100644 index 000000000..4335b0ce5 --- /dev/null +++ b/tests/celld/fact-worker/Cargo.toml @@ -0,0 +1,24 @@ +[workspace] +members = ["."] +resolver = "2" + +[package] +name = "fact-cell-worker" +version = "0.1.0" +edition = "2021" +publish = false +description = "workers-rs cells for zone facts sent over the network by CelldFactSink (Kills, Match, Presence)" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +console_error_panic_hook = "0.1" +distributed = { path = "../../..", default-features = false, features = ["workers-rs"] } +distributed_game = { path = "../../../distributed_game", default-features = false, features = ["cell"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +worker = { version = "0.8", features = ["queue"] } + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/tests/celld/fact-worker/src/lib.rs b/tests/celld/fact-worker/src/lib.rs new file mode 100644 index 000000000..77d5f6719 --- /dev/null +++ b/tests/celld/fact-worker/src/lib.rs @@ -0,0 +1,455 @@ +//! Durable aggregates of a zone server, as celld cells fed by +//! `distributed_game::celld::CelldFactSink` over the internal HTTP path. +//! +//! - `Kills:{principal}`: `kills.record` (one player's kills, in fact order). +//! - `Match:{match_id}`: `match.record_result`. +//! - `Presence:{principal}`: the runtime lifecycle facts `zone.player_joined` +//! and `zone.player_left` (lifecycle facts go to presence, never to the +//! player's gameplay cells). +//! +//! Every route requires `DISTRIBUTED_INTERNAL_SECRET`. The handler's session +//! is rebuilt from the trusted fact headers (`fact_session_from_headers`): +//! the principal and the `FactStamp`; handlers refuse anything else. + +use distributed::cell_host::{ + AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, + InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, +}; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, Session, +}; +use distributed::{sourced, Aggregate, Entity}; +use distributed_game::cell::{fact_session_from_headers, session_stamp}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use worker::*; + +fn rejected(error: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(error.to_string()) +} + +/// Only facts (principal + stamp from the trusted headers) may run. +fn is_fact(ctx: &CausalCommandContext<'_, A>) -> bool { + ctx.session().user_id().is_some() && session_stamp(ctx.session()).is_some() +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct Recorded { + pub count: u32, +} + +// ---------------------------------------------------------------- Kills + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct Kills { + entity: Entity, + mobs: Vec, +} + +#[sourced(entity, aggregate_type = "Kills", events = "KillsEvent")] +impl Kills { + #[event("kills.recorded", version = 1)] + fn record(&mut self, principal: String, mob: String) { + if self.entity.id().is_empty() { + self.entity.set_id(principal); + } + self.mobs.push(mob); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct KillInput { + pub mob: String, +} + +struct RecordKill; +impl PortableCommand for RecordKill +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "kills.record", + )) + .guarded(is_fact::, handle_kill) + } +} + +async fn handle_kill( + ctx: &CausalCommandContext<'_, Kills>, + input: KillInput, +) -> std::result::Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + let repo = ctx.repo(); + let mut kills = match repo.get(&principal).await? { + Some(kills) => kills, + None => repo.create(), + }; + kills.record(principal, input.mob).map_err(rejected)?; + let count = kills.mobs.len() as u32; + repo.commit(kills)?.succeeded(Recorded { count }) +} + +// ---------------------------------------------------------------- Match + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct Match { + entity: Entity, + results: Vec, +} + +#[sourced(entity, aggregate_type = "Match", events = "MatchEvent")] +impl Match { + #[event("match.result_recorded", version = 1)] + fn record(&mut self, match_id: String, winner_slot: u32) { + if self.entity.id().is_empty() { + self.entity.set_id(match_id); + } + self.results.push(winner_slot); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct MatchResultInput { + pub match_id: String, + pub winner_slot: u32, +} + +struct RecordResult; +impl PortableCommand for RecordResult +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "match.record_result", + )) + .guarded(is_fact::, handle_result) + } +} + +async fn handle_result( + ctx: &CausalCommandContext<'_, Match>, + input: MatchResultInput, +) -> std::result::Result>, HandlerError> { + let repo = ctx.repo(); + let mut found = match repo.get(&input.match_id).await? { + Some(found) => found, + None => repo.create(), + }; + found + .record(input.match_id, input.winner_slot) + .map_err(rejected)?; + let count = found.results.len() as u32; + repo.commit(found)?.succeeded(Recorded { count }) +} + +// ---------------------------------------------------------------- Presence + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct Presence { + entity: Entity, + transitions: Vec, +} + +#[sourced(entity, aggregate_type = "Presence", events = "PresenceEvent")] +impl Presence { + #[event("presence.changed", version = 1)] + fn change(&mut self, principal: String, transition: String) { + if self.entity.id().is_empty() { + self.entity.set_id(principal); + } + self.transitions.push(transition); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct JoinedInput { + pub zone: String, + pub via: String, + pub from_zone: Option, +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct LeftInput { + pub zone: String, + pub reason: String, +} + +struct Joined; +impl PortableCommand for Joined +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + distributed_game::lifecycle::PLAYER_JOINED, + )) + .guarded(is_fact::, handle_joined) + } +} + +struct Left; +impl PortableCommand for Left +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + distributed_game::lifecycle::PLAYER_LEFT, + )) + .guarded(is_fact::, handle_left) + } +} + +async fn handle_joined( + ctx: &CausalCommandContext<'_, Presence>, + input: JoinedInput, +) -> std::result::Result>, HandlerError> { + let via = match input.from_zone { + Some(from) => format!("{}<-{from}", input.via), + None => input.via, + }; + presence(ctx, format!("joined:{}:{via}", input.zone)).await +} + +async fn handle_left( + ctx: &CausalCommandContext<'_, Presence>, + input: LeftInput, +) -> std::result::Result>, HandlerError> { + presence(ctx, format!("left:{}:{}", input.zone, input.reason)).await +} + +async fn presence( + ctx: &CausalCommandContext<'_, Presence>, + transition: String, +) -> std::result::Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + let repo = ctx.repo(); + let mut presence = match repo.get(&principal).await? { + Some(presence) => presence, + None => repo.create(), + }; + presence.change(principal, transition).map_err(rejected)?; + let count = presence.transitions.len() as u32; + repo.commit(presence)?.succeeded(Recorded { count }) +} + +// ---------------------------------------------------------------- Hosting + +async fn dispatch( + env: &Env, + cell: &AggregateCell, + command: &str, + req: &mut Request, +) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => return error_response(CellDispatchError::BadRequest("invalid JSON".into())), + }; + let parsed = match CellWaitPathRequest::parse(body) { + Ok(parsed) => parsed, + Err(error) => return error_response(CellDispatchError::BadRequest(error)), + }; + let header = |name: &str| req.headers().get(name).ok().flatten(); + let identity = match ( + header(CELL_SERVICE_ID_HEADER), + header(CELL_PRINCIPAL_PARTITION_HEADER), + ) { + (Some(service), Some(partition)) => { + match CellCommandIdentity::new(service, partition, &parsed.command_id) { + Ok(identity) => identity, + Err(error) => return error_response(error), + } + } + _ => return error_response(CellDispatchError::Unauthorized), + }; + let session = fact_session_from_headers(header).unwrap_or_else(Session::new); + let result = cell + .dispatch_idempotent(command, &identity, parsed.input, session) + .await; + if let Err(error) = cell.drain_outbox(env).await { + worker::console_error!("post-commit drain deferred: {error}"); + } + match result { + Ok(dispatch) => json_status( + json!({ + "payload": dispatch.payload(), + "receipt": { "commandId": dispatch.command_id(), "replayed": dispatch.replayed() }, + }), + 200, + ), + Err(error) => error_response(error), + } +} + +async fn route( + env: &Env, + cell: &AggregateCell, + mut req: Request, + state: impl Fn(&A) -> Value, +) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + if let Err(error) = authenticate(&req, env) { + return error_response(error); + } + let url = req.url()?; + let parts: Vec = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .map(str::to_string) + .collect(); + match (req.method(), parts.get(2)) { + (Method::Get, None) => { + let loaded = cell + .load() + .await + .map_err(|error| Error::RustError(error.to_string()))?; + let events = cell + .history() + .await + .map_err(|error| Error::RustError(error.to_string()))? + .len(); + json_status( + json!({ "events": events, "state": loaded.as_ref().map(state) }), + 200, + ) + } + (Method::Post, Some(command)) => { + let command = command.clone(); + dispatch(env, cell, &command, &mut req).await + } + _ => json_status(json!({ "error": "not found" }), 404), + } +} + +macro_rules! cell_object { + ($name:ident, $aggregate:ty, [$($command:expr),*], $state:expr) => { + #[durable_object] + pub struct $name { + cell: AggregateCell<$aggregate>, + env: Env, + } + + impl DurableObject for $name { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let outbox = CelldOutbox::from_env(&env, "OUTBOX").expect("OUTBOX Queue binding"); + let cell = AggregateCell::<$aggregate>::from_state(state) + .expect("cell identity") + $(.mount($command))* + .with_celld_outbox(outbox); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, $state).await + } + + async fn alarm(&self) -> Result { + self.cell + .alarm(&self.env) + .await + .map_err(|error| Error::RustError(error.to_string()))? + .drain + .deferred + .iter() + .for_each(|error| worker::console_error!("drain deferred: {error}")); + Response::ok("ok") + } + } + }; +} + +cell_object!( + KillsCell, + Kills, + [RecordKill], + |kills: &Kills| json!({ "mobs": kills.mobs }) +); +cell_object!( + MatchCell, + Match, + [RecordResult], + |found: &Match| json!({ "results": found.results }) +); +cell_object!( + PresenceCell, + Presence, + [Joined, Left], + |presence: &Presence| json!({ "transitions": presence.transitions }) +); + +#[event(fetch)] +async fn main(req: Request, env: Env, _ctx: Context) -> Result { + console_error_panic_hook::set_once(); + let url = req.url()?; + let path = url.path(); + if path == "/" || path == "/health" { + return Response::ok("distributed fact cells\n"); + } + if let Err(error) = authenticate(&req, &env) { + return error_response(error); + } + let parts: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + let binding = match parts.first().copied() { + Some("kills") => "KILLS", + Some("match") => "MATCH", + Some("presence") => "PRESENCE", + _ => return json_status(json!({ "error": "unknown cell kind" }), 404), + }; + let Some(shard) = parts.get(1) else { + return json_status(json!({ "error": "missing shard" }), 400); + }; + let stub = env + .durable_object(binding)? + .id_from_name(shard)? + .get_stub()?; + stub.fetch_with_request(req).await +} + +fn authenticate(req: &Request, env: &Env) -> std::result::Result<(), CellDispatchError> { + let configured = env + .secret(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + .or_else(|_| { + env.var(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + }) + .map_err(|_| CellDispatchError::Internal("internal secret is required".into()))?; + let secret = InternalHttpSecret::new(configured).map_err(CellDispatchError::Internal)?; + let candidate = req + .headers() + .get(CELL_INTERNAL_SECRET_HEADER) + .ok() + .flatten() + .ok_or(CellDispatchError::Unauthorized)?; + if secret.matches(&candidate) { + Ok(()) + } else { + Err(CellDispatchError::Unauthorized) + } +} + +fn error_response(error: CellDispatchError) -> Result { + json_status( + json!({ "error": error.client_message(), "code": error.code() }), + error.status_code(), + ) +} + +fn json_status(body: Value, status: u16) -> Result { + Ok(Response::from_json(&body)?.with_status(status)) +} diff --git a/tests/celld/fact-worker/wrangler.jsonc b/tests/celld/fact-worker/wrangler.jsonc new file mode 100644 index 000000000..194f46dc1 --- /dev/null +++ b/tests/celld/fact-worker/wrangler.jsonc @@ -0,0 +1,24 @@ +{ + "name": "distributed-fact-cells", + "main": "build/worker/shim.mjs", + "compatibility_date": "2026-01-01", + "durable_objects": { + "bindings": [ + { "name": "KILLS", "class_name": "KillsCell" }, + { "name": "MATCH", "class_name": "MatchCell" }, + { "name": "PRESENCE", "class_name": "PresenceCell" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["KillsCell", "MatchCell", "PresenceCell"] } + ], + "queues": { + "producers": [ + { "binding": "OUTBOX", "queue": "distributed-fact-outbox" } + ] + }, + "vars": { + "DISTRIBUTED_INTERNAL_SECRET": "test-only-internal-secret-change-me-2026", + "OUTBOX_DRAIN_INTERVAL_MS": "5000" + } +} From fa6e3b96eb2bc6cc7bdca5e64e9c5c58a65dfd1a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 11:10:50 -0500 Subject: [PATCH 12/40] feat(game): zone leases as celld cells with alarm expiry world::zone_lease adds the ZoneLease:{zone_id} aggregate: acquire publishes zone.activated with a token that never repeats, renew re-arms the expiry alarm with only an internal event (no outbox rows), release publishes zone.released. The cell alarm publishes zone.lease_expired { zone_id, holder, token } once a holder stops renewing, re-checking the lease so a racing renewal never gives a false expiry. celld::CellZoneOwnership implements ZoneOwnership over these cells; world::ownership_conformance is the shared behaviour suite (Local, FencedLease, ZoneLease cells). Handlers read time from ctx.cell_now(): the cell host overwrites a cell-clock claim on every dispatch, so callers cannot choose it. Endless Grid and Ripple hosts use CellZoneOwnership with CELLD_URL and DISTRIBUTED_INTERNAL_SECRET (default stays LocalOwnership); their smoke tests pass on celld dev that way. [[tasks/gr-p20-zone-lease-cell]] --- .github/workflows/integration-celld.yaml | 4 + distributed_game/Cargo.toml | 13 +- distributed_game/README.md | 4 +- distributed_game/src/celld.rs | 171 ++++++++ distributed_game/src/world/mod.rs | 6 + distributed_game/src/world/ownership.rs | 59 +++ distributed_game/src/world/zone_lease.rs | 397 ++++++++++++++++++ distributed_game/tests/celld_facts/main.rs | 223 +--------- .../tests/celld_zone_lease/main.rs | 184 ++++++++ distributed_game/tests/support/celld.rs | 229 ++++++++++ distributed_game/tests/world/main.rs | 6 + distributed_game/tests/zone_lease/main.rs | 280 ++++++++++++ examples/game/README.md | 9 + examples/game/server/Cargo.lock | 2 + examples/game/server/Cargo.toml | 2 +- examples/game/server/src/cluster.rs | 44 +- examples/game/server/tests/support/mod.rs | 5 + src/microsvc/cell_host/alarm.rs | 14 + src/microsvc/cell_host/cell.rs | 10 + src/microsvc/cell_host/mod.rs | 13 +- src/microsvc/cell_host/tests/alarms.rs | 28 ++ src/microsvc/service/handlers.rs | 7 + tests/celld/fact-worker/src/lib.rs | 75 +++- tests/celld/fact-worker/wrangler.jsonc | 6 +- 24 files changed, 1553 insertions(+), 238 deletions(-) create mode 100644 distributed_game/src/world/zone_lease.rs create mode 100644 distributed_game/tests/celld_zone_lease/main.rs create mode 100644 distributed_game/tests/support/celld.rs create mode 100644 distributed_game/tests/zone_lease/main.rs diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index c42757276..f08008562 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -71,6 +71,10 @@ jobs: env: CELLD_FACTS_REQUIRED: "1" run: cargo test -p distributed_game --features celld,runtime --test celld_facts + - name: Zone leases in celld cells (CellZoneOwnership, alarm expiry, GameServer) + env: + CELLD_FACTS_REQUIRED: "1" + run: cargo test -p distributed_game --features celld,runtime,net --test celld_zone_lease - name: Retain storage proof results and logs if: always() uses: actions/upload-artifact@v4 diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index aa45f84c9..7e37b85b1 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -18,7 +18,7 @@ runtime = ["dep:tokio"] cell = ["dep:distributed", "dep:sha2"] # `celld::CelldFactSink`: a zone server's facts sent to celld cells over the # internal HTTP path (routes by command; stable command ids; no local store). -celld = ["cell", "dep:reqwest"] +celld = ["cell", "dep:reqwest", "dep:uuid"] # Per-zone Prometheus metrics (dependency-free text exposition). metrics = [] # Load-test bots (library + `game-bots` binary). @@ -44,6 +44,7 @@ tokio-tungstenite = { version = "0.30", optional = true } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"], optional = true } rusqlite = { version = "0.39", features = ["bundled"], optional = true } reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true } +uuid = { version = "1", features = ["v7"], optional = true } [dev-dependencies] trybuild = "=1.0.120" @@ -72,6 +73,16 @@ name = "net_e2e" path = "tests/net_e2e/main.rs" required-features = ["net"] +[[test]] +name = "zone_lease" +path = "tests/zone_lease/main.rs" +required-features = ["cell"] + +[[test]] +name = "celld_zone_lease" +path = "tests/celld_zone_lease/main.rs" +required-features = ["celld", "runtime", "net"] + [[test]] name = "celld_facts" path = "tests/celld_facts/main.rs" diff --git a/distributed_game/README.md b/distributed_game/README.md index 8a6eb532f..0a847a22a 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -866,7 +866,7 @@ trait and its ABI (`run_abi`, `call_bytes`). | *(default)* | Zone/host runtime, facts queue, protocol, schema, replication, authoring derives, game surface, predicted steps (compiles to `wasm32-unknown-unknown`, where `#[predict]` exports its steps) | `serde`, `serde_json`, `distributed_game_macros` | | `runtime` | `spawn_fact_worker` on Tokio | `tokio` (rt, time, sync) | | `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record` | `distributed`, `sha2` | -| `celld` | `celld::CelldFactSink`: facts → celld cells over the internal HTTP path, routed by command | `cell` + `reqwest` | +| `celld` | `celld::CelldFactSink`: facts → celld cells over the internal HTTP path, routed by command; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells | `cell` + `reqwest`, `uuid` | | `metrics` | `metrics::GameMetrics` per-zone Prometheus text | – | | `net` | `net::GameServer` (axum WebSocket), HMAC tickets, `GET /metrics` (implies `metrics`), handoff, admin peer routes, `WorldCoordinator` | `axum`, `hmac`, `sha2`, `base64`, `tokio` | | `bots` | `bots` load-test client library + `game-bots` binary | `net` + `tokio-tungstenite` | @@ -1031,7 +1031,7 @@ principals commit concurrently; settlements arrive in completion order. | Piece | Purpose | |---|---| | `ZoneDirectory` (`StaticDirectory`, `HashRing`) | zone → owning `HostAddr`. `HashRing` uses virtual nodes; adding/removing one of N hosts moves ~1/N of the zones. A Hello for a zone owned elsewhere gets `Redirect(zone, url)`; `url` is the owner's `HostAddr::ws_url` (advertise a proxied public URL with `HostAddr::with_ws_url`) or `ServerConfig::with_redirect_url(\|host, zone\| …)`. | -| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. | +| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the cell's alarm publishes `zone.lease_expired { zone_id, holder, token }` when a holder stops renewing, so services see a crashed zone server. `world::ownership_conformance` is the behaviour every implementation passes. | | `ZoneStateStore` (`MemoryZoneStateStore`, `SqliteZoneStateStore`) | hibernation snapshots, written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | | `ZoneTopology` (`DottedTopology` default: `eldoria.port.docks` → `eldoria.port` → `eldoria`) | parent links for tier-up events. | diff --git a/distributed_game/src/celld.rs b/distributed_game/src/celld.rs index 3884b5077..e770102e8 100644 --- a/distributed_game/src/celld.rs +++ b/distributed_game/src/celld.rs @@ -290,6 +290,177 @@ impl FactSink for CelldFactSink { } } +/// [`ZoneOwnership`](crate::world::ZoneOwnership) over celld +/// `ZoneLease:{zone_id}` cells ([`crate::world::zone_lease`]): zone servers +/// acquire, renew and release over the internal path, and the cell's alarm +/// declares expiry (`zone.lease_expired`) when a holder stops renewing, so +/// services see a crashed zone server. +/// +/// Every call uses a fresh command id. A retried `acquire` by the same +/// holder returns its existing claim, so an ambiguous response is safe. +#[derive(Clone)] +pub struct CellZoneOwnership { + client: reqwest::Client, + base: String, + secret: InternalHttpSecret, + service_id: String, + host: String, + kind: String, +} + +/// Cell kind (URL segment) of zone-lease cells. +pub const ZONE_LEASE_KIND: &str = "zone-lease"; + +impl CellZoneOwnership { + /// Ownership for zone server `host` (its internal identity) against + /// `celld_url`, under command-ledger `service_id`. + pub fn new( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + host: impl Into, + ) -> Result { + let base = celld_url.into().trim_end_matches('/').to_string(); + if !(base.starts_with("http://") || base.starts_with("https://")) { + return Err(format!("celld URL must be http(s): {base}")); + } + let service_id = service_id.into(); + let host = host.into(); + if service_id.trim().is_empty() || host.trim().is_empty() { + return Err("cell zone ownership needs a service id and a host identity".into()); + } + let client = reqwest::Client::builder() + .timeout(DEFAULT_CELLD_FACT_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("celld ownership client: {error}"))?; + Ok(Self { + client, + base, + secret, + service_id, + host, + kind: ZONE_LEASE_KIND.to_string(), + }) + } + + async fn call( + &self, + zone_id: &str, + command: &str, + input: Value, + ) -> Result { + use crate::world::zone_lease::ZONE_HOST_ROLE; + use crate::world::OwnershipError; + if !valid_segment(zone_id) { + return Err(OwnershipError::Backend(format!( + "invalid zone id `{zone_id}`" + ))); + } + let url = format!( + "{}/{}/{}/{}", + self.base, + encode_segment(&self.kind), + encode_segment(zone_id), + encode_segment(command) + ); + let response = self + .client + .post(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .header(CELL_SERVICE_ID_HEADER, &self.service_id) + .header(CELL_PRINCIPAL_PARTITION_HEADER, &self.host) + .header(USER_ID_KEY, &self.host) + .header(distributed::microsvc::ROLE_KEY, ZONE_HOST_ROLE) + .json(&json!({ + "commandId": uuid::Uuid::now_v7().hyphenated().to_string(), + "input": input, + })) + .send() + .await + .map_err(|error| OwnershipError::Backend(format!("celld unreachable: {error}")))?; + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if (200..300).contains(&status) { + return serde_json::from_value(body.get("payload").cloned().unwrap_or(Value::Null)) + .map_err(|error| OwnershipError::Backend(format!("bad claim payload: {error}"))); + } + let message = body.get("error").and_then(Value::as_str).unwrap_or(""); + Err(match crate::world::zone_lease::ownership_error(message) { + OwnershipError::Backend(message) => { + OwnershipError::Backend(format!("HTTP {status}: {message}")) + } + error => error, + }) + } +} + +fn claim_of(payload: crate::world::zone_lease::ClaimPayload) -> crate::world::ZoneClaim { + crate::world::ZoneClaim { + zone_id: payload.zone_id, + holder: payload.holder, + token: payload.token, + expires_at: std::time::SystemTime::UNIX_EPOCH + + Duration::from_millis(payload.expires_at_ms), + } +} + +fn ttl_ms(ttl: Duration) -> u64 { + u64::try_from(ttl.as_millis()).unwrap_or(u64::MAX).max(1) +} + +impl crate::world::ZoneOwnership for CellZoneOwnership { + fn acquire<'a>( + &'a self, + zone_id: &'a str, + holder: &'a str, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + zone_id, + crate::world::zone_lease::ACQUIRE, + json!({ "zone_id": zone_id, "holder": holder, "ttl_ms": ttl_ms(ttl) }), + ) + .await + .map(claim_of) + }) + } + + fn renew<'a>( + &'a self, + claim: &'a crate::world::ZoneClaim, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + &claim.zone_id, + crate::world::zone_lease::RENEW, + json!({ + "zone_id": claim.zone_id, + "holder": claim.holder, + "token": claim.token, + "ttl_ms": ttl_ms(ttl), + }), + ) + .await + .map(claim_of) + }) + } + + fn release<'a>(&'a self, claim: &'a crate::world::ZoneClaim) -> BoxFuture<'a, ()> { + Box::pin(async move { + let _ = self + .call( + &claim.zone_id, + crate::world::zone_lease::RELEASE, + json!({ "zone_id": claim.zone_id, "holder": claim.holder, "token": claim.token }), + ) + .await; + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/distributed_game/src/world/mod.rs b/distributed_game/src/world/mod.rs index 29add916b..ef86ab8b5 100644 --- a/distributed_game/src/world/mod.rs +++ b/distributed_game/src/world/mod.rs @@ -22,6 +22,10 @@ pub mod directory; pub mod ownership; pub mod state; pub mod topology; +#[cfg(feature = "cell")] +// The `sourced` and `Snapshot` derives generate undocumented public items. +#[allow(missing_docs)] +pub mod zone_lease; pub use crate::facts::BoxFuture; pub use allocation::{ @@ -30,6 +34,8 @@ pub use allocation::{ }; pub use barrier::{BarrierMode, FreezeState, StalePolicy}; pub use directory::{HashRing, HostAddr, StaticDirectory, ZoneDirectory}; +#[doc(hidden)] +pub use ownership::ownership_conformance; #[cfg(feature = "lease")] pub use ownership::FencedLeaseOwnership; pub use ownership::{LocalOwnership, OwnershipError, ZoneClaim, ZoneOwnership}; diff --git a/distributed_game/src/world/ownership.rs b/distributed_game/src/world/ownership.rs index 719e9af1a..00be9b711 100644 --- a/distributed_game/src/world/ownership.rs +++ b/distributed_game/src/world/ownership.rs @@ -267,6 +267,60 @@ where } } +/// Behaviour every [`ZoneOwnership`] must show (exclusive claims, `Held`, +/// idempotent re-acquire, `Lost` after release, strictly increasing +/// fencing tokens). `zone_id` must be unused. Panics on a violation. +#[doc(hidden)] +pub async fn ownership_conformance(ownership: &dyn ZoneOwnership, zone_id: &str) { + let ttl = Duration::from_secs(30); + let first = ownership + .acquire(zone_id, "h1#1", ttl) + .await + .expect("first acquire"); + assert_eq!(first.zone_id, zone_id); + assert_eq!(first.holder, "h1#1"); + assert_eq!( + ownership.acquire(zone_id, "h2#1", ttl).await, + Err(OwnershipError::Held { + holder: "h1#1".into() + }), + "a live claim is exclusive" + ); + let again = ownership + .acquire(zone_id, "h1#1", ttl) + .await + .expect("re-acquire"); + assert_eq!( + again.token, first.token, + "the holder gets its own claim back" + ); + let renewed = ownership.renew(&first, ttl).await.expect("renew"); + assert_eq!(renewed.token, first.token, "renewal keeps the token"); + ownership.release(&first).await; + assert_eq!( + ownership.renew(&first, ttl).await, + Err(OwnershipError::Lost), + "a released claim is lost" + ); + let second = ownership + .acquire(zone_id, "h2#1", ttl) + .await + .expect("second acquire"); + assert!(second.token > first.token, "tokens strictly increase"); + assert_eq!( + ownership.renew(&first, ttl).await, + Err(OwnershipError::Lost), + "an old claim stays lost" + ); + ownership.release(&second).await; + let third = ownership + .acquire(zone_id, "h1#2", ttl) + .await + .expect("third acquire"); + assert!(third.token > second.token, "tokens survive release"); + ownership.release(&third).await; +} + #[cfg(test)] mod tests { use super::*; @@ -300,6 +354,11 @@ mod tests { assert!(b.token > a.token); } + #[test] + fn local_ownership_passes_the_conformance_suite() { + block(ownership_conformance(&LocalOwnership::new(), "conformance")); + } + #[test] fn local_tokens_outrank_a_previous_process() { let ttl = Duration::from_secs(1); diff --git a/distributed_game/src/world/zone_lease.rs b/distributed_game/src/world/zone_lease.rs new file mode 100644 index 000000000..e7357e67c --- /dev/null +++ b/distributed_game/src/world/zone_lease.rs @@ -0,0 +1,397 @@ +//! Zone ownership as a cell: `ZoneLease:{zone_id}`. +//! +//! The lease of a zone lives in its own aggregate cell (celld in deployment, +//! the in-process reference host in tests) instead of a store private to the +//! zone server, so services learn when a zone server dies: +//! +//! | Command | Effect | Outward event | +//! |---|---|---| +//! | `zone_lease.acquire` | new holder, token + 1, alarm at expiry | `zone.activated` | +//! | `zone_lease.renew` | later expiry, alarm re-armed | none (no heartbeat events) | +//! | `zone_lease.release` | no holder, alarm cancelled | `zone.released` | +//! | `zone_lease.expire` (alarm only) | no holder, if still expired | `zone.lease_expired` | +//! +//! Fencing tokens increase on every acquisition and are never reset. Expiry +//! is declared by the cell's alarm ([`distributed::cell_host::CellAlarm`]): +//! the alarm handler re-reads the lease and expires it only when the +//! holder's expiry has passed by the cell clock, so a renewal that races the +//! alarm never produces a false expiry. All times are the cell's clock +//! (`ctx.cell_now()`), never the caller's. +//! +//! Callers must hold the `zone-host` role (an internal identity; the +//! commands are not routed through the public GraphQL edge). + +use std::time::{Duration, SystemTime}; + +use distributed::cell_host::{cell_alarm_guard, AggregateCell, CellAlarm}; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, +}; +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; + +/// Aggregate type (cell `ZoneLease:{zone_id}`). +pub const ZONE_LEASE: &str = "ZoneLease"; +/// Role a zone server's internal session carries. +pub const ZONE_HOST_ROLE: &str = "zone-host"; +/// Acquire command. +pub const ACQUIRE: &str = "zone_lease.acquire"; +/// Renew command. +pub const RENEW: &str = "zone_lease.renew"; +/// Release command. +pub const RELEASE: &str = "zone_lease.release"; +/// Alarm-only expiry command. +pub const EXPIRE: &str = "zone_lease.expire"; +/// Rejection message prefix when another holder has the lease. +pub const HELD_PREFIX: &str = "held:"; +/// Rejection message when a claim is no longer the live one. +pub const LOST: &str = "lost"; +/// The expiry alarm's name. +pub const EXPIRY_ALARM: &str = "expiry"; + +/// The lease of one zone. +#[derive(Clone, Debug, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct ZoneLease { + entity: Entity, + holder: Option, + token: u64, + expires_at_ms: u64, +} + +#[sourced(entity, aggregate_type = "ZoneLease", events = "ZoneLeaseEvent")] +impl ZoneLease { + #[event("zone.activated", version = 1, domain = event)] + fn activated(&mut self, zone_id: String, holder: String, token: u64, expires_at_ms: u64) { + if self.entity.id().is_empty() { + self.entity.set_id(zone_id); + } + self.holder = Some(holder); + self.token = token; + self.expires_at_ms = expires_at_ms; + } + + #[event("zone_lease.renewed", version = 1)] + fn renewed(&mut self, expires_at_ms: u64) { + self.expires_at_ms = expires_at_ms; + } + + #[event("zone.released", version = 1, domain = event)] + fn released(&mut self, zone_id: String, holder: String, token: u64) { + let _ = (zone_id, holder, token); + self.holder = None; + } + + #[event("zone.lease_expired", version = 1, domain = event)] + fn lease_expired(&mut self, zone_id: String, holder: String, token: u64) { + let _ = (zone_id, holder, token); + self.holder = None; + } +} + +impl ZoneLease { + /// Current holder, if any. + pub fn holder(&self) -> Option<&str> { + self.holder.as_deref() + } + + /// Latest fencing token (kept after release and expiry). + pub fn token(&self) -> u64 { + self.token + } + + /// Expiry of the current holder (epoch ms, cell clock). + pub fn expires_at_ms(&self) -> u64 { + self.expires_at_ms + } +} + +/// `zone_lease.acquire` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct AcquireInput { + pub zone_id: String, + pub holder: String, + pub ttl_ms: u64, +} + +/// `zone_lease.renew` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct RenewInput { + pub zone_id: String, + pub holder: String, + pub token: u64, + pub ttl_ms: u64, +} + +/// `zone_lease.release` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct ReleaseInput { + pub zone_id: String, + pub holder: String, + pub token: u64, +} + +/// `zone_lease.expire` input (armed by the cell itself). +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct ExpireInput { + pub zone_id: String, +} + +/// A held (or released) claim. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, distributed::CommandOutput)] +pub struct ClaimPayload { + pub zone_id: String, + pub holder: String, + pub token: u64, + pub expires_at_ms: u64, +} + +/// What an expiry alarm did. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, distributed::CommandOutput)] +pub struct ExpiryPayload { + pub expired: bool, +} + +/// Map a rejected lease command's message to an +/// [`OwnershipError`](super::OwnershipError). +pub fn ownership_error(message: &str) -> super::OwnershipError { + let message = message.strip_prefix("rejected: ").unwrap_or(message); + if let Some(holder) = message.strip_prefix(HELD_PREFIX) { + super::OwnershipError::Held { + holder: holder.to_string(), + } + } else if message == LOST { + super::OwnershipError::Lost + } else { + super::OwnershipError::Backend(message.to_string()) + } +} + +fn rejected(message: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(message.to_string()) +} + +fn is_zone_host(ctx: &CausalCommandContext<'_, ZoneLease>) -> bool { + ctx.session().has_role(ZONE_HOST_ROLE) +} + +fn now_ms(ctx: &CausalCommandContext<'_, ZoneLease>) -> Result { + ctx.cell_now() + .and_then(|now| now.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|now| now.as_millis() as u64) + .ok_or_else(|| rejected("zone leases run only in an aggregate cell")) +} + +fn ttl(ttl_ms: u64) -> Result { + if ttl_ms == 0 || ttl_ms > 24 * 60 * 60 * 1_000 { + return Err(rejected("ttl_ms must be within 1..=86400000")); + } + Ok(ttl_ms) +} + +fn validate(zone_id: &str, holder: &str) -> Result<(), HandlerError> { + if zone_id.is_empty() || holder.is_empty() { + return Err(rejected("zone_id and holder must not be empty")); + } + Ok(()) +} + +fn expiry_alarm(zone_id: &str, expires_at_ms: u64) -> Result { + CellAlarm::at( + EXPIRY_ALARM, + SystemTime::UNIX_EPOCH + Duration::from_millis(expires_at_ms), + EXPIRE, + &ExpireInput { + zone_id: zone_id.to_string(), + }, + ) + .map_err(rejected) +} + +async fn handle_acquire( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: AcquireInput, +) -> Result>, HandlerError> { + validate(&input.zone_id, &input.holder)?; + let ttl = ttl(input.ttl_ms)?; + let now = now_ms(ctx)?; + let repo = ctx.repo(); + let mut lease = match repo.get(&input.zone_id).await? { + Some(lease) => lease, + None => repo.create(), + }; + match lease.holder() { + // The holder re-acquiring its live claim gets it back unchanged. + Some(holder) if holder == input.holder => { + let claim = ClaimPayload { + zone_id: input.zone_id, + holder: input.holder, + token: lease.token, + expires_at_ms: lease.expires_at_ms, + }; + return repo.commit(lease)?.succeeded(claim); + } + // Held until the expiry alarm declares otherwise. + Some(holder) => return Err(rejected(format!("{HELD_PREFIX}{holder}"))), + None => {} + } + let token = lease.token + 1; + let expires_at_ms = now + ttl; + lease + .activated( + input.zone_id.clone(), + input.holder.clone(), + token, + expires_at_ms, + ) + .map_err(rejected)?; + ctx.arm_alarm(expiry_alarm(&input.zone_id, expires_at_ms)?)?; + repo.publish_events() + .commit(lease)? + .succeeded(ClaimPayload { + zone_id: input.zone_id, + holder: input.holder, + token, + expires_at_ms, + }) +} + +async fn handle_renew( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: RenewInput, +) -> Result>, HandlerError> { + validate(&input.zone_id, &input.holder)?; + let ttl = ttl(input.ttl_ms)?; + let now = now_ms(ctx)?; + let repo = ctx.repo(); + let mut lease = repo + .get(&input.zone_id) + .await? + .ok_or_else(|| rejected(LOST))?; + if lease.holder() != Some(input.holder.as_str()) || lease.token != input.token { + return Err(rejected(LOST)); + } + let expires_at_ms = now + ttl; + lease.renewed(expires_at_ms).map_err(rejected)?; + ctx.arm_alarm(expiry_alarm(&input.zone_id, expires_at_ms)?)?; + // Internal state only: no outward event, so no outbox row. + repo.commit(lease)?.succeeded(ClaimPayload { + zone_id: input.zone_id, + holder: input.holder, + token: input.token, + expires_at_ms, + }) +} + +async fn handle_release( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: ReleaseInput, +) -> Result>, HandlerError> { + validate(&input.zone_id, &input.holder)?; + let repo = ctx.repo(); + let mut lease = repo + .get(&input.zone_id) + .await? + .ok_or_else(|| rejected(LOST))?; + let claim = ClaimPayload { + zone_id: input.zone_id.clone(), + holder: input.holder.clone(), + token: input.token, + expires_at_ms: lease.expires_at_ms, + }; + if lease.holder() != Some(input.holder.as_str()) || lease.token != input.token { + // Releasing a claim that is already gone is a no-op. + return repo.commit(lease)?.succeeded(claim); + } + lease + .released(input.zone_id, input.holder, input.token) + .map_err(rejected)?; + ctx.cancel_alarm(EXPIRY_ALARM)?; + repo.publish_events().commit(lease)?.succeeded(claim) +} + +async fn handle_expire( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: ExpireInput, +) -> Result>, HandlerError> { + let now = now_ms(ctx)?; + let repo = ctx.repo(); + let Some(mut lease) = repo.get(&input.zone_id).await? else { + return Err(rejected("no lease")); + }; + let expired = match lease.holder.clone() { + Some(holder) if now >= lease.expires_at_ms => { + let token = lease.token; + lease + .lease_expired(input.zone_id, holder, token) + .map_err(rejected)?; + true + } + // Renewed (or released) after this alarm was read: not expired. + _ => false, + }; + if expired { + repo.publish_events() + .commit(lease)? + .succeeded(ExpiryPayload { expired }) + } else { + repo.commit(lease)?.succeeded(ExpiryPayload { expired }) + } +} + +macro_rules! lease_command { + ($name:ident, $command:expr, $input:ty, $output:ty, $guard:expr, $handler:expr) => { + struct $name; + impl PortableCommand for $name + where + D: CausalRouteDependencies + Send + Sync + 'static, + { + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::<$input, Succeeded<$output>>($command)) + .guarded($guard, $handler) + } + } + }; +} + +lease_command!( + Acquire, + ACQUIRE, + AcquireInput, + ClaimPayload, + is_zone_host, + handle_acquire +); +lease_command!( + Renew, + RENEW, + RenewInput, + ClaimPayload, + is_zone_host, + handle_renew +); +lease_command!( + Release, + RELEASE, + ReleaseInput, + ClaimPayload, + is_zone_host, + handle_release +); +lease_command!( + Expire, + EXPIRE, + ExpireInput, + ExpiryPayload, + cell_alarm_guard::, + handle_expire +); + +/// Mount every zone-lease command on a `ZoneLease` cell. +pub fn mount_zone_lease(cell: AggregateCell) -> AggregateCell { + cell.mount(Acquire) + .mount(Renew) + .mount(Release) + .mount(Expire) +} diff --git a/distributed_game/tests/celld_facts/main.rs b/distributed_game/tests/celld_facts/main.rs index 3e4b44e1e..084e018eb 100644 --- a/distributed_game/tests/celld_facts/main.rs +++ b/distributed_game/tests/celld_facts/main.rs @@ -11,10 +11,6 @@ //! esbuild (`CELLD_ESBUILD`). Without a built worker the test skips, unless //! `CELLD_FACTS_REQUIRED=1` (CI), which turns that into a failure. -use std::io::Write as _; -use std::net::TcpListener; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; use std::time::{Duration, Instant}; use distributed::cell_host::InternalHttpSecret; @@ -26,7 +22,9 @@ use distributed_game::{ }; use serde_json::{json, Value}; -const SECRET: &str = "test-only-internal-secret-change-me-2026"; +#[path = "../support/celld.rs"] +mod celld_support; +use celld_support::{Celld, SECRET}; const SERVICE: &str = "zone-server-test"; #[derive(serde::Serialize, FactCommand)] @@ -42,221 +40,6 @@ struct MatchResult { winner_slot: u32, } -fn worker_dir() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")).join("../tests/celld/fact-worker") -} - -struct Celld { - project: PathBuf, - port: u16, - child: Option, - generation: u32, -} - -impl Celld { - fn start_new() -> Option { - let build = worker_dir().join("build"); - if !build.join("worker/shim.mjs").exists() { - assert!( - std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), - "CELLD_FACTS_REQUIRED is set but the fact worker is not built" - ); - eprintln!("skipping celld fact routing: build tests/celld/fact-worker first"); - return None; - } - let project = std::env::temp_dir().join(format!( - "distributed-celld-facts-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - copy_dir(&build, &project.join("build")); - std::fs::copy( - worker_dir().join("wrangler.jsonc"), - project.join("wrangler.jsonc"), - ) - .unwrap(); - let port = TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port(); - let mut celld = Self { - project, - port, - child: None, - generation: 0, - }; - if let Err(reason) = celld.start() { - assert!( - std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), - "celld dev is required but did not start: {reason}" - ); - eprintln!("skipping celld fact routing: {reason}"); - return None; - } - Some(celld) - } - - fn base(&self) -> String { - format!("http://127.0.0.1:{}", self.port) - } - - /// Start `celld dev`; `Err` when it is unavailable or exits early (no - /// celld on PATH, or no esbuild for its bundler). - fn start(&mut self) -> Result<(), String> { - self.generation += 1; - let binary = std::env::var("CELLD").unwrap_or_else(|_| "celld".into()); - let help = Command::new(&binary) - .args(["dev", "--help"]) - .output() - .map_err(|error| format!("cannot run `{binary}`: {error}"))?; - let mut args = vec![ - "dev".to_string(), - self.project.display().to_string(), - "--host".into(), - "127.0.0.1".into(), - "--port".into(), - self.port.to_string(), - "--logs".into(), - ]; - if String::from_utf8_lossy(&help.stdout).contains("--no-watch") { - args.push("--no-watch".into()); - } - let log_path = self.project.join(format!("celld-{}.log", self.generation)); - let log = std::fs::File::create(&log_path).unwrap(); - let child = Command::new(&binary) - .args(&args) - .env("RUST_LOG", "warn") - .stdin(Stdio::null()) - .stdout(log.try_clone().unwrap()) - .stderr(log) - .spawn() - .map_err(|error| format!("cannot start `{binary} dev`: {error}"))?; - self.child = Some(child); - let deadline = Instant::now() + Duration::from_secs(90); - while Instant::now() < deadline { - if std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() - && blocking_get(&format!("{}/health", self.base())).is_some_and(|s| s == 200) - { - return Ok(()); - } - let exited = self - .child - .as_mut() - .and_then(|child| child.try_wait().ok().flatten()); - if let Some(status) = exited { - self.child = None; - let log = std::fs::read_to_string(&log_path).unwrap_or_default(); - return Err(format!( - "celld dev exited ({status}): {}", - log.lines().rev().take(3).collect::>().join(" | ") - )); - } - std::thread::sleep(Duration::from_millis(200)); - } - Err(format!( - "celld dev did not become ready; logs in {}", - self.project.display() - )) - } - - /// Kill the launcher this test started and every descendant. - fn crash(&mut self) { - let Some(mut child) = self.child.take() else { - return; - }; - let root = child.id(); - let _ = Command::new("kill") - .args(["-STOP", &root.to_string()]) - .status(); - let table = Command::new("ps") - .args(["-axo", "pid=,ppid="]) - .output() - .unwrap(); - let pairs = String::from_utf8_lossy(&table.stdout) - .lines() - .filter_map(|line| { - let mut parts = line.split_whitespace(); - Some(( - parts.next()?.parse::().ok()?, - parts.next()?.parse::().ok()?, - )) - }) - .collect::>(); - let mut doomed = vec![root]; - let mut index = 0; - while index < doomed.len() { - let parent = doomed[index]; - doomed.extend( - pairs - .iter() - .filter(|(_, ppid)| *ppid == parent) - .map(|(pid, _)| *pid), - ); - index += 1; - } - for pid in doomed.iter().rev() { - let _ = Command::new("kill") - .args(["-KILL", &pid.to_string()]) - .status(); - } - let _ = child.wait(); - // The port is released once the processes are gone. - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline - && std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() - { - std::thread::sleep(Duration::from_millis(100)); - } - } -} - -impl Drop for Celld { - fn drop(&mut self) { - self.crash(); - if !std::thread::panicking() { - let _ = std::fs::remove_dir_all(&self.project); - } else { - let _ = writeln!( - std::io::stderr(), - "celld artifacts kept in {}", - self.project.display() - ); - } - } -} - -fn copy_dir(from: &Path, to: &Path) { - std::fs::create_dir_all(to).unwrap(); - for entry in std::fs::read_dir(from).unwrap() { - let entry = entry.unwrap(); - let target = to.join(entry.file_name()); - if entry.file_type().unwrap().is_dir() { - copy_dir(&entry.path(), &target); - } else { - std::fs::copy(entry.path(), target).unwrap(); - } - } -} - -fn blocking_get(url: &str) -> Option { - let url = url.strip_prefix("http://")?; - let (host, path) = url.split_once('/')?; - let mut stream = std::net::TcpStream::connect(host).ok()?; - stream.set_read_timeout(Some(Duration::from_secs(2))).ok()?; - write!( - stream, - "GET /{path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n" - ) - .ok()?; - let mut response = String::new(); - std::io::Read::read_to_string(&mut stream, &mut response).ok()?; - response.split_whitespace().nth(1)?.parse().ok() -} - fn sink(base: &str, timeout: Duration) -> CelldFactSink { CelldFactSink::with_timeout( base, diff --git a/distributed_game/tests/celld_zone_lease/main.rs b/distributed_game/tests/celld_zone_lease/main.rs new file mode 100644 index 000000000..cd51235ed --- /dev/null +++ b/distributed_game/tests/celld_zone_lease/main.rs @@ -0,0 +1,184 @@ +//! Zone ownership as celld cells (`CellZoneOwnership` → `ZoneLease:{zone}`) +//! on an isolated `celld dev`: the ownership conformance suite, expiry +//! declared by the cell alarm when a holder stops renewing, renewals that +//! keep the lease, and a `GameServer` running a zone on it. +//! +//! (cd tests/celld/fact-worker && worker-build --release) +//! cargo test -p distributed_game --features celld,runtime,net --test celld_zone_lease + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use distributed::cell_host::InternalHttpSecret; +use distributed_game::celld::CellZoneOwnership; +use distributed_game::net::{GameServer, ServerConfig, TicketKey, ZoneFactory}; +use distributed_game::world::{ownership_conformance, OwnershipError, ZoneOwnership}; +use distributed_game::*; +use serde_json::Value; + +#[path = "../support/celld.rs"] +mod celld_support; +use celld_support::{Celld, SECRET}; + +fn ownership(base: &str, host: &str) -> CellZoneOwnership { + CellZoneOwnership::new( + base, + InternalHttpSecret::new(SECRET).unwrap(), + "zone-lease-test", + host, + ) + .unwrap() +} + +async fn lease(base: &str, zone: &str) -> Value { + reqwest::Client::new() + .get(format!("{base}/zone-lease/{zone}")) + .header("x-distributed-internal-secret", SECRET) + .send() + .await + .unwrap() + .json() + .await + .unwrap() +} + +fn count(state: &Value, event: &str) -> usize { + state["eventNames"].as_array().map_or(0, |names| { + names.iter().filter(|name| *name == event).count() + }) +} + +async fn until(what: &str, within: Duration, mut check: impl AsyncFnMut() -> bool) { + let deadline = Instant::now() + within; + while Instant::now() < deadline { + if check().await { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("timed out: {what}"); +} + +/// A zone with no players or state: enough to run under a lease. +struct Beacon; + +impl Zone for Beacon { + type Config = (); + type Input = u8; + type Timer = (); + type Message = NoMessages; + + fn schema() -> EntitySchema { + EntitySchema::new() + } + + fn on_create(_ctx: &mut Ctx<'_, Self>, _config: ()) -> Self { + Beacon + } + + fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option { + payload.first().copied() + } + + fn on_tick(&mut self, _ctx: &mut Ctx<'_, Self>) {} + + fn replicate(&self, _view: &mut WorldView) {} +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn zone_leases_live_in_celld_cells() { + let Some(celld) = Celld::start_new() else { + return; + }; + let base = celld.base(); + let own = ownership(&base, "host-a"); + + // 1. Fenced, exclusive, strictly increasing tokens; Held and Lost. + ownership_conformance(&own, "zl-conformance").await; + let conformance = lease(&base, "zl-conformance").await; + assert_eq!(count(&conformance, "zone.activated"), 3); + assert_eq!(count(&conformance, "zone.released"), 3); + assert_eq!(count(&conformance, "zone.lease_expired"), 0); + + // 2. A holder that stops renewing: exactly one zone.lease_expired after + // the TTL, declared by the cell's own alarm (nobody calls the cell). + let dead = own + .acquire("zl-expire", "a#1", Duration::from_millis(1_500)) + .await + .unwrap(); + until("lease expiry", Duration::from_secs(20), async || { + lease(&base, "zl-expire").await["state"]["holder"].is_null() + }) + .await; + tokio::time::sleep(Duration::from_secs(2)).await; + let expired = lease(&base, "zl-expire").await; + assert_eq!(count(&expired, "zone.lease_expired"), 1, "{expired}"); + assert_eq!(expired["armedAlarms"], 0); + assert_eq!( + own.renew(&dead, Duration::from_secs(1)).await, + Err(OwnershipError::Lost) + ); + let next = own + .acquire("zl-expire", "b#1", Duration::from_secs(5)) + .await + .unwrap(); + assert!(next.token > dead.token); + own.release(&next).await; + + // 3. Renewals keep the lease past many TTLs, without expiry or outward + // events. + let mut live = own + .acquire("zl-renew", "a#1", Duration::from_millis(1_500)) + .await + .unwrap(); + for _ in 0..10 { + tokio::time::sleep(Duration::from_millis(400)).await; + live = own + .renew(&live, Duration::from_millis(1_500)) + .await + .unwrap(); + } + let renewed = lease(&base, "zl-renew").await; + assert_eq!(renewed["state"]["holder"], "a#1"); + assert_eq!(count(&renewed, "zone.lease_expired"), 0); + assert_eq!(count(&renewed, "zone_lease.renewed"), 10); + own.release(&live).await; + + // 4. A zone server runs a zone under the celld lease: the cell sees the + // holder while the server renews, and the lease ends when it stops. + let mut config = ServerConfig::new(TicketKey::new("celld-lease-test-ticket-secret")); + config.ownership_ttl = Duration::from_secs(3); + config.ownership_renew_every = Duration::from_millis(500); + let server = GameServer::::new(config, ZoneFactory::sync(|_| Some(()))) + .with_host_id("A") + .with_ownership(Arc::new(ownership(&base, "host-a"))); + assert!(server.activate("zl-game").await, "zone activated"); + let held = lease(&base, "zl-game").await; + let holder = held["state"]["holder"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!(holder.starts_with("A#"), "{held}"); + tokio::time::sleep(Duration::from_secs(5)).await; + let kept = lease(&base, "zl-game").await; + assert_eq!( + kept["state"]["holder"], + holder.as_str(), + "renewed past the TTL" + ); + assert_eq!(count(&kept, "zone.lease_expired"), 0); + server.shutdown(); + until( + "lease end after shutdown", + Duration::from_secs(20), + async || lease(&base, "zl-game").await["state"]["holder"].is_null(), + ) + .await; + let ended = lease(&base, "zl-game").await; + assert_eq!( + count(&ended, "zone.released") + count(&ended, "zone.lease_expired"), + 1, + "exactly one terminal event: {ended}" + ); + drop(celld); +} diff --git a/distributed_game/tests/support/celld.rs b/distributed_game/tests/support/celld.rs new file mode 100644 index 000000000..12460743c --- /dev/null +++ b/distributed_game/tests/support/celld.rs @@ -0,0 +1,229 @@ +//! An isolated `celld dev` serving `tests/celld/fact-worker`, owned by one +//! test: its own temp project, port and process tree (killed on drop; only +//! processes this harness started). Skips (returns `None`) when the worker +//! is not built or celld cannot start, unless `CELLD_FACTS_REQUIRED=1`. + +#![allow(dead_code)] + +use std::io::Write as _; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +pub const SECRET: &str = "test-only-internal-secret-change-me-2026"; + +fn worker_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../tests/celld/fact-worker") +} + +pub struct Celld { + project: PathBuf, + port: u16, + child: Option, + generation: u32, +} + +impl Celld { + pub fn start_new() -> Option { + let build = worker_dir().join("build"); + if !build.join("worker/shim.mjs").exists() { + assert!( + std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), + "CELLD_FACTS_REQUIRED is set but the fact worker is not built" + ); + eprintln!("skipping celld fact routing: build tests/celld/fact-worker first"); + return None; + } + let project = std::env::temp_dir().join(format!( + "distributed-celld-facts-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + copy_dir(&build, &project.join("build")); + std::fs::copy( + worker_dir().join("wrangler.jsonc"), + project.join("wrangler.jsonc"), + ) + .unwrap(); + let port = TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let mut celld = Self { + project, + port, + child: None, + generation: 0, + }; + if let Err(reason) = celld.start() { + assert!( + std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), + "celld dev is required but did not start: {reason}" + ); + eprintln!("skipping celld fact routing: {reason}"); + return None; + } + Some(celld) + } + + pub fn base(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Start `celld dev`; `Err` when it is unavailable or exits early (no + /// celld on PATH, or no esbuild for its bundler). + pub fn start(&mut self) -> Result<(), String> { + self.generation += 1; + let binary = std::env::var("CELLD").unwrap_or_else(|_| "celld".into()); + let help = Command::new(&binary) + .args(["dev", "--help"]) + .output() + .map_err(|error| format!("cannot run `{binary}`: {error}"))?; + let mut args = vec![ + "dev".to_string(), + self.project.display().to_string(), + "--host".into(), + "127.0.0.1".into(), + "--port".into(), + self.port.to_string(), + "--logs".into(), + ]; + if String::from_utf8_lossy(&help.stdout).contains("--no-watch") { + args.push("--no-watch".into()); + } + let log_path = self.project.join(format!("celld-{}.log", self.generation)); + let log = std::fs::File::create(&log_path).unwrap(); + let child = Command::new(&binary) + .args(&args) + .env("RUST_LOG", "warn") + .stdin(Stdio::null()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .map_err(|error| format!("cannot start `{binary} dev`: {error}"))?; + self.child = Some(child); + let deadline = Instant::now() + Duration::from_secs(90); + while Instant::now() < deadline { + if std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() + && blocking_get(&format!("{}/health", self.base())).is_some_and(|s| s == 200) + { + return Ok(()); + } + let exited = self + .child + .as_mut() + .and_then(|child| child.try_wait().ok().flatten()); + if let Some(status) = exited { + self.child = None; + let log = std::fs::read_to_string(&log_path).unwrap_or_default(); + return Err(format!( + "celld dev exited ({status}): {}", + log.lines().rev().take(3).collect::>().join(" | ") + )); + } + std::thread::sleep(Duration::from_millis(200)); + } + Err(format!( + "celld dev did not become ready; logs in {}", + self.project.display() + )) + } + + /// Kill the launcher this test started and every descendant. + pub fn crash(&mut self) { + let Some(mut child) = self.child.take() else { + return; + }; + let root = child.id(); + let _ = Command::new("kill") + .args(["-STOP", &root.to_string()]) + .status(); + let table = Command::new("ps") + .args(["-axo", "pid=,ppid="]) + .output() + .unwrap(); + let pairs = String::from_utf8_lossy(&table.stdout) + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + Some(( + parts.next()?.parse::().ok()?, + parts.next()?.parse::().ok()?, + )) + }) + .collect::>(); + let mut doomed = vec![root]; + let mut index = 0; + while index < doomed.len() { + let parent = doomed[index]; + doomed.extend( + pairs + .iter() + .filter(|(_, ppid)| *ppid == parent) + .map(|(pid, _)| *pid), + ); + index += 1; + } + for pid in doomed.iter().rev() { + let _ = Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status(); + } + let _ = child.wait(); + // The port is released once the processes are gone. + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline + && std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() + { + std::thread::sleep(Duration::from_millis(100)); + } + } +} + +impl Drop for Celld { + fn drop(&mut self) { + self.crash(); + if !std::thread::panicking() { + let _ = std::fs::remove_dir_all(&self.project); + } else { + let _ = writeln!( + std::io::stderr(), + "celld artifacts kept in {}", + self.project.display() + ); + } + } +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), target).unwrap(); + } + } +} + +fn blocking_get(url: &str) -> Option { + let url = url.strip_prefix("http://")?; + let (host, path) = url.split_once('/')?; + let mut stream = std::net::TcpStream::connect(host).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok()?; + write!( + stream, + "GET /{path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n" + ) + .ok()?; + let mut response = String::new(); + std::io::Read::read_to_string(&mut stream, &mut response).ok()?; + response.split_whitespace().nth(1)?.parse().ok() +} diff --git a/distributed_game/tests/world/main.rs b/distributed_game/tests/world/main.rs index 78cf2a7e7..0c9adf901 100644 --- a/distributed_game/tests/world/main.rs +++ b/distributed_game/tests/world/main.rs @@ -1119,3 +1119,9 @@ async fn transfer_tickets_carry_extension_claims_across_hosts() { assert_eq!(players[0].principal, "ana"); assert_eq!(players[0].extensions.typed::().unwrap(), party); } + +#[tokio::test] +async fn fenced_lease_ownership_passes_the_conformance_suite() { + let ownership = FencedLeaseOwnership::new(InMemoryLeaseStore::new()); + distributed_game::world::ownership_conformance(&ownership, "conformance").await; +} diff --git a/distributed_game/tests/zone_lease/main.rs b/distributed_game/tests/zone_lease/main.rs new file mode 100644 index 000000000..b45598afc --- /dev/null +++ b/distributed_game/tests/zone_lease/main.rs @@ -0,0 +1,280 @@ +//! `ZoneLease` cells on the in-process reference host, driven by a +//! `ManualClock`: ownership conformance, no outbox rows for renewals, +//! exactly one `zone.lease_expired`, and renewals racing the alarm. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use distributed::cell_host::{ + cell_alarm_session, AggregateCell, CellCommandIdentity, CellDispatchError, CellStreamStore, +}; +use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; +use distributed::{Clock, ManualClock, StreamIdentity}; +use distributed_game::cell::command_id_for; +use distributed_game::world::zone_lease::{ + mount_zone_lease, ownership_error, ClaimPayload, ZoneLease, ACQUIRE, EXPIRE, RELEASE, RENEW, + ZONE_HOST_ROLE, +}; +use distributed_game::world::{ownership_conformance, OwnershipError, ZoneClaim, ZoneOwnership}; +use distributed_game::BoxFuture; +use serde_json::{json, Value}; + +const T0: u64 = 1_700_000_000_000; + +fn clock() -> ManualClock { + ManualClock::new(SystemTime::UNIX_EPOCH + Duration::from_millis(T0)) +} + +fn lease_cell(clock: &ManualClock, zone: &str) -> AggregateCell { + let store = CellStreamStore::for_identity(StreamIdentity::new("ZoneLease", zone).unwrap()) + .with_clock(clock.clone()); + mount_zone_lease(AggregateCell::from_store(store).unwrap()) +} + +fn host_session() -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, "zone-server-a"); + session.set(ROLE_KEY, ZONE_HOST_ROLE); + session +} + +/// `ZoneOwnership` over one in-process cell, as `CellZoneOwnership` is over +/// celld (same commands, same error mapping). +struct InProcess { + cell: AggregateCell, + calls: AtomicU64, +} + +impl InProcess { + async fn call(&self, command: &str, input: Value) -> Result { + let n = self.calls.fetch_add(1, Ordering::SeqCst); + let identity = CellCommandIdentity::new( + "zone-host", + "zone-server-a", + command_id_for("t", &n.to_string()), + ) + .unwrap(); + match self + .cell + .dispatch_idempotent(command, &identity, input, host_session()) + .await + { + Ok(result) => Ok(serde_json::from_value(result.payload().clone()).unwrap()), + Err(CellDispatchError::Rejected { message, .. }) => Err(ownership_error(&message)), + Err(error) => Err(OwnershipError::Backend(format!("{error:?}"))), + } + } +} + +fn claim(payload: ClaimPayload) -> ZoneClaim { + ZoneClaim { + zone_id: payload.zone_id, + holder: payload.holder, + token: payload.token, + expires_at: SystemTime::UNIX_EPOCH + Duration::from_millis(payload.expires_at_ms), + } +} + +impl ZoneOwnership for InProcess { + fn acquire<'a>( + &'a self, + zone_id: &'a str, + holder: &'a str, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + ACQUIRE, + json!({ "zone_id": zone_id, "holder": holder, "ttl_ms": ttl.as_millis() as u64 }), + ) + .await + .map(claim) + }) + } + + fn renew<'a>( + &'a self, + current: &'a ZoneClaim, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + RENEW, + json!({ "zone_id": current.zone_id, "holder": current.holder, + "token": current.token, "ttl_ms": ttl.as_millis() as u64 }), + ) + .await + .map(claim) + }) + } + + fn release<'a>(&'a self, current: &'a ZoneClaim) -> BoxFuture<'a, ()> { + Box::pin(async move { + let _ = self + .call( + RELEASE, + json!({ "zone_id": current.zone_id, "holder": current.holder, + "token": current.token }), + ) + .await; + }) + } +} + +fn in_process(clock: &ManualClock, zone: &str) -> Arc { + Arc::new(InProcess { + cell: lease_cell(clock, zone), + calls: AtomicU64::new(0), + }) +} + +/// Outward event types in the cell's outbox, sorted (the reference host's +/// export is unordered; delivery order comes from `created_at`). +fn outward(cell: &AggregateCell) -> Vec { + let mut types = cell + .durable_outbox() + .unwrap() + .into_iter() + .map(|message| message.event_type) + .collect::>(); + types.sort(); + types +} + +#[tokio::test] +async fn the_lease_cell_passes_the_ownership_conformance_suite() { + let clock = clock(); + let own = in_process(&clock, "conformance"); + ownership_conformance(own.as_ref(), "conformance").await; + assert_eq!( + outward(&own.cell), + [ + "zone.activated", + "zone.activated", + "zone.activated", + "zone.released", + "zone.released", + "zone.released" + ] + ); +} + +#[tokio::test] +async fn renewals_produce_no_outbox_rows_and_move_the_expiry_alarm() { + let clock = clock(); + let own = in_process(&clock, "z1"); + let ttl = Duration::from_secs(10); + let mut current = own.acquire("z1", "a#1", ttl).await.unwrap(); + assert_eq!(outward(&own.cell), ["zone.activated"]); + assert_eq!( + own.cell.next_alarm_due().unwrap(), + Some(SystemTime::UNIX_EPOCH + Duration::from_millis(T0 + 10_000)) + ); + for step in 1..=20u64 { + clock.advance(Duration::from_secs(3)); + current = own.renew(¤t, ttl).await.unwrap(); + assert_eq!( + own.cell.next_alarm_due().unwrap(), + Some(clock.now() + ttl), + "step {step}" + ); + assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + } + assert_eq!( + outward(&own.cell), + ["zone.activated"], + "no heartbeat events" + ); + assert_eq!( + own.cell.load().await.unwrap().unwrap().holder(), + Some("a#1") + ); +} + +#[tokio::test] +async fn a_holder_that_stops_renewing_expires_exactly_once() { + let clock = clock(); + let own = in_process(&clock, "z2"); + let ttl = Duration::from_secs(10); + let held = own.acquire("z2", "a#1", ttl).await.unwrap(); + clock.advance(Duration::from_millis(9_999)); + assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + clock.advance(Duration::from_millis(1)); + let report = own.cell.fire_due_alarms().await.unwrap(); + assert_eq!(report.fired.len(), 1); + assert_eq!(report.fired[0].payload["expired"], true); + assert_eq!(outward(&own.cell), ["zone.activated", "zone.lease_expired"]); + clock.advance(Duration::from_secs(60)); + assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + assert_eq!(outward(&own.cell).len(), 2, "exactly one expiry"); + assert_eq!(own.renew(&held, ttl).await, Err(OwnershipError::Lost)); + let next = own.acquire("z2", "b#1", ttl).await.unwrap(); + assert!(next.token > held.token); +} + +#[tokio::test] +async fn a_renewal_racing_the_alarm_never_produces_a_false_expiry() { + let clock = clock(); + let own = in_process(&clock, "z3"); + let ttl = Duration::from_secs(10); + let held = own.acquire("z3", "a#1", ttl).await.unwrap(); + + // The alarm comes due, but the renewal lands before the wake. + clock.advance(ttl); + let held = own.renew(&held, ttl).await.unwrap(); + assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + + // The wake read the due alarm, then the renewal interleaved, then the + // stale expiry command runs: the handler re-reads the lease and keeps it. + clock.advance(ttl); + let stale = own.cell.armed_alarms().unwrap().remove(0); + let _held = own.renew(&held, ttl).await.unwrap(); + let result = own + .cell + .dispatch_idempotent( + EXPIRE, + &stale.command_identity().unwrap(), + stale.input.clone(), + cell_alarm_session(&stale), + ) + .await + .unwrap(); + assert_eq!(result.payload()["expired"], false); + assert_eq!(outward(&own.cell), ["zone.activated"]); + assert_eq!( + own.cell.load().await.unwrap().unwrap().holder(), + Some("a#1") + ); + + // Callers cannot run the expiry command themselves. + let forged = own + .cell + .dispatch_idempotent( + EXPIRE, + &CellCommandIdentity::new("zone-host", "zone-server-a", command_id_for("t", "forged")) + .unwrap(), + json!({ "zone_id": "z3" }), + host_session(), + ) + .await; + assert!(forged.is_err()); +} + +#[tokio::test] +async fn only_zone_hosts_may_acquire() { + let clock = clock(); + let cell = lease_cell(&clock, "z4"); + let mut player = Session::new(); + player.set(USER_ID_KEY, "player-1"); + let refused = cell + .dispatch_idempotent( + ACQUIRE, + &CellCommandIdentity::new("zone-host", "player-1", command_id_for("t", "p")).unwrap(), + json!({ "zone_id": "z4", "holder": "p#1", "ttl_ms": 1000 }), + player, + ) + .await; + assert!(refused.is_err()); + assert!(cell.load().await.unwrap().is_none()); +} diff --git a/examples/game/README.md b/examples/game/README.md index ce7ce5f47..8f73baae7 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -149,6 +149,15 @@ proxies to it) and **B** on the next port, each listing the other with is derived from it) and keeping hibernation snapshots under one `--state-dir` (`grid-A.sqlite3`, `grid-B.sqlite3`). +Zone ownership is process-local (`LocalOwnership`) by default. With +`CELLD_URL` and `DISTRIBUTED_INTERNAL_SECRET` set (or `--celld-url` / +`--internal-secret`), every host keeps its zone claims in celld +`ZoneLease:{zone}` cells (`CellZoneOwnership`; a celld project mounting +`distributed_game::world::zone_lease`, such as `tests/celld/fact-worker`), +and a host that stops renewing gets a `zone.lease_expired` from the cell's +alarm. The cluster smoke tests honour the same variables: +`CELLD_URL=… DISTRIBUTED_INTERNAL_SECRET=… cargo test --test grid_smoke`. + - **World.** Cell `(x, y)` is zone `grid..` (canonical integers, |x|,|y| ≤ 10⁶; the server uses `FlatTopology`, the dots are not a hierarchy). Its terrain height, biome and props are a pure function of diff --git a/examples/game/server/Cargo.lock b/examples/game/server/Cargo.lock index 04d21531d..8f89cca84 100644 --- a/examples/game/server/Cargo.lock +++ b/examples/game/server/Cargo.lock @@ -845,12 +845,14 @@ dependencies = [ "distributed_game_macros", "futures-util", "hmac 0.12.1", + "reqwest", "rusqlite", "serde", "serde_json", "sha2 0.10.9", "tokio", "tokio-tungstenite 0.30.0", + "uuid", ] [[package]] diff --git a/examples/game/server/Cargo.toml b/examples/game/server/Cargo.toml index 06ecc2518..ee5ab2c11 100644 --- a/examples/game/server/Cargo.toml +++ b/examples/game/server/Cargo.toml @@ -29,7 +29,7 @@ path = "src/main.rs" game_sim = { path = "sim" } # The Crawler aggregate + RecordKill (shared with the read-model service). mobpit_crawler = { path = "crawler" } -distributed_game = { path = "../../../distributed_game", features = ["net", "cell", "sqlite"] } +distributed_game = { path = "../../../distributed_game", features = ["net", "cell", "celld", "sqlite"] } # Mob Pit: native durable SQLite cells + sorted-set leaderboards (Redis optional # at runtime) + the SQLite bus for the read-model service. distributed = { path = "../../..", features = ["cell-sqlite", "redis", "sqlite"] } diff --git a/examples/game/server/src/cluster.rs b/examples/game/server/src/cluster.rs index a6eb1b71d..ad3e753d0 100644 --- a/examples/game/server/src/cluster.rs +++ b/examples/game/server/src/cluster.rs @@ -11,7 +11,10 @@ //! - the admin secret, derived from the shared ticket secret (all hosts of //! a world must be started with the same `--secret`); //! - fenced zone claims (`LocalOwnership`, whose tokens are seeded from the -//! process start time so they outrank snapshots of an earlier run) and SQLite hibernation +//! process start time so they outrank snapshots of an earlier run; or, +//! with `--celld-url`, `CellZoneOwnership`: one celld `ZoneLease` cell per +//! zone, whose alarm declares `zone.lease_expired` when this host stops +//! renewing) and SQLite hibernation //! snapshots ([`SqliteZoneStateStore`], one file per host under //! `--state-dir`, in-memory without it); //! - guest display names shared with every peer: `/dev/ticket` on any host @@ -33,10 +36,12 @@ use axum::middleware::{self, Next}; use axum::response::Response; use axum::routing::post; use axum::{Json, Router}; +use distributed::cell_host::InternalHttpSecret; +use distributed_game::celld::CellZoneOwnership; use distributed_game::net::peer::{post_json, ADMIN_SECRET_HEADER}; use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; use distributed_game::world::{ - HostAddr, LocalOwnership, SqliteZoneStateStore, ZoneDirectory, ZoneStateStore, + HostAddr, LocalOwnership, SqliteZoneStateStore, ZoneDirectory, ZoneOwnership, ZoneStateStore, }; use serde_json::{json, Value}; @@ -65,6 +70,14 @@ pub struct ClusterArgs { /// Dormant after half of this without clients. #[arg(long, default_value_t = 4000)] pub hibernate_after_ms: u64, + /// celld URL serving `ZoneLease` cells (kind `zone-lease`): zone + /// ownership lives there instead of in this process (needs + /// `DISTRIBUTED_INTERNAL_SECRET`). + #[arg(long, env = "CELLD_URL")] + pub celld_url: Option, + /// Internal secret for celld cells (with `--celld-url`). + #[arg(long, env = "DISTRIBUTED_INTERNAL_SECRET", hide_env_values = true)] + pub internal_secret: Option, } /// In-process cluster options (tests and [`run_cluster`]). @@ -78,6 +91,9 @@ pub struct ClusterOptions { pub state_dir: Option, /// Dormant time before hibernation. pub hibernate_after: Duration, + /// Zone ownership in celld `ZoneLease` cells: `(url, internal secret)`. + /// `None` = process-local `LocalOwnership`. + pub celld: Option<(String, String)>, } impl ClusterOptions { @@ -88,6 +104,7 @@ impl ClusterOptions { peers: Vec::new(), state_dir: None, hibernate_after, + celld: None, } } } @@ -250,10 +267,22 @@ pub fn spawn_cluster( config.admin_secret = Some(world.admin_secret.clone()); config.hibernate_after = Some(cluster.hibernate_after); config.host.dormant_after = cluster.hibernate_after / 2; + let ownership: Arc = match &cluster.celld { + Some((url, secret)) => Arc::new( + CellZoneOwnership::new( + url.clone(), + InternalHttpSecret::new(secret.clone()).map_err(std::io::Error::other)?, + format!("game-demos-{}", D::NAME), + world.host.id.clone(), + ) + .map_err(std::io::Error::other)?, + ), + None => Arc::new(LocalOwnership::new()), + }; let server = D::cluster_server(config, &world) .with_host_id(world.host.id.clone()) .with_directory(directory) - .with_ownership(Arc::new(LocalOwnership::new())) + .with_ownership(ownership) .with_state_store(world.state.clone() as Arc); let extra = if options.dev { @@ -303,6 +332,15 @@ pub async fn run_cluster(args: ClusterArgs) -> std::io::Result<( peers, state_dir: args.state_dir, hibernate_after: Duration::from_millis(args.hibernate_after_ms.max(100)), + celld: match (args.celld_url, args.internal_secret) { + (Some(url), Some(secret)) => Some((url, secret)), + (Some(_), None) => { + return Err(std::io::Error::other( + "--celld-url needs DISTRIBUTED_INTERNAL_SECRET (or --internal-secret)", + )) + } + _ => None, + }, }; let running = spawn_cluster::(listener, options, cluster, common::shutdown_signal())?; // Parsed by run.sh and the client tests: keep the format. diff --git a/examples/game/server/tests/support/mod.rs b/examples/game/server/tests/support/mod.rs index 6835a4757..07d620900 100644 --- a/examples/game/server/tests/support/mod.rs +++ b/examples/game/server/tests/support/mod.rs @@ -86,6 +86,11 @@ pub async fn start_cluster(ids: &[&str], hibernate_after: Durati peers: all.clone(), state_dir: Some(state_dir.clone()), hibernate_after, + // With CELLD_URL + DISTRIBUTED_INTERNAL_SECRET the hosts keep + // zone ownership in celld ZoneLease cells (same smoke tests). + celld: std::env::var("CELLD_URL") + .ok() + .zip(std::env::var("DISTRIBUTED_INTERNAL_SECRET").ok()), }; let host = spawn_cluster::(listener, DemoOptions::dev(SECRET), options, async move { let _ = stopped.await; diff --git a/src/microsvc/cell_host/alarm.rs b/src/microsvc/cell_host/alarm.rs index 18270ddc9..2c74bb608 100644 --- a/src/microsvc/cell_host/alarm.rs +++ b/src/microsvc/cell_host/alarm.rs @@ -54,6 +54,10 @@ pub const CELL_ALARM_NAME_CLAIM: &str = "x-distributed-cell-alarm"; pub const CELL_ALARM_DUE_CLAIM: &str = "x-distributed-cell-alarm-due-ms"; /// Session claim with the fired alarm's generation. pub const CELL_ALARM_GENERATION_CLAIM: &str = "x-distributed-cell-alarm-generation"; +/// Session claim with the cell clock's time at dispatch (epoch ms). The cell +/// host overwrites it on every command, so callers cannot choose it; read it +/// with `CausalCommandContext::cell_now`. +pub const CELL_NOW_CLAIM: &str = "x-distributed-cell-now-ms"; /// Command-ledger service id of fired alarms. pub const CELL_ALARM_SERVICE_ID: &str = "distributed.cell-alarm"; /// Longest alarm name. @@ -395,6 +399,16 @@ pub fn cell_alarm_session(alarm: &ArmedCellAlarm) -> Session { session } +/// The cell clock's time at dispatch, when the command runs in a cell. +pub fn cell_now(session: &Session) -> Option { + session + .get(CELL_NOW_CLAIM)? + .parse::() + .ok() + .filter(|ms| *ms >= 0) + .map(ms_to_system_time) +} + /// Whether `session` is the trusted session of a fired alarm. pub fn is_cell_alarm_session(session: &Session) -> bool { session.user_id() == Some(CELL_ALARM_USER_ID) diff --git a/src/microsvc/cell_host/cell.rs b/src/microsvc/cell_host/cell.rs index 7943826dc..cda5766da 100644 --- a/src/microsvc/cell_host/cell.rs +++ b/src/microsvc/cell_host/cell.rs @@ -226,6 +226,7 @@ where input: Value, session: Session, ) -> Result { + let session = self.stamp_now(session); #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] let _command = self .begin_command() @@ -252,6 +253,7 @@ where input: Value, session: Session, ) -> Result { + let session = self.stamp_now(session); #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] let _command = self .begin_command() @@ -266,6 +268,14 @@ where result } + /// Overwrite the cell-clock claim: handlers read the cell's time + /// (`ctx.cell_now()`), never a caller's. + fn stamp_now(&self, mut session: Session) -> Session { + let now = super::alarm::system_time_to_ms(self.routes.repo().repo().now()).unwrap_or(0); + session.set(super::alarm::CELL_NOW_CLAIM, now.to_string()); + session + } + /// Armed alarms of this cell, earliest first. pub fn armed_alarms(&self) -> Result, RepositoryError> { self.routes.repo().repo().armed_alarms() diff --git a/src/microsvc/cell_host/mod.rs b/src/microsvc/cell_host/mod.rs index ed903a0a3..5e3881bc7 100644 --- a/src/microsvc/cell_host/mod.rs +++ b/src/microsvc/cell_host/mod.rs @@ -49,12 +49,13 @@ mod wire; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] pub use alarm::DurableCellAlarm; pub use alarm::{ - cell_alarm_due, cell_alarm_guard, cell_alarm_session, is_cell_alarm_session, ArmedCellAlarm, - CellAlarm, CellAlarmError, CellAlarmFireReport, CellAlarmWrite, FailedCellAlarm, - FiredCellAlarm, CELL_ALARM_DUE_CLAIM, CELL_ALARM_FIRE_BATCH, CELL_ALARM_GENERATION_CLAIM, - CELL_ALARM_NAME_CLAIM, CELL_ALARM_RETRY_BASE, CELL_ALARM_RETRY_MAX, CELL_ALARM_ROLE, - CELL_ALARM_SERVICE_ID, CELL_ALARM_USER_ID, MAX_ARMED_CELL_ALARMS, MAX_CELL_ALARM_INPUT_BYTES, - MAX_CELL_ALARM_NAME_BYTES, MAX_CELL_ALARM_WRITES_PER_COMMAND, + cell_alarm_due, cell_alarm_guard, cell_alarm_session, cell_now, is_cell_alarm_session, + ArmedCellAlarm, CellAlarm, CellAlarmError, CellAlarmFireReport, CellAlarmWrite, + FailedCellAlarm, FiredCellAlarm, CELL_ALARM_DUE_CLAIM, CELL_ALARM_FIRE_BATCH, + CELL_ALARM_GENERATION_CLAIM, CELL_ALARM_NAME_CLAIM, CELL_ALARM_RETRY_BASE, + CELL_ALARM_RETRY_MAX, CELL_ALARM_ROLE, CELL_ALARM_SERVICE_ID, CELL_ALARM_USER_ID, + CELL_NOW_CLAIM, MAX_ARMED_CELL_ALARMS, MAX_CELL_ALARM_INPUT_BYTES, MAX_CELL_ALARM_NAME_BYTES, + MAX_CELL_ALARM_WRITES_PER_COMMAND, }; pub use causal::{ CellCommandIdentity, CellDispatchError, CellDispatchResult, CELL_CAUSATION_ID_HEADER, diff --git a/src/microsvc/cell_host/tests/alarms.rs b/src/microsvc/cell_host/tests/alarms.rs index 4d161c475..5e69293be 100644 --- a/src/microsvc/cell_host/tests/alarms.rs +++ b/src/microsvc/cell_host/tests/alarms.rs @@ -517,3 +517,31 @@ async fn sqlite_cell_alarms_commit_atomically_and_survive_reopening() { drop(cell); let _ = std::fs::remove_dir_all(root); } + +#[tokio::test] +async fn handlers_see_the_cell_clock_and_callers_cannot_forge_it() { + let clock = clock(); + let cell = ticker_cell(&clock); + let mut forged = owner_session(); + forged.set(super::super::CELL_NOW_CLAIM, "1"); + cell.dispatch_idempotent( + "ticker.start", + &identity("0190a000-0000-7000-8000-000000000009"), + json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" }), + forged, + ) + .await + .unwrap(); + // `after(period)` resolved by the cell clock, not the forged claim. + assert_eq!( + cell.next_alarm_due().unwrap(), + Some(clock.now() + Duration::from_millis(1000)) + ); + let mut session = crate::microsvc::Session::new(); + assert_eq!(super::super::cell_now(&session), None); + session.set(super::super::CELL_NOW_CLAIM, "1234"); + assert_eq!( + super::super::cell_now(&session), + Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1234)) + ); +} diff --git a/src/microsvc/service/handlers.rs b/src/microsvc/service/handlers.rs index acbbf70a0..58c16c3bd 100644 --- a/src/microsvc/service/handlers.rs +++ b/src/microsvc/service/handlers.rs @@ -706,6 +706,13 @@ where .map_err(workspace_handler_error) } + /// The cell clock's time when this command was dispatched (the Durable + /// Object clock on celld, the store clock — e.g. a `ManualClock` — on the + /// reference host). `None` outside an aggregate cell. + pub fn cell_now(&self) -> Option { + crate::cell_host::cell_now(self.session) + } + /// Due time of the alarm that fired this command; `None` for any command /// not dispatched by a cell alarm. pub fn alarm_due(&self) -> Option { diff --git a/tests/celld/fact-worker/src/lib.rs b/tests/celld/fact-worker/src/lib.rs index 77d5f6719..c2b05b154 100644 --- a/tests/celld/fact-worker/src/lib.rs +++ b/tests/celld/fact-worker/src/lib.rs @@ -6,6 +6,9 @@ //! - `Presence:{principal}`: the runtime lifecycle facts `zone.player_joined` //! and `zone.player_left` (lifecycle facts go to presence, never to the //! player's gameplay cells). +//! - `ZoneLease:{zone_id}` (kind `zone-lease`): zone ownership +//! (`distributed_game::world::zone_lease`) for `CellZoneOwnership`; its +//! alarm declares `zone.lease_expired`. //! //! Every route requires `DISTRIBUTED_INTERNAL_SECRET`. The handler's session //! is rebuilt from the trusted fact headers (`fact_session_from_headers`): @@ -22,6 +25,7 @@ use distributed::microsvc::{ }; use distributed::{sourced, Aggregate, Entity}; use distributed_game::cell::{fact_session_from_headers, session_stamp}; +use distributed_game::world::zone_lease::{mount_zone_lease, ZoneLease}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use worker::*; @@ -273,7 +277,17 @@ where } _ => return error_response(CellDispatchError::Unauthorized), }; - let session = fact_session_from_headers(header).unwrap_or_else(Session::new); + // Facts carry principal + stamp; zone hosts (lease commands) carry a + // user and role. Both arrive only on the authenticated internal path. + let session = fact_session_from_headers(header).unwrap_or_else(|| { + let mut session = Session::new(); + for key in [distributed::USER_ID_KEY, distributed::microsvc::ROLE_KEY] { + if let Some(value) = header(key) { + session.set(key, value); + } + } + session + }); let result = cell .dispatch_idempotent(command, &identity, parsed.input, session) .await; @@ -317,13 +331,25 @@ where .load() .await .map_err(|error| Error::RustError(error.to_string()))?; - let events = cell + let history = cell .history() .await + .map_err(|error| Error::RustError(error.to_string()))?; + let names = history + .iter() + .map(|event| event.event_name.clone()) + .collect::>(); + let armed = cell + .armed_alarms() .map_err(|error| Error::RustError(error.to_string()))? .len(); json_status( - json!({ "events": events, "state": loaded.as_ref().map(state) }), + json!({ + "events": history.len(), + "eventNames": names, + "armedAlarms": armed, + "state": loaded.as_ref().map(state), + }), 200, ) } @@ -392,6 +418,48 @@ cell_object!( |presence: &Presence| json!({ "transitions": presence.transitions }) ); +#[durable_object] +pub struct ZoneLeaseCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ZoneLeaseCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let outbox = CelldOutbox::from_env(&env, "OUTBOX").expect("OUTBOX Queue binding"); + // Renewals are internal events: snapshots keep loads short. + let cell = mount_zone_lease( + AggregateCell::::from_state_with_snapshots(state, 50) + .expect("zone lease cell identity"), + ) + .with_celld_outbox(outbox); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route( + &self.env, + &self.cell, + req, + |lease: &ZoneLease| json!({ "holder": lease.holder(), "token": lease.token() }), + ) + .await + } + + async fn alarm(&self) -> Result { + let wake = self + .cell + .alarm(&self.env) + .await + .map_err(|error| Error::RustError(error.to_string()))?; + for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + worker::console_error!("lease alarm failed: {} {}", failed.code, failed.message); + } + Response::ok("ok") + } +} + #[event(fetch)] async fn main(req: Request, env: Env, _ctx: Context) -> Result { console_error_panic_hook::set_once(); @@ -408,6 +476,7 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { Some("kills") => "KILLS", Some("match") => "MATCH", Some("presence") => "PRESENCE", + Some("zone-lease") => "ZONE_LEASE", _ => return json_status(json!({ "error": "unknown cell kind" }), 404), }; let Some(shard) = parts.get(1) else { diff --git a/tests/celld/fact-worker/wrangler.jsonc b/tests/celld/fact-worker/wrangler.jsonc index 194f46dc1..8d890016c 100644 --- a/tests/celld/fact-worker/wrangler.jsonc +++ b/tests/celld/fact-worker/wrangler.jsonc @@ -6,11 +6,13 @@ "bindings": [ { "name": "KILLS", "class_name": "KillsCell" }, { "name": "MATCH", "class_name": "MatchCell" }, - { "name": "PRESENCE", "class_name": "PresenceCell" } + { "name": "PRESENCE", "class_name": "PresenceCell" }, + { "name": "ZONE_LEASE", "class_name": "ZoneLeaseCell" } ] }, "migrations": [ - { "tag": "v1", "new_sqlite_classes": ["KillsCell", "MatchCell", "PresenceCell"] } + { "tag": "v1", "new_sqlite_classes": ["KillsCell", "MatchCell", "PresenceCell"] }, + { "tag": "v2", "new_sqlite_classes": ["ZoneLeaseCell"] } ], "queues": { "producers": [ From 4fc1408cb42c212f3563dab17507de66d1337d54 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 11:40:23 -0500 Subject: [PATCH 13/40] fix(game): order same-tick facts and report dropped sockets The Mob Pit presence smoke test failed about half the time with a dropped player stuck Online. Root cause: FactStamp had no order inside one tick. A join and a dropped socket in the same tick and millisecond got equal stamps, so emission_order returned Equal and the presence projection dropped ConnectionLost as stale. - FactStamp gains `seq`, the host's per-activation emission sequence; every stamped fact takes the next number and emission_order compares it within one zone activation. It travels as `game.seq` metadata and the `x-game-seq` session variable (breaking: required on read). - The connection task now tells the zone when a socket write fails. Before, a write error skipped the Disconnect and the zone only found out if a later frame to that client failed to queue. - distributed_cli: the generation-rejecting readiness probe gets a 1500 ms budget; at 100 ms the initial probe timed out under a loaded workspace run and the supervisor exited before the first assertion. Refs [[tasks/gr-p19-zone-lifecycle-facts]] --- distributed_cli/tests/cli_lifecycle.rs | 7 +++- distributed_game/README.md | 10 +++-- distributed_game/src/celld.rs | 1 + distributed_game/src/host.rs | 14 ++++++- distributed_game/src/net/mod.rs | 16 ++++++-- distributed_game/src/stamp.rs | 45 ++++++++++++++++++--- distributed_game/tests/cell_sink/main.rs | 1 + distributed_game/tests/celld_facts/main.rs | 1 + distributed_game/tests/lifecycle/main.rs | 10 +++++ examples/game/server/src/mobpit/presence.rs | 27 +++++++++++++ 10 files changed, 115 insertions(+), 17 deletions(-) diff --git a/distributed_cli/tests/cli_lifecycle.rs b/distributed_cli/tests/cli_lifecycle.rs index 72d2363c4..cd9d7f193 100644 --- a/distributed_cli/tests/cli_lifecycle.rs +++ b/distributed_cli/tests/cli_lifecycle.rs @@ -277,11 +277,16 @@ exit 1 let path = root.join("distributed.lifecycle.json"); let mut config: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); config["dev"]["prepare_ms"] = serde_json::json!(100); + // The probe rejects a replacement by never passing, so this budget is + // also how long a rejection takes. It must still leave headroom for the + // initial generation's probe (a `/bin/sh` spawn) under a loaded + // `cargo test --workspace`: at 100 ms that probe timed out and the + // supervisor failed before the test's first assertion. config["dev"]["processes"]["api"]["ready"] = serde_json::json!({ "program": "/bin/sh", "args": ["{root}/generation-probe.sh", "{root}"], "interval_ms": 10, - "timeout_ms": 100 + "timeout_ms": 1500 }); fs::write(path, serde_json::to_vec_pretty(&config).unwrap()).unwrap(); } diff --git a/distributed_game/README.md b/distributed_game/README.md index 0a847a22a..c3e8412d7 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -701,12 +701,14 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), (`/dev/mobpit/presence`), ordered by stamp. - Every fact carries a `FactStamp`: zone, activation (boot epoch), tick, host wall time `emitted_at_ms` (`Clock::unix_ms`; `ManualClock` in - tests), content version and config version (`HostConfig::config_version`, - `ctx.set_config_version`). The cell sink passes it to handlers as + tests), content version, config version (`HostConfig::config_version`, + `ctx.set_config_version`) and `seq`, the host's per-activation emission + sequence (every stamped fact gets the next number). The cell sink passes it to handlers as trusted `x-game-*` session variables (`cell::session_stamp`), and `stamp_metadata` / `FactStamp::from_metadata` copy it onto events. - **Commit order is not emission order.** Order facts with - `FactStamp::emission_order` (same zone: activation, then tick; across + `FactStamp::emission_order` (same zone: activation, then `seq`, so two + facts of one tick and millisecond still order strictly; across zones: wall time). After a handoff the target's stamps start after the source's departure time, which the transfer ticket carries, so "joined B" sorts after "left A" even when it commits first or the two hosts' @@ -901,7 +903,7 @@ Settlements: `settlement.is::()`, and > do it for you. The host binds the principal from the player's **verified ticket** and -stamps zone, activation, tick, wall time, content and config version. A `FactSink` commits facts on a worker +stamps zone, activation, tick, wall time, content and config version and emission sequence. A `FactSink` commits facts on a worker with bounded retries/backoff; `Zone::on_fact_settled` receives the outcome on a later tick. `cell::CellFactSink` dispatches into an `AggregateCell` with a deterministic command id per fact key, so retries replay instead of diff --git a/distributed_game/src/celld.rs b/distributed_game/src/celld.rs index e770102e8..e60dee140 100644 --- a/distributed_game/src/celld.rs +++ b/distributed_game/src/celld.rs @@ -595,6 +595,7 @@ mod tests { emitted_at_ms: 1_700_000_000_000, activation: 3, config_version: "cfg-2".into(), + seq: 5, }; let fact = Fact::trusted( "k", diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index c76f6195b..a9ad850a7 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -273,6 +273,8 @@ pub(crate) struct Core { dropped_frozen: u64, /// Typed facts keyed without dedupe parts this tick: `(tick, count)`. fact_seq: (u64, u32), + /// Facts stamped this activation ([`FactStamp::seq`] of the latest). + stamped: u64, /// Stamps are never earlier than this (Unix ms): raised past a handoff /// source's departure time when a transfer is admitted. stamp_floor_ms: u64, @@ -328,7 +330,7 @@ impl Core { let Ok(fact) = Fact::typed(key, &input) else { return; }; - let fact = fact.bind(Some(principal.to_string()), self.stamp()); + let fact = fact.bind(Some(principal.to_string()), self.stamp_fact()); self.lifecycle_backlog.push_back(fact); self.flush_lifecycle(); } @@ -380,8 +382,15 @@ impl Core { emitted_at_ms: self.stamp_ms(), activation: self.config.boot_epoch, config_version: self.config.config_version.clone(), + seq: self.stamped, } } + + /// Stamp for a fact: the next emission sequence number. + fn stamp_fact(&mut self) -> FactStamp { + self.stamped += 1; + self.stamp() + } } /// Hook context: everything a zone may touch besides its own state. @@ -618,7 +627,7 @@ impl Ctx<'_, Z> { } fn emit_bound(&mut self, fact: Fact, principal: Option) -> Result<(), EmitError> { - let stamp = self.core.stamp(); + let stamp = self.core.stamp_fact(); self.core .facts .as_mut() @@ -840,6 +849,7 @@ impl ZoneHost { child_inbox: Vec::new(), dropped_frozen: 0, fact_seq: (0, 0), + stamped: 0, stamp_floor_ms: 0, pinned_stamp_ms: None, lifecycle_backlog: VecDeque::new(), diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index 892c30324..4ef4ca156 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -1896,10 +1896,14 @@ where while let Some(item) = outbound.pop_due() { match send_out(&mut socket, item).await { Ok(true) => {} - _ => { + // The zone closed this connection and already forgot the client. + Ok(false) => { notify_zone = false; break 'session; } + // The socket died under a write: the zone must still learn + // the connection is gone (grace + ConnectionLost). + Err(_) => break 'session, } } while let Some(item) = inbound.pop_due() { @@ -1915,9 +1919,13 @@ where out = out_rx.recv() => { let Some(out) = out else { notify_zone = false; break }; if let Some(out) = outbound.push(out) { - if !send_out(&mut socket, out).await.unwrap_or(false) { - notify_zone = false; - break; + match send_out(&mut socket, out).await { + Ok(true) => {} + Ok(false) => { + notify_zone = false; + break; + } + Err(_) => break, } } } diff --git a/distributed_game/src/stamp.rs b/distributed_game/src/stamp.rs index cd237c528..3268500db 100644 --- a/distributed_game/src/stamp.rs +++ b/distributed_game/src/stamp.rs @@ -12,7 +12,9 @@ //! per-principal lanes and retries, and two zones commit independently. A //! handler that must order facts (presence: "left A" vs "joined B" after a //! handoff) compares stamps with [`FactStamp::emission_order`]. Within one -//! zone activation the tick decides; across activations of one zone the +//! zone activation the emission sequence `seq` decides (the host numbers +//! every fact it stamps, so two facts of the same tick and millisecond +//! still have a strict order); across activations of one zone the //! activation (it grows across restarts); across zones the host wall time //! `emitted_at_ms`. A handoff target never stamps the arriving player's //! facts earlier than the source stamped its departure: the transfer ticket @@ -34,6 +36,8 @@ pub const META_EMITTED_AT_MS: &str = "game.emitted_at_ms"; pub const META_ACTIVATION: &str = "game.activation"; /// Metadata key for the config version. pub const META_CONFIG_VERSION: &str = "game.config_version"; +/// Metadata key for the per-activation emission sequence (decimal). +pub const META_SEQ: &str = "game.seq"; /// Session-variable prefix used to carry a stamp through a trusted Session. pub const SESSION_PREFIX: &str = "x-game-"; @@ -58,11 +62,17 @@ pub struct FactStamp { /// Remote config version the zone was running /// ([`HostConfig::config_version`](crate::HostConfig::config_version)). pub config_version: String, + /// Emission sequence within the activation: the host numbers every fact + /// it stamps 1, 2, 3, ... so facts of one tick (a join and a dropped + /// socket in the same tick and millisecond) keep a strict order. A stamp + /// taken for anything else ([`Ctx::stamp`](crate::Ctx::stamp)) carries + /// the sequence of the last fact stamped before it. + pub seq: u64, } impl FactStamp { /// Metadata entries for this stamp. - pub fn entries(&self) -> [(&'static str, String); 6] { + pub fn entries(&self) -> [(&'static str, String); 7] { [ (META_ZONE, self.zone_id.clone()), (META_TICK, self.tick.to_string()), @@ -70,13 +80,14 @@ impl FactStamp { (META_EMITTED_AT_MS, self.emitted_at_ms.to_string()), (META_ACTIVATION, self.activation.to_string()), (META_CONFIG_VERSION, self.config_version.clone()), + (META_SEQ, self.seq.to_string()), ] } /// Session variables (`x-game-zone`, `x-game-tick`, /// `x-game-content-version`, `x-game-emitted-at-ms`, - /// `x-game-activation`, `x-game-config-version`). - pub fn session_vars(&self) -> [(String, String); 6] { + /// `x-game-activation`, `x-game-config-version`, `x-game-seq`). + pub fn session_vars(&self) -> [(String, String); 7] { [ (format!("{SESSION_PREFIX}zone"), self.zone_id.clone()), (format!("{SESSION_PREFIX}tick"), self.tick.to_string()), @@ -96,6 +107,7 @@ impl FactStamp { format!("{SESSION_PREFIX}config-version"), self.config_version.clone(), ), + (format!("{SESSION_PREFIX}seq"), self.seq.to_string()), ] } @@ -110,6 +122,7 @@ impl FactStamp { emitted_at_ms: get("emitted-at-ms")?.parse().ok()?, activation: get("activation")?.parse().ok()?, config_version: get("config-version")?.clone(), + seq: get("seq")?.parse().ok()?, }) } @@ -123,17 +136,20 @@ impl FactStamp { emitted_at_ms: metadata.get(META_EMITTED_AT_MS)?.parse().ok()?, activation: metadata.get(META_ACTIVATION)?.parse().ok()?, config_version: metadata.get(META_CONFIG_VERSION)?.clone(), + seq: metadata.get(META_SEQ)?.parse().ok()?, }) } - /// Emission order of two stamps: same zone and activation by tick (then - /// wall time); same zone across activations by activation; different + /// Emission order of two stamps: same zone and activation by emission + /// sequence (then tick and wall time); same zone across activations by + /// activation; different /// zones by wall time (then zone id, activation and tick, so the order /// is total and deterministic). Commit order says nothing about this. pub fn emission_order(&self, other: &Self) -> Ordering { if self.zone_id == other.zone_id { self.activation .cmp(&other.activation) + .then(self.seq.cmp(&other.seq)) .then(self.tick.cmp(&other.tick)) .then(self.emitted_at_ms.cmp(&other.emitted_at_ms)) } else { @@ -141,6 +157,7 @@ impl FactStamp { .cmp(&other.emitted_at_ms) .then_with(|| self.zone_id.cmp(&other.zone_id)) .then(self.activation.cmp(&other.activation)) + .then(self.seq.cmp(&other.seq)) .then(self.tick.cmp(&other.tick)) } } @@ -177,6 +194,8 @@ mod tests { emitted_at_ms: at, activation, config_version: "combat@12".into(), + // Ticks only grow within an activation, so neither does seq. + seq: tick, } } @@ -192,6 +211,7 @@ mod tests { assert_eq!(meta[META_EMITTED_AT_MS], "1700000000123"); assert_eq!(meta[META_ACTIVATION], "9"); assert_eq!(meta[META_CONFIG_VERSION], "combat@12"); + assert_eq!(meta[META_SEQ], "42"); assert_eq!(FactStamp::from_metadata(&meta), Some(stamp)); } @@ -222,5 +242,18 @@ mod tests { // Ties are broken deterministically. let (x, y) = (stamp("a", 1, 1, 5), stamp("b", 1, 1, 5)); assert_eq!(x.emission_order(&y), y.emission_order(&x).reverse()); + // Same tick and millisecond: the emission sequence decides. + let (first, second) = ( + FactStamp { + seq: 7, + ..stamp("a", 1, 3, 9) + }, + FactStamp { + seq: 8, + ..stamp("a", 1, 3, 9) + }, + ); + assert!(second.emitted_after(&first)); + assert!(!first.emitted_after(&second)); } } diff --git a/distributed_game/tests/cell_sink/main.rs b/distributed_game/tests/cell_sink/main.rs index 1eca5c85f..2bb062127 100644 --- a/distributed_game/tests/cell_sink/main.rs +++ b/distributed_game/tests/cell_sink/main.rs @@ -122,6 +122,7 @@ async fn facts_dispatch_idempotently_with_trusted_principal() { emitted_at_ms: 1_700_000_000_456, activation: 77, config_version: "combat@12".into(), + seq: 12, }; let fact = Fact::trusted( "coin:z1:7", diff --git a/distributed_game/tests/celld_facts/main.rs b/distributed_game/tests/celld_facts/main.rs index 084e018eb..570f96acf 100644 --- a/distributed_game/tests/celld_facts/main.rs +++ b/distributed_game/tests/celld_facts/main.rs @@ -64,6 +64,7 @@ fn stamp(tick: u64) -> FactStamp { emitted_at_ms: 1_700_000_000_000 + tick, activation: 1, config_version: "cfg-1".into(), + seq: tick, } } diff --git a/distributed_game/tests/lifecycle/main.rs b/distributed_game/tests/lifecycle/main.rs index cbf7f5dd7..1b1a5c67d 100644 --- a/distributed_game/tests/lifecycle/main.rs +++ b/distributed_game/tests/lifecycle/main.rs @@ -110,6 +110,16 @@ fn of(facts: &[Fact], principal: &str) -> Vec { fn the_host_emits_every_transition_bound_to_the_verified_principal() { let (facts, hooks) = run_session(7); assert!(facts.iter().all(|f| f.stamp().zone_id == "z1")); + // Every stamp is strictly later than the previous one, even for + // transitions in the same tick and millisecond (emission sequence). + for pair in facts.windows(2) { + assert!( + pair[1].stamp().emitted_after(pair[0].stamp()), + "{:?} then {:?}", + pair[0].stamp(), + pair[1].stamp() + ); + } let ana = of(&facts, "ana"); let commands: Vec<_> = facts .iter() diff --git a/examples/game/server/src/mobpit/presence.rs b/examples/game/server/src/mobpit/presence.rs index eaabe1e40..960cff9c9 100644 --- a/examples/game/server/src/mobpit/presence.rs +++ b/examples/game/server/src/mobpit/presence.rs @@ -110,6 +110,7 @@ mod tests { tick, emitted_at_ms: at, activation, + seq: tick, ..FactStamp::default() } } @@ -137,4 +138,30 @@ mod tests { // Replaying the same fact is a no-op. assert!(!presence.apply("ana", &stamp("pit-2", 4, 2, 1_001), &joined_b)); } + + #[test] + fn a_drop_in_the_join_tick_and_millisecond_still_marks_the_player_away() { + use distributed_game::lifecycle::{ConnectionLost, LostCause}; + let presence = Presence::default(); + let joined = Lifecycle::Joined(PlayerJoined { + zone: "pit-1".into(), + via: JoinVia::Ticket, + from_zone: None, + }); + let lost = Lifecycle::ConnectionLost(ConnectionLost { + zone: "pit-1".into(), + grace_ms: 15_000, + cause: LostCause::Dropped, + }); + // Same zone, activation, tick and wall time: only the host's + // emission sequence orders them. + let at = |seq| FactStamp { + seq, + ..stamp("pit-1", 3, 0, 1_000) + }; + assert!(presence.apply("bo", &at(1), &joined)); + assert!(presence.apply("bo", &at(2), &lost)); + assert_eq!(presence.get("bo").unwrap().status, PresenceStatus::Away); + assert!(!presence.apply("bo", &at(1), &joined), "stale join ignored"); + } } From 2e04351c2defe997cffea8a3509adc8a8f90a7c8 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 11:59:30 -0500 Subject: [PATCH 14/40] feat(game): project zone entities into replicated views Zones are aggregates, their players and mobs are entities, views are read models and facts are domain events. Views stay projections, not copies; only the hand-written replicate() loop goes. - `#[replicated(.., of = Entity)]` with `#[from]` / `#[from(a.b)]` fields generates `View::project(&entity, ViewRest { .. })`; when all fields are `#[from]` it also implements `Project`. A missing or mistyped `#[from]` field is a compile error at the attribute (trybuild). Views without `of` are unchanged. - New `distributed_game::entity`: `EntityId`, `Entity`, `Project`, `EntityCollection`, `Entities` and `WorldView::project`. - `Zone::entities()` registers collections once; the host projects them each patch (`ZoneHost::world_view`), then calls `Zone::replicate`, which now defaults to nothing. `WorldView::add` takes `impl Into`. - Mob Pit has no replicate(): players, mobs, pillars and the zone's own HUD view are projected. A golden test runs a 900-tick fight and checks every projected world and Delta frame byte-for-byte against the old loop; generated TypeScript is unchanged (`run.sh gen --check`). - README maps zone/entity/view/fact to aggregate/entity/read model/ domain event. Refs [[tasks/gr-p16-game-entity]] --- distributed_game/README.md | 80 +++- distributed_game/src/authoring.rs | 4 +- distributed_game/src/entity.rs | 245 ++++++++++++ distributed_game/src/host.rs | 21 +- distributed_game/src/lib.rs | 2 + distributed_game/src/zone.rs | 14 +- distributed_game/tests/entities/main.rs | 247 ++++++++++++ distributed_game_macros/src/lib.rs | 22 +- distributed_game_macros/src/replicated.rs | 116 +++++- .../replicated_from_missing_field.rs | 21 + .../replicated_from_missing_field.stderr | 7 + .../compile_fail/replicated_from_needs_of.rs | 11 + .../replicated_from_needs_of.stderr | 5 + .../replicated_from_wrong_type.rs | 27 ++ .../replicated_from_wrong_type.stderr | 5 + examples/game/server/src/arena/mod.rs | 57 ++- examples/game/server/src/mobpit/zone.rs | 369 ++++++++++++++---- 17 files changed, 1138 insertions(+), 115 deletions(-) create mode 100644 distributed_game/src/entity.rs create mode 100644 distributed_game/tests/entities/main.rs create mode 100644 distributed_game_macros/tests/compile_fail/replicated_from_missing_field.rs create mode 100644 distributed_game_macros/tests/compile_fail/replicated_from_missing_field.stderr create mode 100644 distributed_game_macros/tests/compile_fail/replicated_from_needs_of.rs create mode 100644 distributed_game_macros/tests/compile_fail/replicated_from_needs_of.stderr create mode 100644 distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.rs create mode 100644 distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.stderr diff --git a/distributed_game/README.md b/distributed_game/README.md index c3e8412d7..10141d06e 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -64,7 +64,8 @@ The words used throughout: | **GameServer** | (`net` feature) A WebSocket server hosting many zones, creating them on demand, routing players, and serving `/metrics` and admin routes. | | **Tick** | One fixed simulation step (30 Hz by default). Separate from the **patch** rate (20 Hz by default), how often clients get updates. | | **Player / Observer** | A *player* controls something and sends inputs. An *observer* only watches: a spectator, a leaderboard page, or a client looking into the zone next door. | -| **Entity** | Something clients can see. The zone describes its entities each patch; the engine works out what changed for each client. | +| **Entity** | A thing inside a zone with its own `EntityId`: a player, a mob, a pillar. It holds server state, some of which no client may see. | +| **View** | What clients see of an entity: a `#[derive(Replicated)]` struct *projected* from it each patch (converted, derived, or left out). The engine works out what changed for each client. | | **Schema** | The kinds of entity and their fields (with quantization), usually from `#[derive(Replicated)]` views. Sent to clients on connect, so a generic client needs no generated code; a generated one adds types. | | **Fact** | An outcome worth keeping ("Alice killed the elite"). The zone emits it (typically the cell command's own input) and moves on; a background worker commits it to storage, where the command decides the reward, and reports back later. | | **Cell** | Durable, event-sourced state in `distributed`, for example one player's profile. Facts usually end up as commands on a cell. | @@ -194,7 +195,9 @@ impl Zone for Room { } } - // Describe the world; the engine diffs it per client. + // Describe the world; the engine diffs it per client. (Zones with + // entity collections register them in `entities()` instead and + // project views from them: see "What each client sees".) fn replicate(&self, out: &mut WorldView) { out.add(1, &DotView { x: self.x }); } @@ -325,9 +328,74 @@ that to drop inputs the server has already applied when it reconciles. ### What each client sees -Each patch, `replicate` describes every entity once: `out.add(id, &view)` -with a `#[derive(Replicated)]` view (or a hand-built `EntityState`). The -view's struct is the kind: `#[replicated(kind = "mob", id = 2, priority = +**Zones, entities, views and facts.** The same split as on the business +side of `distributed`: + +| game runtime | business side | here | +|---|---|---| +| zone | aggregate root | one consistency boundary, changed only on its own tick | +| zone entity (`Entity`, `EntityId`) | entity inside the aggregate | `Mob { id, home, target, hp, .. }`, server-only state included | +| view (`Replicated` + `Project`) | read model / projection | `MobView { x, y, hp% , tele }`, recomputed each patch | +| fact | domain event | `crawler.record_kill`, committed by a cell | + +Views are projections, not copies: a view converts (`hp` as a percent), +derives (a telegraph ring from a windup timer), flattens (a nested move +state), clamps, or returns `None` to leave the entity out (a dead mob), +and server-only fields (`target`, `home`, `principal`) simply have no +view field. Declare the entity type with `of` and mark straight copies +`#[from]` (same name) or `#[from(path.to.field)]`; the derive generates +`MobView::project(&mob, MobViewRest { .. })` with only the hand-filled +fields left in `MobViewRest` (a missing or mistyped `#[from]` field is a +compile error at the attribute), and when every field is `#[from]` it +also implements `Project` for you: + +```rust,ignore +#[derive(Replicated)] +#[replicated(kind = "mob", id = 2, of = Mob)] +pub struct MobView { + #[quant(min = -40.0, max = 40.0, bits = 14)] #[from(body.x)] pub x: f32, + #[quant(min = -40.0, max = 40.0, bits = 14)] #[from(body.y)] pub y: f32, + pub hp: u8, // percent: filled by hand +} + +impl Entity for Mob { + fn entity_id(&self) -> EntityId { self.id } +} + +impl Project for Mob { + fn view(&self) -> Option { + (self.hp > 0).then(|| MobView::project(self, MobViewRest { + hp: (self.hp * 100 / self.max_hp) as u8, + })) + } +} +``` + +The zone registers its collections once (`Zone::entities`, like +`schema()`); each patch the host projects every entity in registration +order (`Vec`, slices, `VecDeque`, `BTreeMap` values, `Option`, or one +entity such as the zone itself for a HUD view) and then calls +`Zone::replicate` for anything else: + +```rust,ignore +fn entities() -> Entities { + Entities::new() + .each(|z: &Self| &z.players) + .each(|z: &Self| &z.mobs) + .one(|z: &Self| z) // impl Project for the zone +} +``` + +`replicate` now defaults to nothing and stays for unusual zones: there, +`out.project::(&mob)` pushes one entity's view, and +`out.add(id, &view)` any `#[derive(Replicated)]` view (with or without +`of`; or push a hand-built `EntityState`). `ZoneHost::world_view()` +returns the patch's projected world (tests, tooling). Mob Pit +(`examples/game/server/src/mobpit/zone.rs`) has no `replicate` at all, +and a golden test there checks the projected world and the Delta bytes +against its former hand-written loop. + +**Kinds.** The view's struct is the kind: `#[replicated(kind = "mob", id = 2, priority = 2.0, global)]`, one field per replicated field in wire order (`bool`, `u8`, `u16`, `u32`, `i32`, `f32`, `String` as a short string, `Vec` as bytes), `#[quant(min, max, bits)]` / `#[quant(centered = R, bits)]` (zero @@ -845,7 +913,7 @@ errors for unsupported types and attributes. | Derive | On | Container attribute | Field attributes | Implements | |---|---|---|---|---| -| `Replicated` | struct | `#[replicated(kind = "name", id = N, priority = P, global, x = f, y = f)]` | `quant`, `angle`, `field_type`, `priority`, `correct` | `Replicated` (`kind_def`, `field_values`, `position`) | +| `Replicated` | struct | `#[replicated(kind = "name", id = N, priority = P, global, x = f, y = f, of = Entity)]` | `quant`, `angle`, `field_type`, `priority`, `correct`, `from` / `from(a.b)` (with `of`) | `Replicated` (`kind_def`, `field_values`, `position`); with `of`: `View::project` + `ViewRest` (or, all `#[from]`, `impl Project for Entity`) | | `GameInput` | struct | – | `range`, `buttons`, `quant`, `angle`, `field_type` | `GameInput` (`from_payload`, `to_payload`, `sanitize`) + button consts | | `GameEvent` | enum / struct | `#[game_event(id = N, name = "..")]` per variant / on the struct | `quant`, `angle`, `field_type`, `long` | `GameEvent` (`decode_event`, `to_payload`) + id consts | | `GameRecord` | struct | – | `quant`, `angle`, `field_type`, `long` | `Wire` (nested payload record) | diff --git a/distributed_game/src/authoring.rs b/distributed_game/src/authoring.rs index 95e89b0ac..ebbb6ad69 100644 --- a/distributed_game/src/authoring.rs +++ b/distributed_game/src/authoring.rs @@ -84,8 +84,8 @@ impl Pushed<'_> { impl WorldView { /// Push entity `id` described by a `Replicated` view. - pub fn add(&mut self, id: u32, view: &R) -> Pushed<'_> { - self.push(view.entity(id)); + pub fn add(&mut self, id: impl Into, view: &R) -> Pushed<'_> { + self.push(view.entity(id.into().get())); Pushed(self.last_mut().expect("just pushed")) } } diff --git a/distributed_game/src/entity.rs b/distributed_game/src/entity.rs new file mode 100644 index 000000000..fbcd7c3a8 --- /dev/null +++ b/distributed_game/src/entity.rs @@ -0,0 +1,245 @@ +//! Zone entities and the views projected from them. +//! +//! A zone is the **aggregate root**: one authoritative boundary that owns +//! its state and changes it only on its own tick. The things inside it +//! (players, mobs, projectiles) are **entities**: server state with an +//! [`EntityId`], including data no client may see (targets, spawn points, +//! credits, principals). What clients receive are **views**, the zone's +//! **read models**: `#[derive(Replicated)]` structs projected from an +//! entity each patch. A view is a projection, not a copy: it may convert +//! (HP to a percent), derive (a telegraph from a timer), flatten nested +//! state, or omit the entity entirely (a dead mob). Durable outcomes leave +//! the zone as **facts**, its **domain events**, committed by cells. +//! +//! | game runtime | business side | +//! |---|---| +//! | zone | aggregate root | +//! | zone entity ([`Entity`]) | entity inside the aggregate | +//! | view ([`Replicated`], [`Project`]) | read model / projection | +//! | fact ([`Fact`](crate::Fact)) | domain event | +//! +//! **Declaring a projection.** A view names its entity type with +//! `#[replicated(.., of = Mob)]` and marks the fields that are straight +//! copies `#[from]` (same name) or `#[from(state.x)]` (a field path on the +//! entity). The derive then generates `MobView::project(&mob, MobViewRest +//! { .. })` where `MobViewRest` holds only the hand-filled fields, and the +//! zone writes the projection once: +//! +//! ```ignore +//! impl Project for Mob { +//! fn view(&self) -> Option { +//! (self.state != MobState::Dead).then(|| { +//! MobView::project(self, MobViewRest { hp: self.hp_percent(), tele: self.tele() }) +//! }) +//! } +//! } +//! ``` +//! +//! A view whose fields are all `#[from]` gets the `Project` impl generated. +//! A view without `of` is unchanged: build it by hand or implement +//! `Project` for any type. +//! +//! **Registering collections.** [`Zone::entities`](crate::Zone::entities) +//! returns the zone's entity collections once, at creation; every patch +//! the host calls [`Project::view`] on each entity (`None` omits it) and +//! then [`Zone::replicate`](crate::Zone::replicate) for anything else: +//! +//! ```ignore +//! fn entities() -> Entities { +//! Entities::new() +//! .each(|z: &Self| &z.players) +//! .each(|z: &Self| &z.mobs) +//! .one(|z: &Self| z) // the zone itself projects its HUD view +//! } +//! ``` +//! +//! Collections are visited in registration order and each in its own order +//! (`Vec`, slices, `VecDeque`, `BTreeMap` values, `Option`): the projected +//! world is deterministic, which the replicated state hash relies on. + +use std::collections::{BTreeMap, VecDeque}; +use std::fmt; + +use crate::authoring::{Pushed, Replicated}; +use crate::replication::WorldView; + +/// Id of a replicated zone entity (unique within the zone). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct EntityId(pub u32); + +impl EntityId { + /// The raw id. + pub const fn get(self) -> u32 { + self.0 + } +} + +impl From for EntityId { + fn from(id: u32) -> Self { + Self(id) + } +} + +impl From for u32 { + fn from(id: EntityId) -> Self { + id.0 + } +} + +impl fmt::Display for EntityId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// A zone entity: anything the zone replicates under its own id. +pub trait Entity { + /// Its id. + fn entity_id(&self) -> EntityId; +} + +/// Projection of a zone entity (or the zone itself) into view `V`. +pub trait Project { + /// The view this patch; `None` leaves the entity out (clients that + /// know it get a remove). + fn view(&self) -> Option; +} + +/// Something holding entities, visited in a deterministic order. +pub trait EntityCollection { + /// Entity type. + type Item; + /// Visit every entity. + fn visit(&self, f: &mut dyn FnMut(&Self::Item)); +} + +impl EntityCollection for [E] { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for Vec { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for [E; N] { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for VecDeque { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for BTreeMap { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.values().for_each(f); + } +} + +impl EntityCollection for Option { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +type Projection = Box; + +/// A zone's registered entity collections ([`Zone::entities`](crate::Zone::entities)). +pub struct Entities { + projections: Vec>, +} + +impl Default for Entities { + fn default() -> Self { + Self { + projections: Vec::new(), + } + } +} + +impl fmt::Debug for Entities { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Entities") + .field("collections", &self.projections.len()) + .finish() + } +} + +impl Entities { + /// No collections. + pub fn new() -> Self { + Self::default() + } + + /// Project every entity of a collection into view `V` (inferred when + /// the entity type projects into one view; otherwise + /// `.each::(..)`). + pub fn each(mut self, collection: fn(&Z) -> &C) -> Self + where + V: Replicated, + C: EntityCollection + ?Sized + 'static, + C::Item: Entity + Project, + { + self.projections.push(Box::new(move |zone, out| { + collection(zone).visit(&mut |entity| { + out.project::(entity); + }); + })); + self + } + + /// Project a single entity (the zone itself, `|z| z`, for a HUD view). + pub fn one(mut self, entity: fn(&Z) -> &E) -> Self + where + V: Replicated, + E: Entity + Project + ?Sized + 'static, + { + self.projections.push(Box::new(move |zone, out| { + out.project::(entity(zone)); + })); + self + } + + /// Collections registered. + pub fn len(&self) -> usize { + self.projections.len() + } + + /// True when nothing is registered. + pub fn is_empty(&self) -> bool { + self.projections.is_empty() + } + + /// Push every registered entity's view into `out`. + pub fn project(&self, zone: &Z, out: &mut WorldView) { + for projection in &self.projections { + projection(zone, out); + } + } +} + +impl WorldView { + /// Push `entity`'s view `V` under its id; `None` when the projection + /// omits it. For hand-written [`Zone::replicate`](crate::Zone::replicate). + pub fn project(&mut self, entity: &E) -> Option> + where + V: Replicated, + E: Entity + Project + ?Sized, + { + let view = entity.view()?; + Some(self.add(entity.entity_id(), &view)) + } +} diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index a9ad850a7..a2cb7b046 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -791,6 +791,7 @@ pub struct ZoneHost { zone: Z, core: Core, schema: EntitySchema, + entities: crate::Entities, hot: FixedTimestep, idle: Option, last_advance: Duration, @@ -865,6 +866,7 @@ impl ZoneHost { zone, core, schema: Z::schema(), + entities: Z::entities(), hot, idle, last_advance: now, @@ -990,11 +992,10 @@ impl ZoneHost { /// [`Zone::state_hash`] (or, when the zone has none, its replicated /// [`WorldView`]). pub fn state_hash(&self) -> u64 { - let zone = self.zone.state_hash().unwrap_or_else(|| { - let mut view = WorldView::default(); - self.zone.replicate(&mut view); - crate::replay::hash_world(&view) - }); + let zone = self + .zone + .state_hash() + .unwrap_or_else(|| crate::replay::hash_world(&self.world_view())); let mut h = crate::rng::Fnv64::new(); h.write_u64(self.core.tick) .write_u64(self.core.sim_time.as_nanos() as u64) @@ -1873,9 +1874,17 @@ impl ZoneHost { } } - fn patch(&mut self) { + /// This patch's replicated world: every registered entity collection's + /// projected views ([`Zone::entities`]), then [`Zone::replicate`]. + pub fn world_view(&self) -> WorldView { let mut view = WorldView::default(); + self.entities.project(&self.zone, &mut view); self.zone.replicate(&mut view); + view + } + + fn patch(&mut self) { + let view = self.world_view(); let core = &mut self.core; self.pending.entities = Some(view.entities().len()); let world = EncodedWorld::build(&self.schema, view, &core.config.interest); diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index ea85e92e2..03f0e958a 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -59,6 +59,7 @@ pub mod celld; pub mod claims; pub mod clock; pub mod codec; +pub mod entity; pub mod facts; pub mod handoff; pub mod host; @@ -93,6 +94,7 @@ pub use codec::{Wire, WireType}; pub use distributed_game_macros::{ game_pure, predict, FactCommand, GameEvent, GameInput, GameRecord, Replicated, TicketExtension, }; +pub use entity::{Entities, Entity, EntityCollection, EntityId, Project}; #[cfg(feature = "runtime")] pub use facts::spawn_fact_worker; pub use facts::{ diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 2b01125dd..a6e2710bd 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -300,8 +300,18 @@ pub trait Zone: Sized + Send + 'static { false } - /// Describe replicated entities for this patch. - fn replicate(&self, out: &mut WorldView); + /// The zone's entity collections, registered once at creation + /// ([`entity`](crate::entity)): every patch the host pushes each + /// entity's [`Project`](crate::Project)ed view (in registration order), + /// then calls [`replicate`](Self::replicate). Default: none. + fn entities() -> crate::Entities { + crate::Entities::new() + } + + /// Describe replicated entities for this patch that are not projected + /// from a registered collection ([`entities`](Self::entities)). + /// Default: nothing. + fn replicate(&self, _out: &mut WorldView) {} /// Record hit volumes for lag compensation after each tick /// ([`Ctx::rewind`](crate::Ctx::rewind)). Default: nothing recorded. diff --git a/distributed_game/tests/entities/main.rs b/distributed_game/tests/entities/main.rs new file mode 100644 index 000000000..3b9c1ccf3 --- /dev/null +++ b/distributed_game/tests/entities/main.rs @@ -0,0 +1,247 @@ +//! Zone entities projected into views (`distributed_game::entity`): +//! `#[replicated(.., of = T)]` + `#[from]`, `Project`, and the host +//! projecting registered collections before `Zone::replicate`. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use distributed_game::{ + schema, Ctx, Entities, Entity, EntityId, EntitySchema, FieldValue, HostConfig, ManualClock, + PlayerId, Project, Replicated, WorldView, Zone, ZoneHost, +}; + +#[derive(Debug, Clone, Copy, Default)] +struct Body { + x: f32, + y: f32, +} + +/// Server state: `home` and `hp` never leave the zone as such. +struct Mob { + id: EntityId, + body: Body, + home: (f32, f32), + hp: i32, + max_hp: i32, +} + +impl Entity for Mob { + fn entity_id(&self) -> EntityId { + self.id + } +} + +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "mob", id = 1, of = Mob)] +struct MobView { + #[from(body.x)] + x: f32, + #[from(body.y)] + y: f32, + /// HP percent. + hp: u8, +} + +impl Project for Mob { + fn view(&self) -> Option { + (self.hp > 0).then(|| { + MobView::project( + self, + MobViewRest { + hp: (self.hp * 100 / self.max_hp) as u8, + }, + ) + }) + } +} + +/// Every field `#[from]`: the `Project` impl is generated. +#[derive(Debug, Clone, Copy)] +struct Rock { + id: EntityId, + x: f32, + y: f32, + r: f32, +} + +impl Entity for Rock { + fn entity_id(&self) -> EntityId { + self.id + } +} + +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "rock", id = 2, global, of = Rock)] +struct RockView { + #[from] + x: f32, + #[from] + y: f32, + #[from] + r: f32, +} + +/// The zone projects its own HUD numbers. +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "hud", id = 3, global, of = Field)] +struct HudView { + alive: u8, +} + +/// A view without `of`, pushed by hand in `replicate`. +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "marker", id = 4)] +struct MarkerView { + x: f32, + y: f32, +} + +struct Field { + mobs: Vec, + rocks: BTreeMap, + boss: Option, +} + +impl Entity for Field { + fn entity_id(&self) -> EntityId { + EntityId(900) + } +} + +impl Project for Field { + fn view(&self) -> Option { + let alive = self.mobs.iter().filter(|m| m.hp > 0).count(); + Some(HudView::project(self, HudViewRest { alive: alive as u8 })) + } +} + +fn mob(id: u32, x: f32, hp: i32) -> Mob { + Mob { + id: EntityId(id), + body: Body { x, y: -x }, + home: (x, x), + hp, + max_hp: 40, + } +} + +impl Zone for Field { + type Config = (); + type Input = (); + type Timer = (); + type Message = distributed_game::NoMessages; + + fn schema() -> EntitySchema { + schema![MobView, RockView, HudView, MarkerView] + } + + fn entities() -> Entities { + Entities::new() + .each(|z: &Self| &z.mobs) + .each(|z: &Self| &z.rocks) + .each(|z: &Self| &z.boss) + .one(|z: &Self| z) + } + + fn on_create(_ctx: &mut Ctx<'_, Self>, _config: ()) -> Self { + Self { + mobs: vec![mob(1, 1.0, 40), mob(2, 2.0, 0), mob(3, 3.0, 10)], + rocks: [(7u8, 5.0f32), (3, 4.0)] + .into_iter() + .map(|(k, x)| { + ( + k, + Rock { + id: EntityId(100 + u32::from(k)), + x, + y: 0.0, + r: 0.5, + }, + ) + }) + .collect(), + boss: Some(mob(50, 9.0, 20)), + } + } + + fn decode_input(&self, _player: PlayerId, _payload: &[u8]) -> Option<()> { + None + } + + fn on_tick(&mut self, _ctx: &mut Ctx<'_, Self>) {} + + fn replicate(&self, out: &mut WorldView) { + out.add(EntityId(77), &MarkerView { x: 0.5, y: 0.5 }); + } +} + +#[test] +fn registered_collections_are_projected_in_order_then_replicate_runs() { + let host = ZoneHost::::new( + "field", + HostConfig::default(), + Arc::new(ManualClock::new()), + (), + ); + let view = host.world_view(); + let ids: Vec<(u32, u16)> = view.entities().iter().map(|e| (e.id, e.kind)).collect(); + assert_eq!( + ids, + [ + (1, 1), // mobs in Vec order; mob 2 is dead: no view + (3, 1), // + (103, 2), // rocks in BTreeMap key order (3, 7) + (107, 2), // + (50, 1), // the Option + (900, 3), // the zone itself + (77, 4), // replicate() last + ] + ); + let mob3 = &view.entities()[1]; + assert_eq!( + mob3.fields, + [FieldValue::F(3.0), FieldValue::F(-3.0), FieldValue::U(25)], + "#[from(body.x)] copied, hp derived" + ); + assert_eq!((mob3.x, mob3.y), (3.0, -3.0)); + assert!(view.entities()[2].global, "kind flags carry over"); + assert_eq!(view.entities()[5].fields, [FieldValue::U(2)]); + // Unreplicated server state stays put. + assert_eq!(host.zone().mobs[0].home, (1.0, 1.0)); +} + +#[test] +fn hand_written_replicate_can_project_entities_too() { + let mut out = WorldView::default(); + let dead = mob(9, 0.0, 0); + assert!(out.project::(&dead).is_none(), "None omits"); + out.project::(&mob(8, 1.0, 40)) + .expect("alive") + .priority(3.0); + let rock = Rock { + id: EntityId(5), + x: 1.0, + y: 2.0, + r: 0.25, + }; + out.project::(&rock); + assert_eq!(out.entities().len(), 2); + assert_eq!(out.entities()[0].priority, 3.0); + assert_eq!( + RockView::project(&rock), + RockView { + x: 1.0, + y: 2.0, + r: 0.25 + } + ); + assert_eq!(rock.view(), Some(RockView::project(&rock))); +} + +#[test] +fn entity_ids_convert_and_display_as_the_raw_id() { + let id = EntityId::from(1002); + assert_eq!(u32::from(id), 1002); + assert_eq!(id.get(), 1002); + assert_eq!(id.to_string(), "1002"); +} diff --git a/distributed_game_macros/src/lib.rs b/distributed_game_macros/src/lib.rs index 55c41b630..9612813d2 100644 --- a/distributed_game_macros/src/lib.rs +++ b/distributed_game_macros/src/lib.rs @@ -21,7 +21,8 @@ //! `FieldType` expression (e.g. a shared `const POS: FieldType`). //! - `#[long]` on a `String` payload field: varuint length instead of the //! 255-byte short string. -//! - Replicated only: `#[priority(P)]`, `#[correct(ease = MS | ease | snap | replace)]`. +//! - Replicated only: `#[priority(P)]`, `#[correct(ease = MS | ease | snap | replace)]`, +//! `#[from]` / `#[from(path.to.field)]` (with `#[replicated(.., of = Entity)]`). //! - GameInput only: `#[range(min = A, max = B)]` (clamp), `#[buttons(a, b, ..)]` //! (bit flags, unknown bits cleared; also generates `Type::A`, `Type::B` consts). @@ -43,14 +44,21 @@ fn finish(result: syn::Result) -> TokenStream { } /// One replicated entity kind: `#[replicated(kind = "mob", id = 2, -/// priority = 2.0, global, x = field, y = field)]` on a struct with named -/// fields (Rust type → wire type: `bool`, `u8`, `u16`, `u32`, `i32`, `f32`, -/// `String` → short_str, `Vec` → bytes). The interest position is the -/// `x`/`y` fields unless overridden; a kind without a position must be -/// `global`. +/// priority = 2.0, global, x = field, y = field, of = Entity)]` on a struct +/// with named fields (Rust type → wire type: `bool`, `u8`, `u16`, `u32`, +/// `i32`, `f32`, `String` → short_str, `Vec` → bytes). The interest +/// position is the `x`/`y` fields unless overridden; a kind without a +/// position must be `global`. +/// +/// `of = Entity` makes the struct a view projected from a zone entity +/// (`distributed_game::entity`): fields marked `#[from]` (same name) or +/// `#[from(path.to.field)]` are copied from the entity, and the derive +/// generates `View::project(&entity, ViewRest { .. })`, `ViewRest` holding +/// the remaining hand-filled fields. When every field is `#[from]` it +/// generates `View::project(&entity)` and `impl Project for Entity`. #[proc_macro_derive( Replicated, - attributes(replicated, quant, angle, field_type, priority, correct) + attributes(replicated, quant, angle, field_type, priority, correct, from) )] pub fn derive_replicated(input: TokenStream) -> TokenStream { finish(replicated::expand(parse_macro_input!(input as DeriveInput))) diff --git a/distributed_game_macros/src/replicated.rs b/distributed_game_macros/src/replicated.rs index 6bbc286c4..3d7d724cb 100644 --- a/distributed_game_macros/src/replicated.rs +++ b/distributed_game_macros/src/replicated.rs @@ -2,8 +2,9 @@ use proc_macro2::{Span, TokenStream}; use quote::{quote, quote_spanned}; +use syn::punctuated::Punctuated; use syn::spanned::Spanned; -use syn::{DeriveInput, Expr, Ident, LitInt, LitStr}; +use syn::{DeriveInput, Expr, Field, Ident, LitInt, LitStr, Member, Token, Type}; use crate::shared::{ check_name, field_name, find_attr, named_fields, no_generics, parse_field_attrs, reject, @@ -19,6 +20,7 @@ struct KindAttrs { global: bool, x: Option, y: Option, + of: Option, } fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { @@ -28,8 +30,8 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { "#[derive(Replicated)] needs #[replicated(kind = \"name\", id = N)]", )); }; - let (mut kind, mut id, mut priority, mut global, mut x, mut y) = - (None, None, None, false, None, None); + let (mut kind, mut id, mut priority, mut global, mut x, mut y, mut of) = + (None, None, None, false, None, None, None); attr.parse_nested_meta(|meta| { let key = meta .path @@ -46,9 +48,10 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { "global" => global = true, "x" => x = Some(meta.value()?.parse::()?), "y" => y = Some(meta.value()?.parse::()?), + "of" => of = Some(meta.value()?.parse::()?), _ => { return Err(meta.error( - "expected `kind`, `id`, `priority`, `global`, `x` or `y` in #[replicated(..)]", + "expected `kind`, `id`, `priority`, `global`, `x`, `y` or `of` in #[replicated(..)]", )) } } @@ -65,6 +68,93 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { global, x, y, + of, + }) +} + +/// `#[from]` (same field name) or `#[from(a.b)]` (a field path on the +/// entity): the field path to copy, spanned at the attribute. +fn from_path(field: &Field) -> syn::Result, Span)>> { + let Some(attr) = find_attr(&field.attrs, "from") else { + return Ok(None); + }; + let span = attr.span(); + if matches!(attr.meta, syn::Meta::Path(_)) { + let ident = field.ident.clone().expect("named"); + return Ok(Some((vec![Member::Named(ident)], span))); + } + let path = attr.parse_args_with(Punctuated::::parse_separated_nonempty)?; + Ok(Some((path.into_iter().collect(), span))) +} + +/// With `of = T`: `View::project(&T, ViewRest)` (or, when every field is +/// `#[from]`, `View::project(&T)` plus `impl Project for T`). +fn expand_of( + input: &DeriveInput, + of: &Type, + fields: &[&Field], + rt: &TokenStream, +) -> syn::Result { + let ident = &input.ident; + let vis = &input.vis; + // Zone entities are usually private to their module while views are + // public: a crate-visible constructor never leaks the entity type. + let fn_vis = quote!(pub(crate)); + let rest_ident = Ident::new(&format!("{ident}Rest"), ident.span()); + let mut inits = Vec::new(); + let mut rest_fields = Vec::new(); + for field in fields { + let name = field.ident.as_ref().expect("named"); + match from_path(field)? { + Some((path, span)) => { + // A missing path is "no field", a different type + // "mismatched types", both at the `#[from]`. + inits.push(quote_spanned! {span=> + #name: (src #(. #path)*).clone() + }); + } + None => { + let docs = field.attrs.iter().filter(|a| a.path().is_ident("doc")); + let (fvis, ty) = (&field.vis, &field.ty); + rest_fields.push(quote! { #(#docs)* #fvis #name: #ty }); + inits.push(quote! { #name: rest.#name }); + } + } + } + if rest_fields.is_empty() { + return Ok(quote! { + impl #ident { + /// Project the entity: every field is `#[from]`. + #[allow(clippy::clone_on_copy)] + #fn_vis fn project(src: &#of) -> Self { + Self { #(#inits),* } + } + } + + impl #rt::Project<#ident> for #of { + fn view(&self) -> ::core::option::Option<#ident> { + ::core::option::Option::Some(#ident::project(self)) + } + } + }); + } + let doc = + format!("The hand-filled (non-`#[from]`) fields of [`{ident}`], for [`{ident}::project`]."); + Ok(quote! { + #[doc = #doc] + #[derive(Debug, Clone, PartialEq)] + #vis struct #rest_ident { + #(#rest_fields),* + } + + impl #ident { + /// Project the entity: `#[from]` fields are copied from `src`, + /// the others taken from `rest`. + #[allow(clippy::clone_on_copy)] + #fn_vis fn project(src: &#of, rest: #rest_ident) -> Self { + Self { #(#inits),* } + } + } }) } @@ -164,6 +254,22 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { let kind = &attrs.kind; let id = attrs.id; let global = attrs.global; + let projection = match &attrs.of { + Some(of) => expand_of(&input, of, &fields, &rt)?, + None => { + if let Some(field) = fields + .iter() + .find(|f| find_attr(&f.attrs, "from").is_some()) + { + let attr = find_attr(&field.attrs, "from").expect("found"); + return Err(syn::Error::new( + attr.span(), + "#[from] needs the entity type: #[replicated(.., of = Entity)]", + )); + } + TokenStream::new() + } + }; let kind_priority = attrs .priority .as_ref() @@ -190,5 +296,7 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { #p::std::vec![#(#values),*] } } + + #projection }) } diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.rs b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.rs new file mode 100644 index 000000000..3ae3ebf54 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.rs @@ -0,0 +1,21 @@ +use distributed_game::Replicated; + +struct Mob { + x: f32, + y: f32, +} + +#[derive(Replicated)] +#[replicated(kind = "mob", id = 2, of = Mob)] +struct MobView { + #[from] + x: f32, + #[from] + y: f32, + #[from] + tier: u8, +} + +fn main() { + let _ = Mob { x: 0.0, y: 0.0 }.x; +} diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.stderr b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.stderr new file mode 100644 index 000000000..ddf110407 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.stderr @@ -0,0 +1,7 @@ +error[E0609]: no field `tier` on type `&Mob` + --> tests/compile_fail/replicated_from_missing_field.rs:16:5 + | +16 | tier: u8, + | ^^^^ unknown field + | + = note: available fields are: `x`, `y` diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.rs b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.rs new file mode 100644 index 000000000..17c670358 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.rs @@ -0,0 +1,11 @@ +use distributed_game::Replicated; + +#[derive(Replicated)] +#[replicated(kind = "mob", id = 2)] +struct MobView { + #[from] + x: f32, + y: f32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.stderr b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.stderr new file mode 100644 index 000000000..b17c524dd --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.stderr @@ -0,0 +1,5 @@ +error: #[from] needs the entity type: #[replicated(.., of = Entity)] + --> tests/compile_fail/replicated_from_needs_of.rs:6:5 + | +6 | #[from] + | ^ diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.rs b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.rs new file mode 100644 index 000000000..9834d968d --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.rs @@ -0,0 +1,27 @@ +use distributed_game::Replicated; + +struct State { + x: f32, + y: f32, +} + +struct Player { + state: State, + hp: i32, +} + +#[derive(Replicated)] +#[replicated(kind = "player", id = 1, of = Player)] +struct PlayerView { + #[from(state.x)] + x: f32, + #[from(state.y)] + y: f32, + #[from] + hp: u8, +} + +fn main() { + let p = Player { state: State { x: 0.0, y: 0.0 }, hp: 1 }; + let _ = (p.state.x, p.state.y, p.hp); +} diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.stderr b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.stderr new file mode 100644 index 000000000..c50c90e65 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.stderr @@ -0,0 +1,5 @@ +error[E0308]: mismatched types + --> tests/compile_fail/replicated_from_wrong_type.rs:20:5 + | +20 | #[from] + | ^ expected `u8`, found `i32` diff --git a/examples/game/server/src/arena/mod.rs b/examples/game/server/src/arena/mod.rs index 409ce1710..b1507883e 100644 --- a/examples/game/server/src/arena/mod.rs +++ b/examples/game/server/src/arena/mod.rs @@ -32,8 +32,8 @@ use std::collections::BTreeMap; use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; use distributed_game::{ - schema, Ctx, EntitySchema, GameEvent, GameInput, GameSurface, InputCredits, LeaveReason, - PlayerId, Replicated, View, WorldView, Zone, + schema, Ctx, Entity, EntityId, EntitySchema, GameEvent, GameInput, GameSurface, InputCredits, + LeaveReason, PlayerId, Replicated, View, WorldView, Zone, }; use crate::common::{Demo, Guests}; @@ -102,15 +102,53 @@ pub struct DroneView { pub yaw: f32, } -/// A static pillar (visible everywhere). +/// A static pillar of the shared floor ([`movement::PILLARS`]). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Pillar { + /// Entity id (100 + index). + pub id: EntityId, + /// Centre x. + pub x: f32, + /// Centre y. + pub y: f32, + /// Radius. + pub r: f32, +} + +impl Pillar { + /// Every pillar of the floor. + pub fn all() -> Vec { + movement::PILLARS + .iter() + .enumerate() + .map(|(i, &(x, y, r))| Pillar { + id: EntityId(100 + i as u32), + x, + y, + r, + }) + .collect() + } +} + +impl Entity for Pillar { + fn entity_id(&self) -> EntityId { + self.id + } +} + +/// A static pillar (visible everywhere), a straight projection of [`Pillar`]. #[derive(Debug, Clone, PartialEq, Replicated)] -#[replicated(kind = "pillar", id = 3, global)] +#[replicated(kind = "pillar", id = 3, global, of = Pillar)] pub struct PillarView { /// Centre x. + #[from] pub x: f32, /// Centre y. + #[from] pub y: f32, /// Radius. + #[from] pub r: f32, } @@ -316,15 +354,8 @@ impl Zone for Arena { }; out.add(d.id, &view).priority(0.5); } - for (i, (x, y, r)) in movement::PILLARS.iter().enumerate() { - out.add( - 100 + i as u32, - &PillarView { - x: *x, - y: *y, - r: *r, - }, - ); + for pillar in Pillar::all() { + out.project::(&pillar); } } } diff --git a/examples/game/server/src/mobpit/zone.rs b/examples/game/server/src/mobpit/zone.rs index 42f4feb52..619de90e5 100644 --- a/examples/game/server/src/mobpit/zone.rs +++ b/examples/game/server/src/mobpit/zone.rs @@ -18,9 +18,15 @@ //! leash or target dead → `return` home → `idle`. Dead mobs respawn after 4 s //! with a new generation. //! -//! Kinds are `#[derive(Replicated)]` views, events the typed [`PitEvent`], -//! input the `#[derive(GameInput)]` [`PitInput`]; the browser client is -//! generated from [`game_surface`]. +//! The zone is the aggregate; players, mobs and pillars are its entities +//! (each with an [`EntityId`]), and the kinds are `#[derive(Replicated)]` +//! views *projected* from them (`of = Player`, `#[from]` fields copied, the +//! rest derived in `impl Project<..>`): a mob's HP becomes a percent, its +//! windup timer the `tele` ring, a dead mob no view at all. The zone +//! registers its collections once ([`Zone::entities`]) and projects its own +//! HUD numbers ([`PitView`]); there is no hand-written `replicate()`. +//! Events are the typed [`PitEvent`], input the `#[derive(GameInput)]` +//! [`PitInput`]; the browser client is generated from [`game_surface`]. //! //! A kill emits a typed fact **without awaiting storage**: the cell //! command's own input, [`RecordKillInput`] `{mob, tier, name}`, keyed by @@ -38,51 +44,60 @@ use std::sync::Arc; use std::time::Instant; use distributed_game::{ - schema, ConsumePolicy, Ctx, EmitError, EntitySchema, Fact, FactOutcome, FactSettlement, - GameEvent, GameInput, GameRecord, GameSurface, LeaveReason, ObserverId, PlayerId, Replicated, - View, WorldView, Zone, + schema, ConsumePolicy, Ctx, EmitError, Entities, Entity, EntityId, EntitySchema, Fact, + FactOutcome, FactSettlement, GameEvent, GameInput, GameRecord, GameSurface, LeaveReason, + ObserverId, PlayerId, Project, Replicated, View, Zone, }; use super::board::BoardSnapshot; use super::crawler::{level_floor, KillReceipt, RecordKillInput}; use super::world::MobPitWorld; use crate::arena::movement::{self, MoveState, PILLARS, PLAYER_RADIUS}; -use crate::arena::PillarView; +use crate::arena::{Pillar, PillarView}; use crate::common::Guests; -/// Entity id of the stats entity. -pub const PIT_ENTITY: u32 = 900; +/// Entity id of the stats entity (the zone's own HUD view). +pub const PIT_ENTITY: EntityId = EntityId(900); /// A player: the Arena's predicted [`MoveState`] plus server-owned combat -/// state (replaced, never predicted, on the client). +/// state (replaced, never predicted, on the client). Projected from the +/// zone's `Player` entity: the move state is flattened, HP and level are +/// clamped, and its principal, credits and timers stay on the server. #[derive(Debug, Clone, PartialEq, Replicated)] -#[replicated(kind = "player", id = 1, priority = 2.0)] +#[replicated(kind = "player", id = 1, priority = 2.0, of = Player)] pub struct PlayerView { /// Position x. #[field_type(movement::POS)] #[correct(ease = 150)] + #[from(state.x)] pub x: f32, /// Position y. #[field_type(movement::POS)] #[correct(ease = 150)] + #[from(state.y)] pub y: f32, /// Velocity x. #[field_type(movement::VEL)] #[correct(snap)] + #[from(state.vx)] pub vx: f32, /// Velocity y. #[field_type(movement::VEL)] #[correct(snap)] + #[from(state.vy)] pub vy: f32, /// Remaining dash frames. #[correct(snap)] + #[from(state.dash)] pub dash: u8, /// Remaining dash cooldown frames. #[correct(snap)] + #[from(state.cd)] pub cd: u8, /// Display name. #[priority(0.2)] #[correct(replace)] + #[from] pub name: String, /// Hit points. #[correct(replace)] @@ -90,6 +105,7 @@ pub struct PlayerView { /// Facing (rad). #[angle(bits = 8)] #[correct(replace)] + #[from] pub face: f32, /// Swing animation frames left. #[correct(replace)] @@ -102,20 +118,25 @@ pub struct PlayerView { pub dead: bool, } -/// A mob (dead mobs are not replicated). +/// A mob (dead mobs are not replicated); its home, target, generation and +/// wander point stay on the server. #[derive(Debug, Clone, PartialEq, Replicated)] -#[replicated(kind = "mob", id = 2)] +#[replicated(kind = "mob", id = 2, of = Mob)] pub struct MobView { /// Position x. #[quant(min = -movement::HALF, max = movement::HALF, bits = 14)] + #[from] pub x: f32, /// Position y. #[quant(min = -movement::HALF, max = movement::HALF, bits = 14)] + #[from] pub y: f32, /// Facing (rad). #[angle(bits = 8)] + #[from] pub face: f32, /// 1 grunt · 2 brute · 3 elite. + #[from] pub tier: u8, /// HP percent. pub hp: u8, @@ -125,9 +146,9 @@ pub struct MobView { pub tele: u8, } -/// Zone numbers for the HUD (global). +/// Zone numbers for the HUD (global), projected from the zone itself. #[derive(Debug, Clone, PartialEq, Replicated)] -#[replicated(kind = "pit", id = 4, global)] +#[replicated(kind = "pit", id = 4, global, of = MobPit)] pub struct PitView { /// Facts emitted and not settled. pub pending: u16, @@ -342,7 +363,7 @@ enum MobState { } struct Mob { - id: u32, + id: EntityId, tier: u8, home: (f32, f32), x: f32, @@ -358,7 +379,7 @@ struct Mob { } struct Player { - entity: u32, + id: EntityId, principal: String, name: String, state: MoveState, @@ -394,6 +415,7 @@ pub struct MobPit { world: Arc, players: BTreeMap, mobs: Vec, + pillars: Vec, pending: HashMap, /// Facts refused by backpressure, retried each tick (oldest first). backlog: VecDeque<(PlayerId, Fact, f32, f32)>, @@ -487,7 +509,7 @@ impl MobPit { fn emit_kill(&mut self, ctx: &mut Ctx<'_, Self>, killer: PlayerId, mob: usize) { let m = &self.mobs[mob]; - let (x, y, tier, id, generation) = (m.x, m.y, m.tier, m.id, m.generation); + let (x, y, tier, id, generation) = (m.x, m.y, m.tier, m.id.get(), m.generation); let Some(player) = self.players.get(&killer) else { return; }; @@ -507,7 +529,7 @@ impl MobPit { tier, x, y, - killer: player.entity, + killer: player.id.get(), killer_name: player.name.clone(), }); self.backlog.push_back((killer, fact, x, y)); @@ -828,12 +850,20 @@ impl Zone for MobPit { schema![PlayerView, MobView, PillarView, PitView] } + fn entities() -> Entities { + Entities::new() + .each(|z: &Self| &z.players) + .each(|z: &Self| &z.mobs) + .each(|z: &Self| &z.pillars) + .one(|z: &Self| z) + } + fn on_create(_ctx: &mut Ctx<'_, Self>, config: MobPitConfig) -> Self { let mobs = mob_layout() .into_iter() .enumerate() .map(|(i, (tier, home))| Mob { - id: i as u32 + 1, + id: EntityId(i as u32 + 1), tier, home, x: home.0, @@ -853,6 +883,7 @@ impl Zone for MobPit { world: config.world, players: BTreeMap::new(), mobs, + pillars: Pillar::all(), pending: HashMap::new(), backlog: VecDeque::new(), board_version: u64::MAX, @@ -879,11 +910,11 @@ impl Zone for MobPit { .or(ticket_name) .unwrap_or_else(|| principal.clone()); let (x, y) = spawn_point(player); - let entity = 1000 + player.0; + let entity = EntityId(1000 + player.0); self.players.insert( player, Player { - entity, + id: entity, principal: principal.clone(), name, state: MoveState { @@ -905,7 +936,7 @@ impl Zone for MobPit { ctx.send( player, &PitEvent::Hello { - entity, + entity: entity.get(), half: movement::HALF, boot: self.world.boot().to_string(), attack_range: ATTACK_RANGE, @@ -1035,9 +1066,148 @@ impl Zone for MobPit { fn has_pending_work(&self) -> bool { !self.backlog.is_empty() } +} + +impl Entity for Player { + fn entity_id(&self) -> EntityId { + self.id + } +} + +impl Project for Player { + fn view(&self) -> Option { + Some(PlayerView::project( + self, + PlayerViewRest { + hp: self.hp.clamp(0, 255) as u8, + swing: self.swing_anim, + level: self.level.min(u32::from(u16::MAX)) as u16, + dead: self.dead_until.is_some(), + }, + )) + } +} + +impl Entity for Mob { + fn entity_id(&self) -> EntityId { + self.id + } +} + +impl Project for Mob { + fn view(&self) -> Option { + if self.state == MobState::Dead { + return None; + } + let max_hp = tier_stats(self.tier).0; + let tele = if self.state == MobState::Windup { + 255 - (u32::from(self.timer) * 255 / u32::from(WINDUP_TICKS)) + } else { + 0 + }; + Some(MobView::project( + self, + MobViewRest { + hp: (self.hp.max(0) * 100 / max_hp) as u8, + state: self.state as u8, + tele: tele as u8, + }, + )) + } +} + +impl Entity for MobPit { + fn entity_id(&self) -> EntityId { + PIT_ENTITY + } +} + +impl Project for MobPit { + fn view(&self) -> Option { + let clamp16 = |n: usize| n.min(65_535) as u16; + Some(PitView::project( + self, + PitViewRest { + pending: clamp16(self.pending.len()), + slow_ms: self.world.sink_state().latency().as_millis().min(65_535) as u16, + tick_ms: self.tick_ms, + tick_max_ms: self.tick_max_ms, + work_us: self.work_us as u32, + committed: self.committed, + backlog: clamp16(self.backlog.len()), + }, + )) + } +} + +/// Zone numbers for tests and tooling (`GameServer::inspect`). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PitStats { + /// Facts emitted and not yet settled (incl. backpressure backlog). + pub pending: usize, + /// Tick interval EMA (ms). + pub tick_ms: f32, + /// Longest tick interval of the last 30 ticks (ms). + pub tick_max_ms: f32, + /// Settlements received. + pub committed: u32, + /// Living mobs. + pub mobs_alive: usize, + /// Players. + pub players: usize, +} + +impl MobPit { + /// Current numbers. + pub fn stats(&self) -> PitStats { + PitStats { + pending: self.pending.len() + self.backlog.len(), + tick_ms: self.tick_ms, + tick_max_ms: self.tick_max_ms, + committed: self.committed, + mobs_alive: self + .mobs + .iter() + .filter(|m| m.state != MobState::Dead) + .count(), + players: self.players.len(), + } + } +} - fn replicate(&self, out: &mut WorldView) { - for p in self.players.values() { +/// Bot payload: the direction as a move plus attack always held. +pub fn encode_bot_input(dir: [f32; 2]) -> Vec { + let axis = |v: f32| (v.clamp(-1.0, 1.0) * 127.0).round() as i8; + PitInput { + mx: axis(dir[0]), + my: axis(dir[1]), + buttons: PitInput::ATTACK, + } + .to_payload() +} + +/// Golden test for the projected views (gr-p16): the hand-written +/// `replicate()` loop Mob Pit used before its views were projected from +/// registered entity collections, kept verbatim as the reference. A real +/// host runs a fight (deaths, windups, respawns); every tick the world the +/// runtime projects must equal the legacy one, entity for entity, and the +/// Delta frames built from each must be byte-identical. +#[cfg(test)] +mod projection_golden { + use std::sync::Arc; + use std::time::Duration; + + use distributed_game::replication::{ClientReplica, EncodedWorld}; + use distributed_game::{ + HostConfig, InterestConfig, ManualClock, Role, TicketClaims, Viewer, WorldView, ZoneHost, + }; + + use super::*; + use crate::mobpit::{MobPitWorld, WorldOptions}; + + /// The pre-gr-p16 `MobPit::replicate`, unchanged. + fn legacy_replicate(pit: &MobPit, out: &mut WorldView) { + for p in pit.players.values() { let s = p.state; let view = PlayerView { x: s.x, @@ -1053,9 +1223,9 @@ impl Zone for MobPit { level: p.level.min(u32::from(u16::MAX)) as u16, dead: p.dead_until.is_some(), }; - out.add(p.entity, &view); + out.add(u32::from(p.id), &view); } - for m in &self.mobs { + for m in &pit.mobs { if m.state == MobState::Dead { continue; } @@ -1074,7 +1244,7 @@ impl Zone for MobPit { state: m.state as u8, tele: tele as u8, }; - out.add(m.id, &view); + out.add(u32::from(m.id), &view); } for (i, (x, y, r)) in PILLARS.iter().enumerate() { out.add( @@ -1088,60 +1258,109 @@ impl Zone for MobPit { } let clamp16 = |n: usize| n.min(65_535) as u16; let stats = PitView { - pending: clamp16(self.pending.len()), - slow_ms: self.world.sink_state().latency().as_millis().min(65_535) as u16, - tick_ms: self.tick_ms, - tick_max_ms: self.tick_max_ms, - work_us: self.work_us as u32, - committed: self.committed, - backlog: clamp16(self.backlog.len()), + pending: clamp16(pit.pending.len()), + slow_ms: pit.world.sink_state().latency().as_millis().min(65_535) as u16, + tick_ms: pit.tick_ms, + tick_max_ms: pit.tick_max_ms, + work_us: pit.work_us as u32, + committed: pit.committed, + backlog: clamp16(pit.backlog.len()), }; out.add(PIT_ENTITY, &stats); } -} -/// Zone numbers for tests and tooling (`GameServer::inspect`). -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct PitStats { - /// Facts emitted and not yet settled (incl. backpressure backlog). - pub pending: usize, - /// Tick interval EMA (ms). - pub tick_ms: f32, - /// Longest tick interval of the last 30 ticks (ms). - pub tick_max_ms: f32, - /// Settlements received. - pub committed: u32, - /// Living mobs. - pub mobs_alive: usize, - /// Players. - pub players: usize, -} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn projected_world_and_delta_bytes_match_the_legacy_replicate() { + let world = MobPitWorld::open(WorldOptions::temporary()).unwrap(); + let clock = ManualClock::new(); + let mut host = ZoneHost::::new( + "pit-1", + HostConfig { + boot_epoch: 7, + ..HostConfig::default() + }, + Arc::new(clock.clone()), + MobPitConfig { + guests: Guests::default(), + world, + }, + ); + let mut clients = Vec::new(); + for name in ["ana", "bo", "cy"] { + let claims = TicketClaims::new(name, "pit-1", u64::MAX, name, Role::Player); + let joined = host.join(&claims, 0, "").unwrap(); + let Viewer::Player(player) = joined.viewer else { + panic!("player") + }; + clients.push((joined.client, player)); + } + let schema = MobPit::schema(); + let interest = InterestConfig::default(); + let (mut legacy_replica, mut projected_replica) = + (ClientReplica::new(), ClientReplica::new()); + let (mut saw_dead, mut saw_windup, mut saw_hurt_player) = (false, false, false); + for tick in 0..900u32 { + for (i, (client, player)) in clients.iter().enumerate() { + // Walk toward the nearest living mob, swinging all the time. + let me = host.zone().players[player].state; + let target = host + .zone() + .mobs + .iter() + .filter(|m| m.state != MobState::Dead) + .min_by(|a, b| { + dist((me.x, me.y), (a.x, a.y)).total_cmp(&dist((me.x, me.y), (b.x, b.y))) + }) + .map(|m| (m.x, m.y)) + .unwrap_or((0.0, 0.0)); + let (dx, dy) = (target.0 - me.x, target.1 - me.y); + let d = (dx * dx + dy * dy).sqrt().max(1e-3); + let wobble = if (tick / 40 + i as u32).is_multiple_of(3) { + 0.4 + } else { + 1.0 + }; + let payload = PitInput { + mx: (dx / d * 127.0 * wobble) as i8, + my: (dy / d * 127.0 * wobble) as i8, + buttons: PitInput::ATTACK, + } + .to_payload(); + host.push_input(*client, tick + 1, tick, &payload); + } + clock.advance(Duration::from_micros(33_334)); + host.advance(); + host.drain_outbound(); -impl MobPit { - /// Current numbers. - pub fn stats(&self) -> PitStats { - PitStats { - pending: self.pending.len() + self.backlog.len(), - tick_ms: self.tick_ms, - tick_max_ms: self.tick_max_ms, - committed: self.committed, - mobs_alive: self + let mut legacy = WorldView::default(); + legacy_replicate(host.zone(), &mut legacy); + let projected = host.world_view(); + assert_eq!(projected.entities(), legacy.entities(), "tick {tick}"); + + let view = host.zone().players[&clients[0].1].state; + let view = View::new(view.x, view.y, VIEW_RADIUS); + let frame = |replica: &mut ClientReplica, world: WorldView| { + let encoded = EncodedWorld::build(&schema, world, &interest); + replica + .build_delta(&schema, &encoded, &interest, &view, 1200, tick, tick) + .0 + }; + assert_eq!( + frame(&mut projected_replica, projected), + frame(&mut legacy_replica, legacy), + "delta bytes at tick {tick}" + ); + + let pit = host.zone(); + saw_dead |= pit.mobs.iter().any(|m| m.state == MobState::Dead); + saw_windup |= pit .mobs .iter() - .filter(|m| m.state != MobState::Dead) - .count(), - players: self.players.len(), + .any(|m| m.state == MobState::Windup && m.timer > 0); + saw_hurt_player |= pit.players.values().any(|p| p.hp < PLAYER_HP); } + assert!(saw_dead, "a mob died (its view is omitted)"); + assert!(saw_windup, "a mob telegraphed (derived `tele`)"); + assert!(saw_hurt_player, "a player took damage"); } } - -/// Bot payload: the direction as a move plus attack always held. -pub fn encode_bot_input(dir: [f32; 2]) -> Vec { - let axis = |v: f32| (v.clamp(-1.0, 1.0) * 127.0).round() as i8; - PitInput { - mx: axis(dir[0]), - my: axis(dir[1]), - buttons: PitInput::ATTACK, - } - .to_payload() -} From 2eb4f9f06e59eaa99bba1a6763f2373a49eb22e9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 12:24:22 -0500 Subject: [PATCH 15/40] feat(macros): #[command] on aggregate methods A command is now one aggregate method. Inside a #[sourced] impl, `#[command("crawler.record_kill", shard = .., load = .., outcome = ..)]` generates the portable mount (`pub struct RecordKill`), its handler and the registration that portable_command! + a hand-written handler gave. - The `#[principal]` parameter is mandatory and filled from the verified session only; an input field named like it fails to compile (const check over the new `CommandInputType::FIELD_NAMES`, filled by `#[derive(CommandInput)]`). - Input: one input parameter, or `input = T` to map the other params to T's fields by name. load = required | or_create | create; shard = an input field or the principal; outcome = succeeded | eventual (method's Ok value is the payload) or Succeeded

/Eventual

+ payload = f. Optional roles, guard, field, transition, mount. Errors -> Rejected; captured domain events are published with the commit. - The domain_commands witness is used as the transition (also for non-pub command methods). - Receipt helper `Entity::last_recorded()` -> `Recorded { sequence, timestamp }` + `unix_ms()`. - Mob Pit `crawler.record_kill` migrated; its spec now also declares the `crawler.mob_killed` emit set via `domain_commands::RecordKill`. - Golden test: command specs, GraphQL SDL and client manifest equal the hand-written registration. trybuild fixtures for the principal rules. - The handler-fn `#[command(id = ..)]` names the sourced form when given a string literal outside #[sourced]. README leads with the attribute. Refs [[tasks/gr-p17-command-macro]] --- README.md | 84 ++- distributed_macros/src/aggregate_command.rs | 630 ++++++++++++++++++ distributed_macros/src/command.rs | 11 + distributed_macros/src/command_types.rs | 11 + distributed_macros/src/lib.rs | 10 +- distributed_macros/src/sourced.rs | 31 +- .../aggregate_command_missing_principal.rs | 26 + ...aggregate_command_missing_principal.stderr | 5 + .../aggregate_command_outside_sourced.rs | 11 + .../aggregate_command_outside_sourced.stderr | 5 + .../aggregate_command_payload_needs_type.rs | 29 + ...ggregate_command_payload_needs_type.stderr | 5 + ...aggregate_command_principal_input_field.rs | 35 + ...egate_command_principal_input_field.stderr | 5 + examples/game/server/crawler/src/lib.rs | 137 ++-- src/command/mod.rs | 3 +- src/command/types.rs | 28 + src/entity/entity.rs | 31 + src/entity/mod.rs | 2 +- src/lib.rs | 4 +- tests/aggregate_command/main.rs | 472 +++++++++++++ 21 files changed, 1481 insertions(+), 94 deletions(-) create mode 100644 distributed_macros/src/aggregate_command.rs create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_missing_principal.rs create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_missing_principal.stderr create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.rs create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.stderr create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.rs create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.stderr create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.rs create mode 100644 distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.stderr create mode 100644 tests/aggregate_command/main.rs diff --git a/README.md b/README.md index ca528c63c..256e4806b 100644 --- a/README.md +++ b/README.md @@ -373,9 +373,28 @@ perform online shadow swaps, or make several independent projections one transaction. Keep services stopped if a multi-projection maintenance run fails; capture fresh inventories and rerun before resuming them. -Handlers stay thin: most Todo commands are `portable_command!` — shard, -invoke one domain method, commit Eventual. `todo.create` keeps a `handle:` -escape hatch when the body needs extra checks. +Handlers stay thin. The shortest form is the command on the aggregate +method itself: `#[command]` inside the `#[sourced]` impl generates the +handler and the portable mount (see [Command contracts](#command-contracts)): + +```rust,ignore +#[command( + "todo.complete", + shard = todo_id, + load = required, + input = TodoCompleteInput, + outcome = Eventual, + payload = TodoStatusPayload::from_todo, + roles = ["user", "admin"], + field = "todos_complete", +)] +pub fn complete(&mut self, #[principal] owner_id: &str) -> Result<(), TodoError> { .. } +``` + +The Todo playground spells the same registration out with +`portable_command!` — shard, invoke one domain method, commit Eventual. +`todo.create` keeps a `handle:` escape hatch when the body needs extra +checks. ```rust,ignore portable_command! { @@ -1603,7 +1622,64 @@ run_source(service, source, RunOptions::idempotent()).await?; ## Command contracts -Command semantics live in `distributed::command`, independently of the query +**Commands on the aggregate.** Most commands are one aggregate method: +take the actor from the session, load (or create) the aggregate, call the +method, reject on its error, commit with a payload. Write the method and +mark it `#[command]` inside the `#[sourced]` impl; the macro generates the +handler and the portable mount (`pub struct RecordKill`, mounted with +`routes.mount(RecordKill)` or `AggregateCell::mount(RecordKill)`): + +```rust,ignore +#[sourced(entity, aggregate_type = "Crawler", events = "CrawlerEvent", domain_state = CrawlerState)] +impl Crawler { + #[command("crawler.record_kill", shard = principal, load = or_create, outcome = succeeded)] + pub fn record_kill( + &mut self, + #[principal] principal: &str, + input: RecordKillInput, + ) -> Result { + let xp = xp_for_tier(input.tier).ok_or("unknown tier")?; + self.mob_killed(principal.into(), input.name, input.mob, input.tier, xp) + .map_err(|e| e.to_string())?; + // Receipt helper: the recorded event's stream position and time. + let recorded = self.entity.last_recorded().ok_or("nothing recorded")?; + Ok(KillReceipt { sequence: recorded.sequence, at_ms: recorded.unix_ms(), .. }) + } + + #[event("crawler.mob_killed", version = 1, domain)] + fn mob_killed(&mut self, id: String, name: String, mob: String, tier: u8, xp: u32) { .. } +} +``` + +- **Principal.** Exactly one parameter is `#[principal]` (`&str` or + `String`); the generated handler fills it from the verified session + (`x-user-id`, else `Unauthorized`), never from the input. It is + mandatory, and an input whose field (Rust or serialized name) is named + like it does not compile. +- **Input.** The other parameter is the `CommandInput` type; or, with + `input = Type`, the other parameters are that input's fields by name + (`&str` parameters borrow), so a domain method keeps plain arguments. +- **`shard`** is an input field (also exposed as `RecordKill::shard`) or + the principal parameter; **`load`** is `required` (`NotFound` when + missing), `or_create` or `create`. +- **`outcome`** is `succeeded` / `eventual` (the method's `Ok` value is + the payload) or `Succeeded

` / `Eventual

` with `payload = f` + (`fn(&Aggregate) -> P`). The method's error (`Display`) becomes + `Rejected`. Captured domain events are always published with the commit. +- Optional: `roles = ["player"]`, `guard = f`, `field = "graphql_field"`, + `transition = path` (the emitted-event witness; by default the + `domain_commands::*` type `#[sourced]` derives for the method), + `mount = Name` (the struct name, default from the command name's last + segment). + +The generated registration is exactly the hand-written one +(`command_transition::(name)` + `field_name`, +`roles`, `guarded`/`handle`): the command specs, GraphQL SDL and client +manifest are identical (`tests/aggregate_command`). For unusual bodies write +the handler and register it with `portable_command! { .., handle: f }` or +the builder below. + +**Contracts.** Command semantics live in `distributed::command`, independently of the query transport. Use `CommandInput` and `CommandOutput` to describe Serde input/output shapes, and `command::{typed_command, Succeeded, Eventual, Atomic, PreparedCommand}` to declare and prepare commands. The GraphQL adapter derives its surface from diff --git a/distributed_macros/src/aggregate_command.rs b/distributed_macros/src/aggregate_command.rs new file mode 100644 index 000000000..f9bed2edd --- /dev/null +++ b/distributed_macros/src/aggregate_command.rs @@ -0,0 +1,630 @@ +//! `#[command("aggregate.verb", ..)]` on an aggregate method inside a +//! `#[sourced]` impl: the method is the command. `#[sourced]` strips the +//! attribute and emits a portable mount (`pub struct Verb`) whose generated +//! handler binds the principal from the verified session, loads (or +//! creates) the aggregate, calls the method, maps its error to `Rejected`, +//! builds the payload and commits with publication. +//! +//! ```ignore +//! #[command("crawler.record_kill", shard = principal, load = or_create, outcome = succeeded)] +//! pub fn record_kill(&mut self, #[principal] principal: &str, input: RecordKillInput) +//! -> Result { .. } +//! ``` +//! +//! Options: `shard = ` (required unless +//! `load = create`), `load = required | or_create | create`, +//! `outcome = succeeded | eventual | Succeeded

| Eventual

`, +//! `payload = path` (`fn(&Aggregate) -> P`; otherwise the method's `Ok` +//! value is the payload), `roles = ["player", ..]`, `guard = expr`, +//! `field = "graphql_field"`, `input = Type` (map the remaining method +//! parameters to the input's fields by name instead of passing the whole +//! input), `transition = path` (override the domain transition witness), +//! `mount = Ident` (name of the generated struct). + +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, quote_spanned}; +use syn::parse::{Parse, ParseStream}; +use syn::{Expr, FnArg, Ident, ImplItemFn, LitStr, Pat, Path, ReturnType, Token, Type}; + +#[derive(Clone, Copy, PartialEq)] +enum Load { + Required, + OrCreate, + Create, +} + +enum Outcome { + /// `succeeded` / `eventual`: the payload type is the method's `Ok` type. + Keyword { eventual: bool, span: Span }, + /// `Succeeded

` / `Eventual

`. + Type { ty: Box, eventual: bool }, +} + +struct Args { + name: LitStr, + shard: Option, + load: Load, + outcome: Outcome, + payload: Option, + roles: Option>, + guard: Option, + field: Option, + input: Option, + transition: Option, + mount: Option, +} + +impl Parse for Args { + fn parse(input: ParseStream<'_>) -> syn::Result { + let name: LitStr = input.parse().map_err(|_| { + input.error("#[command] on an aggregate method starts with the command name: #[command(\"aggregate.verb\", ..)]") + })?; + validate_name(&name)?; + let mut shard = None; + let mut load = None; + let mut outcome = None; + let mut payload = None; + let mut roles = None; + let mut guard = None; + let mut field = None; + let mut input_ty = None; + let mut transition = None; + let mut mount = None; + let mut seen = std::collections::HashSet::new(); + while !input.is_empty() { + input.parse::()?; + if input.is_empty() { + break; + } + let key: Ident = input.parse()?; + if !seen.insert(key.to_string()) { + return Err(syn::Error::new( + key.span(), + format!("duplicate #[command] option `{key}`"), + )); + } + input.parse::()?; + match key.to_string().as_str() { + "shard" => shard = Some(input.parse::()?), + "load" => { + let value: Ident = input.parse()?; + load = Some(match value.to_string().as_str() { + "required" => Load::Required, + "or_create" => Load::OrCreate, + "create" => Load::Create, + other => { + return Err(syn::Error::new( + value.span(), + format!("load must be `required`, `or_create` or `create`, not `{other}`"), + )) + } + }); + } + "outcome" => { + let ty: Type = input.parse()?; + outcome = Some(parse_outcome(ty)?); + } + "payload" => payload = Some(input.parse::()?), + "roles" => { + let expr: Expr = input.parse()?; + roles = Some(role_list(&expr)?); + } + "guard" => guard = Some(input.parse::()?), + "field" => field = Some(input.parse::()?), + "input" => input_ty = Some(input.parse::()?), + "transition" => transition = Some(input.parse::()?), + "mount" => mount = Some(input.parse::()?), + other => { + return Err(syn::Error::new( + key.span(), + format!( + "unknown #[command] option `{other}`; expected `shard`, `load`, `outcome`, `payload`, `roles`, `guard`, `field`, `input`, `transition` or `mount`" + ), + )) + } + } + } + let load = load.ok_or_else(|| { + syn::Error::new( + name.span(), + "#[command] needs `load = required | or_create | create`", + ) + })?; + let outcome = outcome.ok_or_else(|| { + syn::Error::new( + name.span(), + "#[command] needs `outcome = succeeded | eventual` (or `Succeeded

` / `Eventual

`)", + ) + })?; + if load != Load::Create && shard.is_none() { + return Err(syn::Error::new( + name.span(), + "#[command] needs `shard = ` to load the aggregate", + )); + } + Ok(Self { + name, + shard, + load, + outcome, + payload, + roles, + guard, + field, + input: input_ty, + transition, + mount, + }) + } +} + +fn validate_name(name: &LitStr) -> syn::Result<()> { + let value = name.value(); + let segments: Vec<&str> = value.split('.').collect(); + if segments.len() < 2 + || segments.iter().any(|segment| { + segment.is_empty() + || !segment + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') + }) + { + return Err(syn::Error::new( + name.span(), + "command name must be 2+ dot-separated snake_case segments (e.g. \"character.equip\")", + )); + } + Ok(()) +} + +fn parse_outcome(ty: Type) -> syn::Result { + let Type::Path(path) = &ty else { + return Err(syn::Error::new_spanned( + &ty, + "outcome must be `succeeded`, `eventual`, `Succeeded

` or `Eventual

`", + )); + }; + let last = path.path.segments.last().expect("non-empty path"); + let ident = last.ident.to_string(); + let bare = path.path.segments.len() == 1 && last.arguments.is_empty(); + match ident.as_str() { + "succeeded" | "eventual" if bare => Ok(Outcome::Keyword { + eventual: ident == "eventual", + span: last.ident.span(), + }), + "Succeeded" | "Eventual" if !last.arguments.is_empty() => Ok(Outcome::Type { + eventual: ident == "Eventual", + ty: Box::new(ty.clone()), + }), + _ => Err(syn::Error::new_spanned( + &ty, + "outcome must be `succeeded`, `eventual`, `Succeeded

` or `Eventual

`", + )), + } +} + +fn role_list(expr: &Expr) -> syn::Result> { + let Expr::Array(array) = expr else { + return Err(syn::Error::new_spanned( + expr, + "roles must be an array of string literals, e.g. [\"player\"]", + )); + }; + let mut roles = Vec::new(); + for elem in &array.elems { + match elem { + Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(value), + .. + }) if !value.value().trim().is_empty() => roles.push(value.clone()), + other => { + return Err(syn::Error::new_spanned( + other, + "roles entries must be non-empty string literals", + )) + } + } + } + if roles.is_empty() { + return Err(syn::Error::new_spanned( + expr, + "roles must name at least one role (omit `roles` for an internal command)", + )); + } + Ok(roles) +} + +/// One parameter after `self`. +struct Param { + name: Ident, + ty: Type, +} + +/// A `#[command]` aggregate method, collected by `#[sourced]`. +pub(crate) struct CommandMethod { + args: Args, + method: Ident, + principal: Param, + /// Whole-input parameter (no `input = T`), or the parameters mapped to + /// input fields by name. + params: Vec, + /// Every parameter name after `self`, in declaration order. + order: Vec, + ok_type: Option, +} + +impl CommandMethod { + pub(crate) fn method(&self) -> &Ident { + &self.method + } +} + +/// Strip `#[command(..)]` (and `#[principal]`) from `method`; `None` when it +/// is not a command. +pub(crate) fn take(method: &mut ImplItemFn) -> syn::Result> { + let Some(index) = method + .attrs + .iter() + .position(|attr| attr.path().is_ident("command")) + else { + return Ok(None); + }; + let attr = method.attrs.remove(index); + let args: Args = attr.parse_args()?; + if method.sig.asyncness.is_some() { + return Err(syn::Error::new_spanned( + method.sig.asyncness, + "#[command] aggregate methods are synchronous decisions; the generated handler does the I/O", + )); + } + if !matches!( + method.sig.inputs.first(), + Some(FnArg::Receiver(receiver)) if receiver.reference.is_some() && receiver.mutability.is_some() + ) { + return Err(syn::Error::new_spanned( + &method.sig, + "#[command] methods take `&mut self`", + )); + } + let mut principal = None; + let mut params = Vec::new(); + let mut order = Vec::new(); + for arg in method.sig.inputs.iter_mut().skip(1) { + let FnArg::Typed(typed) = arg else { continue }; + let Pat::Ident(pat) = &*typed.pat else { + return Err(syn::Error::new_spanned( + &typed.pat, + "#[command] parameters must be plain identifiers", + )); + }; + let is_principal = typed + .attrs + .iter() + .any(|attr| attr.path().is_ident("principal")); + typed + .attrs + .retain(|attr| !attr.path().is_ident("principal")); + let param = Param { + name: pat.ident.clone(), + ty: (*typed.ty).clone(), + }; + order.push(param.name.clone()); + if is_principal { + if principal.is_some() { + return Err(syn::Error::new_spanned( + &typed.pat, + "only one #[principal] parameter", + )); + } + principal = Some(param); + } else { + params.push(param); + } + } + let principal = principal.ok_or_else(|| { + syn::Error::new_spanned( + &method.sig.inputs, + "a #[command] method needs a `#[principal]` parameter (`&str` or `String`): the actor comes from the verified session, never from the input", + ) + })?; + if args.input.is_none() && params.len() != 1 { + return Err(syn::Error::new_spanned( + &method.sig.inputs, + "a #[command] method takes `#[principal]` and one input parameter; with `input = Type` its other parameters are the input's fields by name", + )); + } + let ok_type = match &method.sig.output { + ReturnType::Type(_, ty) => result_ok_type(ty), + ReturnType::Default => None, + }; + if ok_type.is_none() { + return Err(syn::Error::new_spanned( + &method.sig, + "#[command] methods return `Result` (`E: Display` becomes the rejection; `T` is the payload unless `payload = ..` is set)", + )); + } + if let (Outcome::Keyword { span, .. }, Some(_)) = (&args.outcome, &args.payload) { + return Err(syn::Error::new( + *span, + "with `payload = ..` name the payload type: `outcome = Succeeded

` or `Eventual

`", + )); + } + Ok(Some(CommandMethod { + args, + method: method.sig.ident.clone(), + principal, + params, + order, + ok_type, + })) +} + +/// `T` of `Result` (any path ending in `Result` with two arguments, or +/// one for an alias like `SourcedResult`); `()` for a bare alias. +fn result_ok_type(ty: &Type) -> Option { + let Type::Path(path) = ty else { return None }; + let last = path.path.segments.last()?; + if !last.ident.to_string().ends_with("Result") { + return None; + } + match &last.arguments { + syn::PathArguments::AngleBracketed(args) => match args.args.first()? { + syn::GenericArgument::Type(ok) => Some(ok.clone()), + _ => None, + }, + syn::PathArguments::None => Some(syn::parse_quote!(())), + syn::PathArguments::Parenthesized(_) => None, + } +} + +fn is_reference(ty: &Type) -> bool { + matches!(ty, Type::Reference(_)) +} + +fn pascal(segment: &str) -> String { + segment + .split('_') + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } + }) + .collect() +} + +/// The mount, its handler and its registration. `transition` is the domain +/// transition witness `#[sourced]` discovered for the method, if any. +pub(crate) fn expand( + command: &CommandMethod, + aggregate: &Type, + transition: Option, + framework: &TokenStream, +) -> syn::Result { + let fw = framework; + let args = &command.args; + let name = &args.name; + let name_value = name.value(); + let last = name_value.rsplit('.').next().expect("validated"); + let mount = args + .mount + .clone() + .unwrap_or_else(|| format_ident!("{}", pascal(last), span = name.span())); + let method = &command.method; + let principal = &command.principal; + let principal_name = principal.name.to_string(); + + let input_ty: Type = match &args.input { + Some(ty) => ty.clone(), + None => command.params[0].ty.clone(), + }; + + // Payload type. + let payload_ty: Type = match &args.outcome { + Outcome::Type { ty, .. } => { + let Type::Path(path) = &**ty else { + unreachable!() + }; + let last = path.path.segments.last().expect("path"); + match &last.arguments { + syn::PathArguments::AngleBracketed(a) => match a.args.first() { + Some(syn::GenericArgument::Type(p)) => p.clone(), + _ => return Err(syn::Error::new_spanned(ty, "expected `Succeeded

`")), + }, + _ => return Err(syn::Error::new_spanned(ty, "expected `Succeeded

`")), + } + } + Outcome::Keyword { .. } => command.ok_type.clone().expect("checked"), + }; + let eventual = match &args.outcome { + Outcome::Keyword { eventual, .. } | Outcome::Type { eventual, .. } => *eventual, + }; + let outcome_ty = if eventual { + quote!(#fw::command::Eventual<#payload_ty>) + } else { + quote!(#fw::command::Succeeded<#payload_ty>) + }; + let terminal = if eventual { + quote!(eventual) + } else { + quote!(succeeded) + }; + + // Shard. + let shard_by_principal = args.shard.as_ref().is_some_and(|s| *s == principal.name); + let shard_fn = match &args.shard { + Some(field) if !shard_by_principal => Some(quote! { + /// Aggregate id (shard) of a command input. + pub fn shard(input: &#input_ty) -> ::std::string::String { + ::std::string::ToString::to_string(&input.#field) + } + }), + _ => None, + }; + let shard_expr = if shard_by_principal { + quote!(::std::clone::Clone::clone(&__principal)) + } else { + quote!(Self::shard(&input)) + }; + let load = match args.load { + Load::Required => quote! { + let __id = #shard_expr; + let mut __checkout = #fw::microsvc::require_loaded(ctx.get(&__id).await?, __id)?; + }, + Load::OrCreate => quote! { + let __id = #shard_expr; + let mut __checkout = match ctx.get(&__id).await? { + ::std::option::Option::Some(checkout) => checkout, + ::std::option::Option::None => ctx.create(), + }; + }, + Load::Create => quote! { + let mut __checkout = ctx.create(); + }, + }; + + // Call arguments. + let principal_arg = if is_reference(&principal.ty) { + quote!(&__principal) + } else { + quote!(::std::convert::Into::into(::std::clone::Clone::clone( + &__principal + ))) + }; + let call_args: Vec = if args.input.is_some() { + command + .params + .iter() + .map(|param| { + let field = ¶m.name; + if is_reference(¶m.ty) { + quote_spanned!(field.span()=> &input.#field) + } else { + quote_spanned!(field.span()=> input.#field) + } + }) + .collect() + } else { + vec![quote!(input)] + }; + let payload = match &args.payload { + Some(payload) => quote! { + let _ = __out; + let __payload: #payload_ty = #payload(__checkout.aggregate()); + }, + None => quote! { let __payload: #payload_ty = __out; }, + }; + + // Compile-time principal check: no input field may be named like the + // principal parameter (the session is its only source). + let message = format!( + "#[command(\"{name_value}\")]: the input has a field named `{principal_name}`, the #[principal] parameter; the principal comes from the verified session only, so it can never be an input field" + ); + let principal_check = quote_spanned! {principal.name.span()=> + const _: () = { + if #fw::command::__input_has_field( + <#input_ty as #fw::command::CommandInputType>::FIELD_NAMES, + #principal_name, + ) { + ::core::panic!(#message); + } + }; + }; + + let transition = match &args.transition { + Some(path) => Some(quote!(#path)), + None => transition, + }; + let registration = match &transition { + Some(transition) => quote! { + routes.command_transition::<#transition, #input_ty, #outcome_ty>(Self::COMMAND) + }, + None => quote! { + routes.typed_command(#fw::command::typed_command::<#input_ty, #outcome_ty>(Self::COMMAND)) + }, + }; + let field = args.field.as_ref().map(|field| quote!(.field_name(#field))); + let roles = args + .roles + .as_ref() + .map(|roles| quote!(.roles([#(#roles),*]))); + let finish = match &args.guard { + Some(guard) => quote!(.guarded(#guard, Self::handle)), + None => quote!(.handle(Self::handle)), + }; + let doc = format!( + "Mounts `{name_value}` (`{}::{method}`): principal from the verified session, then load, call, commit.", + quote!(#aggregate).to_string().replace(' ', "") + ); + + let args_in_order = order_args(command, &principal_arg, &call_args); + + Ok(quote! { + #[doc = #doc] + #[derive(Debug, Clone, Copy, Default)] + pub struct #mount; + + #principal_check + + impl #mount { + /// Command name. + pub const COMMAND: &'static str = #name; + + #shard_fn + + /// The generated handler (also usable from a hand-written mount). + pub async fn handle( + ctx: &#fw::microsvc::CausalCommandContext<'_, #aggregate>, + input: #input_ty, + ) -> ::std::result::Result< + #fw::command::PreparedCommand<#outcome_ty>, + #fw::microsvc::HandlerError, + > { + // The principal: the verified session only. + let __principal: ::std::string::String = ctx.user_id()?.to_owned(); + #load + let __out = __checkout + .aggregate_mut() + .#method(#(#args_in_order),*) + .map_err(|error| #fw::microsvc::HandlerError::Rejected(error.to_string()))?; + #payload + ctx.publish_events().commit(__checkout)?.#terminal(__payload) + } + } + + impl #fw::microsvc::PortableCommand for #mount + where + D: #fw::microsvc::CausalRouteDependencies + + ::std::marker::Send + + ::std::marker::Sync + + 'static, + { + fn install(self, routes: #fw::microsvc::Routes) -> #fw::microsvc::Routes { + #registration + #field + #roles + #finish + } + } + }) +} + +/// Arguments in the method's declaration order. +fn order_args( + command: &CommandMethod, + principal_arg: &TokenStream, + call_args: &[TokenStream], +) -> Vec { + let mut out = Vec::new(); + let mut rest = call_args.iter(); + for name in &command.order { + if *name == command.principal.name { + out.push(principal_arg.clone()); + } else if let Some(arg) = rest.next() { + out.push(arg.clone()); + } + } + out +} diff --git a/distributed_macros/src/command.rs b/distributed_macros/src/command.rs index 46fd280b0..6126d1db8 100644 --- a/distributed_macros/src/command.rs +++ b/distributed_macros/src/command.rs @@ -176,6 +176,17 @@ pub fn expand( item: proc_macro2::TokenStream, ) -> syn::Result { let framework = crate::shared::framework_path()?; + if let Ok(name) = syn::parse2::( + attr.clone() + .into_iter() + .take(1) + .collect::(), + ) { + return Err(syn::Error::new( + name.span(), + "`#[command(\"aggregate.verb\", ..)]` on an aggregate method must be inside a `#[sourced]` impl (the handler-function form is `#[command(id = \"..\", ..)]`)", + )); + } let args = syn::parse2::(attr)?; let function = syn::parse2::(item)?; let id = args.id.ok_or_else(|| { diff --git a/distributed_macros/src/command_types.rs b/distributed_macros/src/command_types.rs index 343e8a79e..39d0a83f4 100644 --- a/distributed_macros/src/command_types.rs +++ b/distributed_macros/src/command_types.rs @@ -149,6 +149,7 @@ fn expand( let mut field_tokens = Vec::new(); let mut effect_input_markers = Vec::new(); + let mut field_names: Vec = Vec::new(); for field in &fields.named { validate_serde_field_shape(&field.attrs, serde_direction)?; let field_name = field @@ -165,6 +166,11 @@ fn expand( .map(|rule| rule.apply(rust_field_name)) .unwrap_or_else(|| rust_field_name.to_string()) }); + for name in [rust_field_name.to_string(), field_name_str.clone()] { + if !field_names.contains(&name) { + field_names.push(name); + } + } let (type_name, nullable, list, item_nullable, nested) = map_type(&field.ty, field, &nested_trait, &method)?; let unsigned_integer = unsigned_integer_tokens(&field.ty, &framework); @@ -221,8 +227,13 @@ fn expand( } let type_name_str = name.to_string(); + let field_names_const = matches!(serde_direction, SerdeDirection::Deserialize).then(|| { + quote! { const FIELD_NAMES: &'static [&'static str] = &[#(#field_names),*]; } + }); Ok(quote! { impl #trait_path for #name { + #field_names_const + fn #method() -> #type_def { #type_def::new( #type_name_str, diff --git a/distributed_macros/src/lib.rs b/distributed_macros/src/lib.rs index bd24f3f4a..9652fc597 100644 --- a/distributed_macros/src/lib.rs +++ b/distributed_macros/src/lib.rs @@ -1,4 +1,5 @@ mod aggregate; +mod aggregate_command; mod application; mod command; mod command_input_defaults; @@ -20,7 +21,14 @@ use proc_macro::TokenStream; use syn::DeriveInput; /// Generate one typed command's portable contract and optional executable -/// mount from the same handler declaration. +/// mount from the same handler declaration (`#[command(id = "..", ..)]` on +/// an async handler function). +/// +/// The aggregate-method form, `#[command("aggregate.verb", shard = .., +/// load = .., outcome = ..)]` on a method inside a `#[sourced]` impl, is +/// handled by `#[sourced]` (see the `aggregate_command` module docs): it +/// generates the handler, binding the `#[principal]` parameter from the +/// verified session, and the portable mount. #[proc_macro_attribute] pub fn command(attr: TokenStream, item: TokenStream) -> TokenStream { command::expand(attr.into(), item.into()) diff --git a/distributed_macros/src/sourced.rs b/distributed_macros/src/sourced.rs index 3e16ea43e..4e1e4c10c 100644 --- a/distributed_macros/src/sourced.rs +++ b/distributed_macros/src/sourced.rs @@ -715,8 +715,13 @@ pub(crate) fn expand_sourced(attr: TokenStream2, item: TokenStream2) -> syn::Res let mut seen_variants: std::collections::HashMap = std::collections::HashMap::new(); + let mut commands = Vec::new(); for item in &mut impl_block.items { if let syn::ImplItem::Fn(method) = item { + if let Some(command) = crate::aggregate_command::take(method)? { + commands.push(command); + continue; + } match find_and_remove_event_attr(&mut method.attrs) { Ok(Some(event_attr)) => { // Event methods are replayed as `self.method(...)`, so they @@ -863,8 +868,26 @@ pub(crate) fn expand_sourced(attr: TokenStream2, item: TokenStream2) -> syn::Res } } + let command_methods: std::collections::BTreeSet = commands + .iter() + .map(|command| command.method().to_string()) + .collect(); let domain_command_transitions = - discover_domain_command_transitions(&impl_block, &event_methods); + discover_domain_command_transitions(&impl_block, &event_methods, &command_methods); + let self_ty = (*impl_block.self_ty).clone(); + let command_mounts = commands + .iter() + .map(|command| { + let witness = domain_command_transitions + .iter() + .find(|transition| transition.method_name == *command.method()) + .map(|transition| { + let ty = method_name_to_type_ident(&transition.method_name); + quote!(domain_commands::#ty) + }); + crate::aggregate_command::expand(command, &self_ty, witness, &framework) + }) + .collect::>>()?; let deletion_identity = if uses_deletion_identity { let identity = format_ident!("{struct_name}DomainIdentity"); @@ -1065,6 +1088,7 @@ pub(crate) fn expand_sourced(attr: TokenStream2, item: TokenStream2) -> syn::Res #(#generated_domain_types)* #deletion_identity #domain_commands_module + #(#command_mounts)* #upcaster_wrappers #aggregate_impl }; @@ -1166,6 +1190,7 @@ fn ungroup_expr(expression: &Expr) -> &Expr { fn discover_domain_command_transitions( impl_block: &ItemImpl, event_methods: &[EventMethodInfo], + command_methods: &std::collections::BTreeSet, ) -> Vec { let recorders: std::collections::BTreeMap = event_methods .iter() @@ -1198,7 +1223,9 @@ fn discover_domain_command_transitions( .collect::>(); let mut transitions = Vec::new(); for (name, method) in &methods { - if !matches!(method.vis, syn::Visibility::Public(_)) { + // Public methods are transitions; so is every `#[command]` method + // (its generated mount needs the witness whatever its visibility). + if !matches!(method.vis, syn::Visibility::Public(_)) && !command_methods.contains(name) { continue; } diff --git a/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.rs b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.rs new file mode 100644 index 000000000..65412335f --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.rs @@ -0,0 +1,26 @@ +use distributed::{sourced, Entity}; +use serde::Deserialize; + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct EquipInput { + pub hero_id: String, + pub owner: String, +} + +#[derive(Default)] +pub struct Hero { + entity: Entity, +} + +#[sourced(entity, aggregate_type = "hero")] +impl Hero { + #[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] + pub fn equip(&mut self, input: EquipInput) -> Result<(), String> { + let _ = input.owner; + Ok(()) + } +} + +fn main() { + let _ = Hero::default().entity; +} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.stderr b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.stderr new file mode 100644 index 000000000..fa6fbf4da --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.stderr @@ -0,0 +1,5 @@ +error: a #[command] method needs a `#[principal]` parameter (`&str` or `String`): the actor comes from the verified session, never from the input + --> tests/compile_fail/aggregate_command_missing_principal.rs:18:18 + | +18 | pub fn equip(&mut self, input: EquipInput) -> Result<(), String> { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.rs b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.rs new file mode 100644 index 000000000..9e6eb6354 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.rs @@ -0,0 +1,11 @@ +use distributed::command; + +pub struct Hero; + +#[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] +pub fn equip(actor: &str) -> Result<(), String> { + let _ = actor; + Ok(()) +} + +fn main() {} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.stderr b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.stderr new file mode 100644 index 000000000..f7589f842 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.stderr @@ -0,0 +1,5 @@ +error: `#[command("aggregate.verb", ..)]` on an aggregate method must be inside a `#[sourced]` impl (the handler-function form is `#[command(id = "..", ..)]`) + --> tests/compile_fail/aggregate_command_outside_sourced.rs:5:11 + | +5 | #[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] + | ^^^^^^^^^^^^ diff --git a/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.rs b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.rs new file mode 100644 index 000000000..7746646a6 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.rs @@ -0,0 +1,29 @@ +use distributed::{sourced, Entity}; +use serde::Deserialize; + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct EquipInput { + pub hero_id: String, +} + +#[derive(Default)] +pub struct Hero { + entity: Entity, +} + +fn payload(_hero: &Hero) -> u32 { + 0 +} + +#[sourced(entity, aggregate_type = "hero")] +impl Hero { + #[command("hero.equip", shard = hero_id, load = required, outcome = eventual, payload = payload)] + pub fn equip(&mut self, #[principal] actor: &str, input: EquipInput) -> Result<(), String> { + let _ = (actor, input); + Ok(()) + } +} + +fn main() { + let _ = (Hero::default().entity, payload); +} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.stderr b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.stderr new file mode 100644 index 000000000..af6ae6a9e --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.stderr @@ -0,0 +1,5 @@ +error: with `payload = ..` name the payload type: `outcome = Succeeded

` or `Eventual

` + --> tests/compile_fail/aggregate_command_payload_needs_type.rs:20:73 + | +20 | #[command("hero.equip", shard = hero_id, load = required, outcome = eventual, payload = payload)] + | ^^^^^^^^ diff --git a/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.rs b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.rs new file mode 100644 index 000000000..73368d33e --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.rs @@ -0,0 +1,35 @@ +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct EquipInput { + pub hero_id: String, + pub actor: String, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct Equipped { + pub hero_id: String, +} + +#[derive(Default)] +pub struct Hero { + entity: Entity, +} + +#[sourced(entity, aggregate_type = "hero")] +impl Hero { + #[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] + pub fn equip(&mut self, #[principal] actor: &str, input: EquipInput) -> Result { + let _ = (actor, input.actor); + self.touched().map_err(|e| e.to_string())?; + Ok(Equipped { hero_id: input.hero_id }) + } + + #[event("hero.touched")] + fn touched(&mut self) {} +} + +fn main() { + let _ = Hero::default().entity; +} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.stderr b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.stderr new file mode 100644 index 000000000..87d11dc54 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.stderr @@ -0,0 +1,5 @@ +error[E0080]: evaluation panicked: #[command("hero.equip")]: the input has a field named `actor`, the #[principal] parameter; the principal comes from the verified session only, so it can never be an input field + --> tests/compile_fail/aggregate_command_principal_input_field.rs:23:42 + | +23 | pub fn equip(&mut self, #[principal] actor: &str, input: EquipInput) -> Result { + | ^^^^^ evaluation of `_` failed here diff --git a/examples/game/server/crawler/src/lib.rs b/examples/game/server/crawler/src/lib.rs index adb311930..0a933ca9b 100644 --- a/examples/game/server/crawler/src/lib.rs +++ b/examples/game/server/crawler/src/lib.rs @@ -2,12 +2,16 @@ //! its own native SQLite cell (`distributed` feature `cell-sqlite`, opened by //! `game_demos`). //! -//! The only command is [`RECORD_KILL`]. It is dispatched by the zone's fact +//! The only command is [`RECORD_KILL`], declared on the aggregate method +//! [`Crawler::record_kill`] with `#[command]` (which generates the +//! [`RecordKill`] mount and its handler). It is dispatched by the zone's fact //! worker through `distributed_game::cell::CellFactSink`, never by a client: //! //! - the **principal** comes from the trusted server-built session -//! (`x-user-id`, bound by the host from the player's verified ticket); the -//! input has no identity field and unknown fields are rejected; +//! (`x-user-id`, bound by the host from the player's verified ticket) into +//! the method's `#[principal]` parameter; the input has no identity field +//! (a field named like it would not compile) and unknown fields are +//! rejected; //! - the **XP amount** is decided here from the mob tier ([`xp_for_tier`]); //! the zone only reports which mob died; //! - the command id is derived from the fact key, so a retry or re-emission @@ -19,18 +23,13 @@ //! that outbox onto a bus and the read-model service (`mobpit_service`) //! projects it into the `player_stats` table behind GraphQL `@live`. -use distributed::command::{typed_command, PreparedCommand, Succeeded}; -use distributed::microsvc::{ - CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, -}; use distributed::{sourced, DomainState, Entity}; use serde::{Deserialize, Serialize}; -use std::time::UNIX_EPOCH; /// Aggregate type (cell instance names are `Crawler:{principal}`). pub const AGGREGATE_TYPE: &str = "Crawler"; /// Command dispatched for every `MobKilled` fact. -pub const RECORD_KILL: &str = "crawler.record_kill"; +pub const RECORD_KILL: &str = RecordKill::COMMAND; /// Event recorded by [`RECORD_KILL`]. pub const MOB_KILLED_EVENT: &str = "crawler.mob_killed"; @@ -110,6 +109,50 @@ impl From<&Crawler> for CrawlerState { domain_state = CrawlerState )] impl Crawler { + /// Credit a kill to the session principal: XP from the mob tier, the + /// name capped at 16 characters. The receipt carries the recorded + /// event's stream position (the leaderboard occurrence). + /// + /// The cell is the principal's own shard, so a mismatched principal + /// cannot load it either. Event, ledger receipt, snapshot and the outbox + /// row of the domain event commit in one cell transaction. + #[command( + "crawler.record_kill", + shard = principal, + load = or_create, + outcome = succeeded + )] + pub fn record_kill( + &mut self, + #[principal] principal: &str, + input: RecordKillInput, + ) -> Result { + let xp = + xp_for_tier(input.tier).ok_or_else(|| format!("unknown mob tier {}", input.tier))?; + let name: String = input.name.chars().take(16).collect(); + self.mob_killed( + principal.to_string(), + name.clone(), + input.mob.clone(), + input.tier, + xp, + ) + .map_err(|e| e.to_string())?; + let recorded = self.entity.last_recorded().ok_or("no event recorded")?; + Ok(KillReceipt { + crawler: principal.to_string(), + name, + mob: input.mob, + tier: input.tier, + xp, + total_xp: self.xp, + kills: self.kills, + level: self.level, + sequence: recorded.sequence, + at_ms: recorded.unix_ms(), + }) + } + #[event("crawler.mob_killed", version = 1, domain)] fn mob_killed(&mut self, id: String, name: String, mob: String, tier: u8, xp: u32) { let _ = (mob, tier); @@ -139,8 +182,8 @@ pub type MobKilledArgs = (String, String, String, u8, u32); /// [`RECORD_KILL`] input. No principal, no XP: both are server-decided. /// -/// `FactCommand` binds it to [`RECORD_KILL`] (the name [`RecordKill`] -/// mounts it under), so the zone emits it typed — `ctx.emit_keyed(killer, +/// `FactCommand` binds it to [`RECORD_KILL`] (the name the generated +/// [`RecordKill`] mounts it under), so the zone emits it typed — `ctx.emit_keyed(killer, /// RecordKillInput { .. }, &[mob, generation])` — and the derive rejects /// identity or reward fields at compile time. #[derive( @@ -181,78 +224,6 @@ pub struct KillReceipt { pub at_ms: u64, } -/// Mounts [`RECORD_KILL`] on a `Crawler` cell. -pub struct RecordKill; - -impl PortableCommand for RecordKill -where - D: CausalRouteDependencies + Send + Sync + 'static, -{ - fn install(self, routes: Routes) -> Routes { - routes - .typed_command(typed_command::>( - RECORD_KILL, - )) - .guarded( - |ctx: &CausalCommandContext<'_, Crawler>| ctx.session().user_id().is_some(), - handle_record_kill, - ) - } -} - -async fn handle_record_kill( - ctx: &CausalCommandContext<'_, Crawler>, - input: RecordKillInput, -) -> Result>, HandlerError> { - // Identity: the trusted session only. The cell is the principal's own - // shard, so a mismatched principal cannot load it either. - let principal = ctx.session().user_id().unwrap_or_default().to_string(); - let xp = xp_for_tier(input.tier) - .ok_or_else(|| HandlerError::Rejected(format!("unknown mob tier {}", input.tier)))?; - let name: String = input.name.chars().take(16).collect(); - let repo = ctx.repo(); - let mut crawler = match repo.get(&principal).await? { - Some(crawler) => crawler, - None => repo.create(), - }; - crawler - .mob_killed( - principal.clone(), - name.clone(), - input.mob.clone(), - input.tier, - xp, - ) - .map_err(|e| HandlerError::Rejected(e.to_string()))?; - let (sequence, at_ms) = crawler - .entity - .events() - .last() - .map(|record| { - let at = record - .timestamp - .duration_since(UNIX_EPOCH) - .map_or(0, |d| d.as_millis() as u64); - (record.sequence, at) - }) - .unwrap_or_default(); - let receipt = KillReceipt { - crawler: principal, - name, - mob: input.mob, - tier: input.tier, - xp, - total_xp: crawler.xp, - kills: crawler.kills, - level: crawler.level, - sequence, - at_ms, - }; - // Event, ledger receipt, snapshot and the outbox row of the domain event - // commit in one cell transaction. - repo.publish_events().commit(crawler)?.succeeded(receipt) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/command/mod.rs b/src/command/mod.rs index 81b8ff028..146a156ec 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -13,7 +13,8 @@ mod types; pub(crate) use types::scalar_type_name; pub use types::{ - CommandInputType, CommandOutputType, CommandTypeDef, CommandTypeField, CommandUnsignedInteger, + __input_has_field, CommandInputType, CommandOutputType, CommandTypeDef, CommandTypeField, + CommandUnsignedInteger, }; mod direct_projection; diff --git a/src/command/types.rs b/src/command/types.rs index 7f055aefc..db672027b 100644 --- a/src/command/types.rs +++ b/src/command/types.rs @@ -121,9 +121,37 @@ impl CommandTypeDef { /// Serde's deserialization direction. Transport-specific naming restrictions /// are checked when an adapter exposes the declaration. pub trait CommandInputType { + /// Rust and serialized names of the input's top-level fields + /// (`#[derive(CommandInput)]` fills it). Aggregate `#[command]` methods + /// check at compile time that none of them is named like the principal + /// parameter, which only the verified session may supply. + const FIELD_NAMES: &'static [&'static str] = &[]; + fn command_type() -> CommandTypeDef; } +/// True when `fields` contains `name` (usable in `const` items; the +/// aggregate `#[command]` principal check). +#[doc(hidden)] +pub const fn __input_has_field(fields: &[&str], name: &str) -> bool { + let name = name.as_bytes(); + let mut i = 0; + while i < fields.len() { + let field = fields[i].as_bytes(); + if field.len() == name.len() { + let mut j = 0; + while j < name.len() && field[j] == name[j] { + j += 1; + } + if j == name.len() { + return true; + } + } + i += 1; + } + false +} + /// Structural description of a command result, following Serde serialization. /// /// Prefer deriving [`CommandOutput`](crate::CommandOutput). Atomic read-model diff --git a/src/entity/entity.rs b/src/entity/entity.rs index c435ef881..10f4f86ab 100644 --- a/src/entity/entity.rs +++ b/src/entity/entity.rs @@ -15,6 +15,25 @@ use crate::SourcedResult; use super::{BitcodePayloadCodec, EventRecord, EventRecordError, PayloadCodec}; +/// Position and time of an event recorded by the current unit of work +/// ([`Entity::last_recorded`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Recorded { + /// Stream sequence (1-based; equals the stream version after commit). + pub sequence: u64, + /// Event time. + pub timestamp: SystemTime, +} + +impl Recorded { + /// Event time as Unix milliseconds (0 before the epoch). + pub fn unix_ms(&self) -> u64 { + self.timestamp + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis() as u64) + } +} + #[derive(Serialize, Deserialize)] pub struct Entity { id: String, @@ -330,6 +349,18 @@ impl Entity { &self.events[start..] } + /// Where and when the latest event recorded by the current unit of work + /// (not yet committed) lands in the stream: its sequence and timestamp, + /// as they will be committed. `None` when nothing was recorded. Command + /// results (receipts, leaderboard occurrences) read this instead of + /// digging through [`events`](Self::events). + pub fn last_recorded(&self) -> Option { + self.new_events().last().map(|record| Recorded { + sequence: record.sequence, + timestamp: record.timestamp, + }) + } + /// Mark all current events as committed. Called by repository after successful commit. pub fn mark_committed(&mut self) { self.committed_version = self.version; diff --git a/src/entity/mod.rs b/src/entity/mod.rs index acac9eaf6..568d08bc5 100644 --- a/src/entity/mod.rs +++ b/src/entity/mod.rs @@ -2,7 +2,7 @@ mod entity; mod event_record; mod upcaster; -pub use entity::Entity; +pub use entity::{Entity, Recorded}; pub use event_record::{ BitcodePayloadCodec, EventRecord, EventRecordError, PayloadCodec, BITCODE_PAYLOAD_CODEC, BITCODE_PAYLOAD_CODEC_VERSION, diff --git a/src/lib.rs b/src/lib.rs index f73f9d679..7fa71b7fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -69,8 +69,8 @@ pub mod trace_context; // Re-export entity types at crate root for convenience pub use entity::{ upcast_events, upcast_events_for_replay, upcast_payload, BitcodePayloadCodec, Entity, - EventRecord, EventRecordError, EventUpcaster, PayloadCodec, UpcastError, BITCODE_PAYLOAD_CODEC, - BITCODE_PAYLOAD_CODEC_VERSION, + EventRecord, EventRecordError, EventUpcaster, PayloadCodec, Recorded, UpcastError, + BITCODE_PAYLOAD_CODEC, BITCODE_PAYLOAD_CODEC_VERSION, }; // Placement-independent application composition. The module path remains the diff --git a/tests/aggregate_command/main.rs b/tests/aggregate_command/main.rs new file mode 100644 index 000000000..6939960f3 --- /dev/null +++ b/tests/aggregate_command/main.rs @@ -0,0 +1,472 @@ +//! `#[command]` on aggregate methods inside `#[sourced]`: the generated +//! handler binds the principal from the session only, loads / creates the +//! aggregate, maps errors to rejections, builds the payload and commits; the +//! generated registration matches the hand-written form (command specs, +//! GraphQL SDL and the client manifest). + +use distributed::cell_host::AggregateCell; +use distributed::command::{Eventual, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + require_loaded, CausalCommandContext, HandlerError, Session, USER_ID_KEY, +}; +use distributed::{sourced, DomainState, Entity}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct Hero { + entity: Entity, + owner: String, + name: String, + slots: Vec<(String, String)>, + naps: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, DomainState)] +#[domain_state(version = 1)] +pub struct HeroState { + pub hero_id: String, + pub owner: String, + pub name: String, + pub naps: u32, +} + +impl From<&Hero> for HeroState { + fn from(hero: &Hero) -> Self { + Self { + hero_id: hero.entity.id().to_string(), + owner: hero.owner.clone(), + name: hero.name.clone(), + naps: hero.naps, + } + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +#[serde(deny_unknown_fields)] +pub struct CreateHero { + pub hero_id: String, + pub name: String, +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +#[serde(deny_unknown_fields)] +pub struct EquipInput { + pub hero_id: String, + pub slot: String, + pub item: String, +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +#[serde(deny_unknown_fields)] +pub struct NapInput { + pub minutes: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, distributed::CommandOutput)] +pub struct HeroPayload { + pub hero_id: String, + pub name: String, + pub items: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, distributed::CommandOutput)] +pub struct NapReceipt { + pub hero_id: String, + pub naps: u32, + pub sequence: u64, + pub at_ms: u64, +} + +#[derive(Debug)] +pub enum HeroError { + NotOwner, + Exists, + EmptySlot, + Sourced(String), +} + +impl std::fmt::Display for HeroError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NotOwner => f.write_str("not your hero"), + Self::Exists => f.write_str("hero exists"), + Self::EmptySlot => f.write_str("slot is empty"), + Self::Sourced(e) => f.write_str(e), + } + } +} + +fn sourced_error(e: impl std::fmt::Display) -> HeroError { + HeroError::Sourced(e.to_string()) +} + +pub fn hero_payload(hero: &Hero) -> HeroPayload { + HeroPayload { + hero_id: hero.entity.id().to_string(), + name: hero.name.clone(), + items: hero.slots.len() as u32, + } +} + +pub fn signed_in(ctx: &CausalCommandContext<'_, Hero>) -> bool { + ctx.session().user_id().is_some() +} + +#[sourced( + entity, + aggregate_type = "hero", + events = "HeroEvent", + domain_state = HeroState +)] +impl Hero { + /// Whole input, new aggregate, the method's `Ok` value is the payload. + #[command("hero.create", load = create, outcome = eventual, roles = ["player"])] + pub fn create( + &mut self, + #[principal] owner: &str, + input: CreateHero, + ) -> Result { + if !self.entity.id().is_empty() { + return Err(HeroError::Exists); + } + self.created(input.hero_id, owner.to_string(), input.name) + .map_err(sourced_error)?; + Ok(hero_payload(self)) + } + + /// Parameters mapped to input fields by name (`&str` borrows), a payload + /// function, a guard and a custom GraphQL field. + #[command( + "hero.equip", + shard = hero_id, + load = required, + input = EquipInput, + outcome = Eventual, + payload = hero_payload, + roles = ["player"], + field = "heroes_equip", + guard = signed_in + )] + pub fn equip( + &mut self, + slot: String, + #[principal] actor: &str, + item: &str, + ) -> Result<(), HeroError> { + if actor != self.owner { + return Err(HeroError::NotOwner); + } + if slot.is_empty() { + return Err(HeroError::EmptySlot); + } + self.equipped(slot, item.to_string()).map_err(sourced_error) + } + + /// Sharded by the session principal (`String` parameter), or-created, + /// receipt from `Entity::last_recorded`. + #[command("hero.nap", shard = sleeper, load = or_create, outcome = succeeded)] + pub fn nap( + &mut self, + #[principal] sleeper: String, + input: NapInput, + ) -> Result { + let _ = input.minutes; + if self.entity.id().is_empty() { + self.created(sleeper.clone(), sleeper.clone(), sleeper.clone()) + .map_err(sourced_error)?; + } + self.napped().map_err(sourced_error)?; + let recorded = self.entity.last_recorded().expect("recorded"); + Ok(NapReceipt { + hero_id: sleeper, + naps: self.naps, + sequence: recorded.sequence, + at_ms: recorded.unix_ms(), + }) + } + + #[event("hero.created", version = 1, domain)] + fn created(&mut self, hero_id: String, owner: String, name: String) { + self.entity.set_id(hero_id); + self.owner = owner; + self.name = name; + } + + #[event("hero.equipped", version = 1, domain)] + fn equipped(&mut self, slot: String, item: String) { + self.slots.push((slot, item)); + } + + #[event("hero.napped", version = 1, domain)] + fn napped(&mut self) { + self.naps += 1; + } +} + +fn session(user: Option<&str>) -> Session { + let mut session = Session::new(); + if let Some(user) = user { + session.set(USER_ID_KEY, user); + session.set("x-roles", "player"); + } + session +} + +fn cell(id: &str) -> AggregateCell { + AggregateCell::::new(id) + .unwrap() + .mount(Create) + .mount(Equip) + .mount(Nap) +} + +#[tokio::test] +async fn generated_handlers_bind_the_session_principal_and_commit() { + let cell = cell("h1"); + let created = cell + .dispatch( + "hero.create", + json!({ "hero_id": "h1", "name": "Ana" }), + session(Some("ana")), + ) + .await + .unwrap(); + assert_eq!( + created, + json!({ "hero_id": "h1", "name": "Ana", "items": 0 }) + ); + let equipped = cell + .dispatch( + "hero.equip", + json!({ "hero_id": "h1", "slot": "hand", "item": "sword" }), + session(Some("ana")), + ) + .await + .unwrap(); + assert_eq!(equipped["items"], 1); + // Domain errors become rejections; the owner check sees the session's + // principal, never the input. + let stolen = cell + .dispatch( + "hero.equip", + json!({ "hero_id": "h1", "slot": "hand", "item": "axe" }), + session(Some("bo")), + ) + .await; + assert!( + matches!(&stolen, Err(HandlerError::Rejected(m)) if m == "not your hero"), + "{stolen:?}" + ); + // No session principal: refused before the aggregate is touched. + let anonymous = cell + .dispatch( + "hero.equip", + json!({ "hero_id": "h1", "slot": "hand", "item": "axe" }), + session(None), + ) + .await; + assert!(anonymous.is_err(), "{anonymous:?}"); + // Unknown fields (an attempt to pass identity) are refused by the input. + let forged = cell + .dispatch( + "hero.equip", + json!({ "hero_id": "h1", "slot": "hand", "item": "axe", "actor": "ana" }), + session(Some("bo")), + ) + .await; + assert!(forged.is_err(), "{forged:?}"); + let hero = cell.load().await.unwrap().unwrap(); + assert_eq!(hero.slots, [("hand".to_string(), "sword".to_string())]); + assert_eq!(hero.owner, "ana"); + + // `load = required` on a missing aggregate. + let missing = AggregateCell::::new("nobody") + .unwrap() + .mount(Equip) + .dispatch( + "hero.equip", + json!({ "hero_id": "nobody", "slot": "hand", "item": "axe" }), + session(Some("ana")), + ) + .await; + assert!(missing.is_err(), "{missing:?}"); +} + +#[tokio::test] +async fn or_create_sharded_by_principal_returns_the_recorded_position() { + let cell = AggregateCell::::new("cy").unwrap().mount(Nap); + let first = cell + .dispatch("hero.nap", json!({ "minutes": 5 }), session(Some("cy"))) + .await + .unwrap(); + // created (1) + napped (2) + assert_eq!(first["sequence"], 2); + assert_eq!(first["naps"], 1); + assert!(first["at_ms"].as_u64().unwrap() > 1_600_000_000_000); + let second = cell + .dispatch("hero.nap", json!({ "minutes": 5 }), session(Some("cy"))) + .await + .unwrap(); + assert_eq!(second["sequence"], 3); + assert_eq!(second["hero_id"], "cy"); + let history = cell.history().await.unwrap(); + assert_eq!(history.len(), 3); + assert_eq!(history[2].sequence, 3); +} + +#[test] +fn mounts_expose_command_names_and_shards() { + assert_eq!(Create::COMMAND, "hero.create"); + assert_eq!(Equip::COMMAND, "hero.equip"); + assert_eq!(Nap::COMMAND, "hero.nap"); + let input = EquipInput { + hero_id: "h9".into(), + slot: "s".into(), + item: "i".into(), + }; + assert_eq!(Equip::shard(&input), "h9"); + assert_eq!( + ::FIELD_NAMES, + ["hero_id", "slot", "item"] + ); +} + +// ---- golden: the generated registration equals the hand-written form ----- + +mod handwritten { + use super::*; + + pub async fn create( + ctx: &CausalCommandContext<'_, Hero>, + input: CreateHero, + ) -> Result>, HandlerError> { + let actor = ctx.user_id()?.to_owned(); + let mut hero = ctx.create(); + let payload = hero + .aggregate_mut() + .create(&actor, input) + .map_err(|e| HandlerError::Rejected(e.to_string()))?; + ctx.publish_events().commit(hero)?.eventual(payload) + } + + pub async fn equip( + ctx: &CausalCommandContext<'_, Hero>, + input: EquipInput, + ) -> Result>, HandlerError> { + let actor = ctx.user_id()?.to_owned(); + let mut hero = require_loaded(ctx.get(&input.hero_id).await?, input.hero_id.clone())?; + hero.aggregate_mut() + .equip(input.slot, &actor, &input.item) + .map_err(|e| HandlerError::Rejected(e.to_string()))?; + let payload = hero_payload(hero.aggregate()); + ctx.publish_events().commit(hero)?.eventual(payload) + } + + pub async fn nap( + ctx: &CausalCommandContext<'_, Hero>, + input: NapInput, + ) -> Result>, HandlerError> { + let actor = ctx.user_id()?.to_owned(); + let mut hero = match ctx.get(&actor).await? { + Some(hero) => hero, + None => ctx.create(), + }; + let receipt = hero + .aggregate_mut() + .nap(actor, input) + .map_err(|e| HandlerError::Rejected(e.to_string()))?; + ctx.publish_events().commit(hero)?.succeeded(receipt) + } + + pub fn install(routes: distributed::microsvc::Routes) -> distributed::microsvc::Routes + where + D: distributed::microsvc::CausalRouteDependencies + Send + Sync + 'static, + { + routes + .command_transition::>( + "hero.create", + ) + .roles(["player"]) + .handle(create) + .command_transition::>( + "hero.equip", + ) + .field_name("heroes_equip") + .roles(["player"]) + .guarded(signed_in, equip) + .command_transition::>("hero.nap") + .handle(nap) + } +} + +#[test] +fn generated_command_specs_equal_the_hand_written_registration() { + let generated = AggregateCell::::new("x") + .unwrap() + .mount(Create) + .mount(Equip) + .mount(Nap); + let handwritten = AggregateCell::::new("x") + .unwrap() + .mount(handwritten::install); + assert_eq!(generated.command_names(), handwritten.command_names()); + let specs = |routes: distributed::microsvc::Routes<_>| { + serde_json::to_value(routes.command_specs().unwrap()).unwrap() + }; + let repo = + || distributed::AggregateRepository::<_, Hero>::new(distributed::InMemoryRepository::new()); + let generated = specs( + distributed::microsvc::Routes::new() + .with_repo(repo()) + .mount(Create) + .mount(Equip) + .mount(Nap), + ); + let handwritten = specs(handwritten::install( + distributed::microsvc::Routes::new().with_repo(repo()), + )); + assert_eq!(generated, handwritten); + // The domain transition witness carries each command's outward events. + let text = generated.to_string(); + assert!(text.contains("hero.equipped"), "{text}"); +} + +#[cfg(feature = "graphql")] +#[test] +fn generated_graphql_sdl_and_client_manifest_equal_the_hand_written_form() { + use distributed::graphql::{ + build_surface, graphql_sdl_from_surface, surface_for_role, DistributedClientSurfaceExport, + SurfaceOptions, + }; + use distributed::microsvc::{Routes, Service}; + + let render = |generated: bool| { + let routes = Routes::new().with_repo(distributed::AggregateRepository::<_, Hero>::new( + distributed::InMemoryRepository::new(), + )); + let routes = if generated { + routes.mount(Create).mount(Equip).mount(Nap) + } else { + handwritten::install(routes) + }; + let service = Service::new().named("heroes").routes(routes); + let surface = build_surface(&[], &SurfaceOptions::postgres()) + .unwrap() + .with_service(&service) + .unwrap(); + let selected = + surface_for_role(&surface, "player", &std::collections::BTreeMap::new()).unwrap(); + let sdl = graphql_sdl_from_surface(&selected).unwrap(); + let manifest = DistributedClientSurfaceExport::from_selected("heroes", selected) + .unwrap() + .manifest() + .unwrap(); + (sdl, serde_json::to_value(manifest).unwrap()) + }; + let (generated_sdl, generated_manifest) = render(true); + let (handwritten_sdl, handwritten_manifest) = render(false); + assert!(generated_sdl.contains("heroes_equip"), "{generated_sdl}"); + assert_eq!(generated_sdl, handwritten_sdl); + assert_eq!(generated_manifest, handwritten_manifest); +} From 633d309640b080e3a7dc10343d6a1521afdf6e8a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 13:13:47 -0500 Subject: [PATCH 16/40] feat(game)!: run Mob Pit cells on celld, drop cell-sqlite Mob Pit's durable state moves to celld; the native `cell-sqlite` backend is removed (decision: celld for local dev and deployment, no native cell mode; the in-process reference host stays for unit tests). - examples/game/cells: workers-rs worker with `crawler/{principal}` (the #[command] RecordKill + CelldOutbox), `crawler-roster/all` (celld cannot enumerate cells) and `presence/{principal}` (lifecycle facts, stamp ordered). Its Queue consumer relays outbox messages to the zone server's secret-protected /internal/celld-queue/relay, which publishes them on the projection SQLite bus for mobpit-service. - Zone server: MobPitFactSink over CelldFactSink (by_principal for crawler.record_kill, lifecycle_route for presence, enlist before the first kill), RetryPolicy::until_settled; profile, crawlers and board rebuild read cells over the internal path. `--celld-url`/CELLD_URL and DISTRIBUTED_INTERNAL_SECRET are required. - game_demos::devcelld and client/tests/helpers/celld.mjs start a private `celld dev` per test (CELLD_BIN); run.sh starts celld for mobpit, leaderboard, sveltekit and smoke (0.6.2+ required, esbuild from the client). CI installs celld v0.6.2 and worker-build. BREAKING CHANGE: the `cell-sqlite` feature, CellSqlitePath, AggregateCell::open_sqlite*, CellStreamStore::open_sqlite* and the native SQL executor are gone. game_demos: CrawlerFactSink -> MobPitFactSink, WorldOptions::new takes CelldOptions, spawn_with builds DemoRoutes. [[tasks/gr-p32-mobpit-celld]] [[tasks/gr-p24-zone-celld-fact-routing]] --- .github/workflows/integration-game-demos.yaml | 18 +- Cargo.toml | 9 - build.rs | 2 +- distributed_game/README.md | 12 +- docs/coordination-primitives.md | 6 +- examples/game/.gitignore | 6 +- examples/game/README.md | 82 ++- examples/game/cells/Cargo.toml | 30 + examples/game/cells/src/lib.rs | 633 ++++++++++++++++++ examples/game/cells/wrangler.jsonc | 37 + examples/game/client/tests/helpers/celld.mjs | 111 +++ .../game/client/tests/helpers/demo-server.mjs | 7 +- examples/game/client/tests/helpers/mobpit.mjs | 40 +- .../tests/leaderboard-headless.test.mjs | 8 +- .../client/tests/mobpit-headless.test.mjs | 20 +- examples/game/run.sh | 121 +++- examples/game/server/Cargo.lock | 2 +- examples/game/server/Cargo.toml | 11 +- examples/game/server/crawler/src/lib.rs | 26 +- .../game/server/service/tests/projection.rs | 43 +- examples/game/server/src/common/mod.rs | 35 +- examples/game/server/src/devcelld.rs | 272 ++++++++ examples/game/server/src/lib.rs | 1 + examples/game/server/src/mobpit/board.rs | 33 +- examples/game/server/src/mobpit/mod.rs | 67 +- examples/game/server/src/mobpit/routes.rs | 54 +- examples/game/server/src/mobpit/store.rs | 495 +++++++------- examples/game/server/src/mobpit/world.rs | 144 ++-- examples/game/server/src/mobpit/zone.rs | 7 +- examples/game/server/tests/mobpit_smoke.rs | 119 +++- examples/game/sveltekit/tests/pit.test.mjs | 24 +- src/microsvc/cell_host/alarm.rs | 1 + src/microsvc/cell_host/cell.rs | 29 +- src/microsvc/cell_host/mod.rs | 29 +- src/microsvc/cell_host/native_sql_executor.rs | 294 -------- src/microsvc/cell_host/native_sqlite.rs | 362 ---------- src/microsvc/cell_host/sql_store.rs | 144 +--- src/microsvc/cell_host/store.rs | 129 +--- src/microsvc/cell_host/tests.rs | 2 - src/microsvc/cell_host/tests/alarms.rs | 57 -- src/microsvc/cell_host/tests/native_sqlite.rs | 471 ------------- src/repository/migrations.rs | 5 +- src/repository/mod.rs | 3 +- tests/celld/README.md | 37 +- 44 files changed, 2022 insertions(+), 2016 deletions(-) create mode 100644 examples/game/cells/Cargo.toml create mode 100644 examples/game/cells/src/lib.rs create mode 100644 examples/game/cells/wrangler.jsonc create mode 100644 examples/game/client/tests/helpers/celld.mjs create mode 100644 examples/game/server/src/devcelld.rs delete mode 100644 src/microsvc/cell_host/native_sql_executor.rs delete mode 100644 src/microsvc/cell_host/native_sqlite.rs delete mode 100644 src/microsvc/cell_host/tests/native_sqlite.rs diff --git a/.github/workflows/integration-game-demos.yaml b/.github/workflows/integration-game-demos.yaml index 7c9aec875..a94fbe118 100644 --- a/.github/workflows/integration-game-demos.yaml +++ b/.github/workflows/integration-game-demos.yaml @@ -7,7 +7,9 @@ name: Game demos smoke # headless tests, and the SvelteKit demo end to end: `distributed build # examples/game/server` (GraphQL client, svelte-check, vite build), then # Playwright against the production build, a real Mob Pit server and the real -# mobpit-service read model (SQLite only; no external services). +# mobpit-service read model. Mob Pit's durable cells run on celld: the tests +# start their own `celld dev` (CELLD_BIN, pinned below; no Docker) serving +# examples/game/cells, built with worker-build. on: workflow_call: @@ -17,6 +19,8 @@ jobs: runs-on: ubuntu-latest env: CARGO_TERM_COLOR: always + # Pinned celld for Mob Pit's cells (run.sh requires 0.6.2+). + CELLD_VERSION: v0.6.2 # The shared ticket rate limiter's Lua script and the leaderboard store # (conformance suite, projector, GraphQL boards) run against a real Redis. services: @@ -44,6 +48,18 @@ jobs: js/package-lock.json examples/game/client/package-lock.json examples/game/sveltekit/package-lock.json + - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + examples/game/server -> target + examples/game/cells -> target + shared-key: game-demos + - name: Install celld (pinned) and worker-build + run: | + curl -fsSL https://celld.dev/install.sh | CELLD_VERSION="$CELLD_VERSION" sh + echo "CELLD_BIN=$HOME/.local/bin/celld" >> "$GITHUB_ENV" + "$HOME/.local/bin/celld" --version + cargo install worker-build --locked - name: Install SvelteKit demo dependencies and Chromium working-directory: examples/game/sveltekit run: npm ci --no-audit --no-fund && npx playwright install --with-deps chromium diff --git a/Cargo.toml b/Cargo.toml index 1aeb3e497..9b341917f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,10 +62,6 @@ redis = ["dep:redis", "dep:tokio", "leaderboard-background"] # task) without Redis, e.g. over `InMemoryLeaderboard`. leaderboard-background = ["dep:tokio"] workers-rs = ["dep:worker"] -# Native durable aggregate cells: one synchronous SQLite file per cell, same -# SQL store and one-transaction-per-command contract as Durable Object cells. -# Separate from `sqlite` (async SQLx pool + tokio); no-op on wasm32. -cell-sqlite = ["dep:rusqlite"] otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing", "dep:tracing-opentelemetry"] graphql = ["dep:async-graphql", "dep:async-graphql-axum", "dep:hmac", "dep:jsonwebtoken", "http", "axum?/ws"] @@ -109,11 +105,6 @@ uuid = { version = "1", features = ["v7"] } url = { version = "2", optional = true } worker = { version = "0.8", features = ["queue"], optional = true } -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -# libsqlite3-sys 0.37 stays inside SQLx 0.9's accepted range, so `sqlite` and -# `cell-sqlite` link one bundled SQLite when enabled together. -rusqlite = { version = "0.39", features = ["bundled"], optional = true } - [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] js-sys = "0.3" uuid = { version = "1", features = ["v7", "js"] } diff --git a/build.rs b/build.rs index 56da375ba..c406446f3 100644 --- a/build.rs +++ b/build.rs @@ -162,7 +162,7 @@ fn emit_migration_inventory() { fn emit_dialect(generated: &mut String, name: &str, dialect: Dialect, inventory: &Inventory) { match dialect { - Dialect::Sqlite => generated.push_str("#[cfg(any(feature = \"sqlite\", all(feature = \"workers-rs\", target_arch = \"wasm32\"), all(feature = \"cell-sqlite\", not(target_arch = \"wasm32\"))))]\n"), + Dialect::Sqlite => generated.push_str("#[cfg(any(feature = \"sqlite\", all(feature = \"workers-rs\", target_arch = \"wasm32\")))]\n"), Dialect::Postgres => generated.push_str("#[cfg(feature = \"postgres\")]\n"), } generated.push_str("pub(crate) const "); diff --git a/distributed_game/README.md b/distributed_game/README.md index 10141d06e..efe528ce0 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -747,9 +747,11 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), fact with no principal; `ctx.fact(&input)` builds the `Fact` for a retry backlog. `Fact::new(key, command, json)` still works. - A background worker commits facts with retries. - `cell::CellFactSink` turns each one into a command on that player's - cell, with a command id derived from the fact key. A retry or a - duplicate is therefore a harmless replay, never a double reward. + `celld::CelldFactSink` sends each one to its celld cell (that player's + cell, a match cell, ...) as a command over the internal path, with a + command id derived from the fact key. A retry or a duplicate is + therefore a harmless replay, never a double reward. (`cell::CellFactSink` + does the same into an in-process reference cell, for unit tests.) - `on_fact_settled` reports the outcome on a later tick. - Keys must be unique across restarts; `ctx.fact_key` includes the activation's boot epoch for you (see [Durable facts](#durable-facts)). @@ -765,8 +767,8 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), new after a restart); they share the principal's commit lane (order kept with `max_concurrent_commits > 1`) and wait in a host backlog rather than being dropped when the queue is full. `lifecycle::Lifecycle::of(&fact)` - decodes them in a sink. Mob Pit projects them into a presence read model - (`/dev/mobpit/presence`), ordered by stamp. + decodes them in a sink. Mob Pit routes them to per-player celld presence + cells (`CelldFactSink::lifecycle_route`), ordered by stamp. - Every fact carries a `FactStamp`: zone, activation (boot epoch), tick, host wall time `emitted_at_ms` (`Clock::unix_ms`; `ManualClock` in tests), content version, config version (`HostConfig::config_version`, diff --git a/docs/coordination-primitives.md b/docs/coordination-primitives.md index c3d031f48..2c88d2fcf 100644 --- a/docs/coordination-primitives.md +++ b/docs/coordination-primitives.md @@ -90,8 +90,8 @@ drain. The Worker's `alarm()` handler calls `AggregateCell::alarm(&env)`, which fires due alarms, then drains the outbox and re-plans the wake. Commands that arm an alarm move the wake earlier right after they commit. -**Reference host.** The in-process host and native `cell-sqlite` cells have -no timer. Give the store a `ManualClock` +**Reference host.** The in-process host (unit tests; durable cells run on +celld) has no timer. Give the store a `ManualClock` (`CellStreamStore::with_clock`), advance it and call `AggregateCell::fire_due_alarms()`: the same ledger and settlement path runs deterministically. @@ -248,7 +248,7 @@ A test network delivers outbox rows whose `deliver_at` is due on a ```sh cargo test --features sqlite,postgres --test saga_transfer --test coordination_lease --test scheduled_outbox cargo test --features sqlite --lib -- outbox lease clock -cargo test --features cell-sqlite,workers-rs --lib cell_host +cargo test --features workers-rs --lib cell_host (cd tests/celld/alarm-worker && worker-build --release) && node tests/celld/alarms.mjs ``` diff --git a/examples/game/.gitignore b/examples/game/.gitignore index fdd3cf487..4688cb85f 100644 --- a/examples/game/.gitignore +++ b/examples/game/.gitignore @@ -1,2 +1,6 @@ -# Mob Pit / Leaderboard durable cells (run.sh default MOBPIT_DATA_DIR). +# Mob Pit / Leaderboard local files and celld dev state (run.sh default MOBPIT_DATA_DIR). /.data/ +# The Mob Pit cells worker build (worker-build) and celld state. +/cells/build/ +/cells/target/ +/cells/.celld/ diff --git a/examples/game/README.md b/examples/game/README.md index 8f73baae7..7744b7c4a 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -9,7 +9,7 @@ guests and lets any page change the network simulation. | Demo | Run | Mechanic it demonstrates | |---|---|---| | **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | -| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player native SQLite cells (`cell-sqlite`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence (`/dev/mobpit/presence`) projected from the runtime's player lifecycle facts | +| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player celld cells (`cells/`, `celld dev` started by `run.sh`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence cells fed by the runtime's player lifecycle facts (`/dev/mobpit/presence`) | | **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | | **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | | **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | @@ -225,9 +225,12 @@ examples/game/run.sh sveltekit # → http://localhost:5573/ — sign in with `sveltekit/` is Mob Pit as a SvelteKit app over two backends: the Mob Pit zone server (gameplay) and `mobpit-service` (the business-side read model). -The script starts `game-demos mobpit` (no `--dev`) on `GAME_DEMOS_PORT` -(default 2967 here; data in `.data/mobpit-sveltekit`) with a fresh ticket secret and -`--projection-bus sqlite:…/bus.db`, then runs `distributed dev server/`: the +The script starts `celld dev` with the Mob Pit cells (`cells/`; state in +`.data/mobpit-sveltekit/celld`), then `game-demos mobpit` (no `--dev`) on +`GAME_DEMOS_PORT` (default 2967 here; data in `.data/mobpit-sveltekit`) with +a fresh ticket secret and `--projection-bus sqlite:…/bus.db` (celld's outbox +Queue relays the Crawler domain events to it), then runs +`distributed dev server/`: the application lifecycle generates the `$distributed` GraphQL client, starts `mobpit-service` on `MOBPIT_SERVICE_PORT` (default `GAME_DEMOS_PORT + 2`) and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: @@ -436,17 +439,23 @@ after dying. Level `n` starts at `40·(n−1)²` XP. `emit_fact_for` only enqueues: the host binds the killer's **verified ticket principal**; the zone never awaits storage. Backpressure (256 unsettled facts) parks facts in a backlog retried next tick. -2. The zone's fact worker (`GameServer::with_fact_sink`) runs - `store::CrawlerFactSink`: optional artificial latency, then - `CellFactSink::commit` into the killer's own cell - (`$MOBPIT_DATA_DIR/Crawler%3A{principal}.sqlite3`, `AggregateCell::from_store` - over `CellStreamStore::open_sqlite`). The command id is +2. The zone's fact worker (`GameServer::with_fact_sink`, retrying until + settled) runs `store::MobPitFactSink`: optional artificial latency, then + `distributed_game::celld::CelldFactSink` posts the fact over the + internal path (`DISTRIBUTED_INTERNAL_SECRET`, principal and `FactStamp` + as trusted headers) to the killer's own celld cell, `crawler/{principal}` + (`cells/`, a workers-rs Durable Object per player). The command id is `sha256(service ‖ fact key)`, so the cell's command ledger turns any repeat - of the key into a replay of the stored receipt. -3. `crawler.rs`: `crawler.record_kill` takes the principal from the trusted - session (`x-user-id`) and **decides the XP from the tier**; the input has - no identity or XP field and unknown fields are rejected. Event, ledger - receipt and snapshot commit in one SQLite transaction (`synchronous=FULL`). + of the key into a replay of the stored receipt. Before a principal's first + kill the sink enlists it in the `crawler-roster/all` cell (celld cannot + enumerate cells; the roster is what the board rebuild and + `/dev/mobpit/crawlers` read). celld down: the fact stays queued in the + zone (backpressure at capacity), never in a local store. +3. `crawler/src/lib.rs`: `crawler.record_kill` (a `#[command]` method) takes + the principal from the trusted session (`x-user-id`) and **decides the XP + from the tier**; the input has no identity or XP field and unknown fields + are rejected. Event, ledger receipt, snapshot and outbox row commit in one + Durable Object SQLite transaction. 4. Tick N+k: `Zone::on_fact_settled` sends the killer `saved` (with `waitedMs`) and `profile`; the popup turns from "saving…" to "+XP saved ✓". The sink also hands the kill to the leaderboard @@ -454,10 +463,13 @@ after dying. Level `n` starts at `40·(n−1)²` XP. 5. The read model (`crawler/`, `service/`): `crawler.mob_killed` is a **domain event** — the same transaction writes the post-kill `CrawlerState` (principal, name, XP, kills, level) to the cell's outbox - (`repo.publish_events()`). With `--projection-bus sqlite:` the sink - drains that outbox onto a `SqliteBus` after the commit (and a sweep drains - every cell at startup and every 2 s, so a crash between commit and drain - or a bus outage only delays it). `mobpit-service` consumes the bus with a + (`repo.publish_events()`). celld drains the outbox to the `mobpit-outbox` + Queue; the cells worker's Queue consumer relays each message to the zone + server's `/internal/celld-queue/relay` (secret-authenticated), which + publishes it onto the `SqliteBus` at `--projection-bus` (default + `sqlite:{data-dir}/bus.db`). A crash between commit and drain, or a relay + outage, only delays it (the cell's alarm re-drains; the Queue retries). + `mobpit-service` consumes the bus with a modeled projector (`projection!` → `upsert_player_stats`), checkpointed and deduplicated in its own SQLite database, and serves `player_stats` over GraphQL with `@live`. The service has no commands: nothing on that @@ -484,13 +496,16 @@ examples/game/run.sh mobpit # restart (same .data/mobpit), The ticket secret is persisted in the data dir (`dev-ticket-secret`), so the page's guest token still maps to the same principal, and the profile comes -back from the cell. What survives: +back from the cell. `run.sh` keeps celld's state in `.data/mobpit/celld` +(`.celld/dev`), so celld itself can be killed and restarted too. What +survives: -- **Settled facts** (you saw "saved ✓"): durable. The commit was acknowledged - only after the SQLite transaction (event + ledger receipt) was fsynced. +- **Settled facts** (you saw "saved ✓"): durable. celld acknowledged the + commit only after the cell's SQLite transaction (event + ledger receipt) + was durable. - **Facts in flight** at the crash: each is either fully committed (the - transaction finished; you just never saw the settlement) or not at all - (rolled back with the WAL) — never half-applied, never twice. The zone's + transaction finished; you just never saw the settlement) or not at all — + never half-applied, never twice. The zone's in-memory state (mob HP, positions, the fact queue) dies with the process, so an uncommitted kill is lost like any other unsaved world state; there is no zone-side outbox (see `distributed_game` README, *Facts on dispose*). @@ -501,12 +516,21 @@ back from the cell. What survives: epoch (`Ctx::boot_epoch`, grows across restarts) because zone ticks and mob generations restart at 0. -`tests/mobpit_smoke.rs` proves it with the real binary: bots fight, storage is -made slow, the server gets `SIGKILL` with facts in flight, restarts on the -same directory; then `settled ≤ durable kills ≤ settled + in flight`, the -board (rebuilt at boot) equals the cells, and re-committing a pre-crash fact -(same key and input) against the files returns the original receipt with -`replayed: true` and unchanged kills. +`tests/mobpit_smoke.rs` proves it with the real binary on its own +`celld dev`: bots fight, storage is made slow, the server gets `SIGKILL` with +facts in flight, then celld is killed (`SIGKILL` of its process tree) and +restarted on its state, and the server restarts; then `settled ≤ durable +kills ≤ settled + in flight`, the board (rebuilt at boot) equals the cells, +and re-committing a pre-crash fact (same key and input) returns the original +receipt with `replayed: true` and unchanged kills. + +**celld.** `run.sh mobpit|leaderboard|sveltekit|smoke` need celld 0.6.2+ +(`CELLD_BIN`, default `celld` on PATH; install with +`curl -fsSL https://celld.dev/install.sh | CELLD_VERSION=v0.6.2 sh`) and +`worker-build` (`cargo install worker-build --locked`) for `cells/`. The +tests (`tests/mobpit_smoke.rs`, `service/tests/projection.rs`, the headless +and Playwright tests) each start their own `celld dev` on a temp project +(`game_demos::devcelld`, `client/tests/helpers/celld.mjs`). ## Leaderboard diff --git a/examples/game/cells/Cargo.toml b/examples/game/cells/Cargo.toml new file mode 100644 index 000000000..84c70ca40 --- /dev/null +++ b/examples/game/cells/Cargo.toml @@ -0,0 +1,30 @@ +# Mob Pit's durable cells on celld (workers-rs, wasm32): the per-player +# Crawler aggregate, the crawler roster, per-player presence, and the outbox +# Queue consumer that relays Crawler domain events to the zone server. +# Own workspace (cdylib for wasm32-unknown-unknown); built with +# `worker-build --release` by `examples/game/run.sh`. +[workspace] +members = ["."] +resolver = "2" + +[package] +name = "mobpit-cells" +version = "0.1.0" +edition = "2021" +publish = false +description = "Mob Pit Crawler, roster and presence cells for celld (dev demo)" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +console_error_panic_hook = "0.1" +distributed = { path = "../../..", default-features = false, features = ["workers-rs"] } +distributed_game = { path = "../../../distributed_game", default-features = false, features = ["cell"] } +mobpit_crawler = { path = "../server/crawler" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +worker = { version = "0.8", features = ["queue"] } + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs new file mode 100644 index 000000000..9a372491a --- /dev/null +++ b/examples/game/cells/src/lib.rs @@ -0,0 +1,633 @@ +//! Mob Pit's durable cells on celld, fed by the zone server's +//! `distributed_game::celld::CelldFactSink` over the internal HTTP path. +//! +//! - `crawler/{principal}` (`CrawlerCell`): the `mobpit_crawler::Crawler` +//! aggregate with its `#[command]` `crawler.record_kill` (XP decided here +//! from the mob tier; the principal comes from the trusted fact session). +//! Each commit writes the `crawler.mob_killed` domain event to the cell +//! outbox, which drains to the `mobpit-outbox` Queue. +//! - `crawler-roster/all` (`CrawlerRosterCell`): the principals that have a +//! Crawler cell. Durable Objects cannot be enumerated, so the zone server +//! enlists a principal (`crawler.enlist`, idempotent by principal) before +//! its first kill commits; the board rebuild and `/dev/mobpit/crawlers` +//! read it. +//! - `presence/{principal}` (`PresenceCell`): the runtime's player lifecycle +//! facts (`zone.*`). A fact emitted before the one that set the current +//! entry (`FactStamp::emission_order`) is ignored, so a late "left pit-1" +//! cannot mark a player who already joined pit-2 offline. +//! +//! Every route requires `DISTRIBUTED_INTERNAL_SECRET`; commands also require +//! the trusted fact session (principal + `FactStamp` headers, rebuilt with +//! `fact_session_from_headers`), so nothing but a zone server's fact worker +//! can run them. +//! +//! The Queue consumer relays each outbox message to `CELLD_QUEUE_RELAY_URL` +//! (the zone server's authenticated `/internal/celld-queue/relay`), which +//! publishes it onto the projection bus read by `mobpit-service`. Delivery +//! is at-least-once with stable message ids; the projector deduplicates. + +use std::collections::BTreeSet; + +use distributed::bus::{CelldQueueEnvelope, CelldQueueHttpPublisher, CelldQueueRelay}; +use distributed::cell_host::{ + AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, + InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, +}; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, +}; +use distributed::{sourced, Aggregate, Entity}; +use distributed_game::cell::{fact_session_from_headers, session_stamp}; +use distributed_game::lifecycle::{ + Lifecycle, CONNECTION_LOST, PLAYER_DISCONNECTED, PLAYER_JOINED, PLAYER_LEFT, PLAYER_RECONNECTED, +}; +use distributed_game::FactStamp; +use mobpit_crawler::{ + Crawler, MobKilledArgs, RecordKill, CRAWLER_KIND, ENLIST, MOB_KILLED_EVENT, PRESENCE_KIND, + ROSTER_KIND, ROSTER_SHARD, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use worker::*; + +fn rejected(error: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(error.to_string()) +} + +// ---------------------------------------------------------------- Roster + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct CrawlerRoster { + entity: Entity, + principals: BTreeSet, +} + +#[sourced( + entity, + aggregate_type = "CrawlerRoster", + events = "CrawlerRosterEvent" +)] +impl CrawlerRoster { + #[event("crawler_roster.enlisted", version = 1)] + fn enlisted(&mut self, roster: String, principal: String) { + if self.entity.id().is_empty() { + self.entity.set_id(roster); + } + self.principals.insert(principal); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct EnlistInput {} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct Enlisted { + pub crawlers: u32, +} + +struct Enlist; +impl PortableCommand for Enlist +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(ENLIST)) + .handle(handle_enlist) + } +} + +async fn handle_enlist( + ctx: &CausalCommandContext<'_, CrawlerRoster>, + _input: EnlistInput, +) -> std::result::Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + let repo = ctx.repo(); + let mut roster = match repo.get(ROSTER_SHARD).await? { + Some(roster) => roster, + None => repo.create(), + }; + if !roster.principals.contains(&principal) { + roster + .enlisted(ROSTER_SHARD.to_string(), principal) + .map_err(rejected)?; + } + let crawlers = roster.principals.len() as u32; + repo.commit(roster)?.succeeded(Enlisted { crawlers }) +} + +// ---------------------------------------------------------------- Presence + +/// The ordering key of the fact that set a presence entry. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StampKey { + pub zone_id: String, + pub activation: u64, + pub tick: u64, + pub emitted_at_ms: u64, + pub seq: u64, +} + +impl StampKey { + fn of(stamp: &FactStamp) -> Self { + Self { + zone_id: stamp.zone_id.clone(), + activation: stamp.activation, + tick: stamp.tick, + emitted_at_ms: stamp.emitted_at_ms, + seq: stamp.seq, + } + } + + fn stamp(&self) -> FactStamp { + FactStamp { + zone_id: self.zone_id.clone(), + activation: self.activation, + tick: self.tick, + emitted_at_ms: self.emitted_at_ms, + seq: self.seq, + ..FactStamp::default() + } + } +} + +/// One presence transition. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PresenceChange { + pub zone: String, + /// `online`, `away` or `offline`. + pub status: String, + pub since_ms: u64, + pub stamp: StampKey, +} + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct PlayerPresence { + entity: Entity, + current: Option, +} + +#[sourced( + entity, + aggregate_type = "PlayerPresence", + events = "PlayerPresenceEvent" +)] +impl PlayerPresence { + #[event("presence.changed", version = 1)] + fn changed(&mut self, principal: String, change: PresenceChange) { + if self.entity.id().is_empty() { + self.entity.set_id(principal); + } + self.current = Some(change); + } +} + +/// Any lifecycle fact input (the union of `distributed_game::lifecycle`'s +/// inputs; decoded per command with `Lifecycle::decode`). +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct LifecycleInput { + pub zone: String, + pub via: Option, + pub from_zone: Option, + pub reason: Option, + pub grace_ms: Option, + pub cause: Option, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct PresenceApplied { + /// False when a later-emitted fact already set the entry. + pub applied: bool, +} + +/// One mount per lifecycle command (handlers are plain `async fn`s). +macro_rules! lifecycle_command { + ($mount:ident, $handler:ident, $command:expr) => { + struct $mount; + impl PortableCommand for $mount + where + D: CausalRouteDependencies + Send + Sync + 'static, + { + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + $command, + )) + .handle($handler) + } + } + + async fn $handler( + ctx: &CausalCommandContext<'_, PlayerPresence>, + input: LifecycleInput, + ) -> std::result::Result>, HandlerError> { + handle_lifecycle(ctx, $command, input).await + } + }; +} + +lifecycle_command!(Joined, handle_joined, PLAYER_JOINED); +lifecycle_command!(Lost, handle_lost, CONNECTION_LOST); +lifecycle_command!(Reconnected, handle_reconnected, PLAYER_RECONNECTED); +lifecycle_command!(Disconnected, handle_disconnected, PLAYER_DISCONNECTED); +lifecycle_command!(Left, handle_left, PLAYER_LEFT); + +async fn handle_lifecycle( + ctx: &CausalCommandContext<'_, PlayerPresence>, + command: &'static str, + input: LifecycleInput, +) -> std::result::Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + let stamp = session_stamp(ctx.session()) + .ok_or_else(|| rejected("lifecycle facts need a fact stamp"))?; + let event = Lifecycle::decode(command, &serde_json::to_value(&input).map_err(rejected)?) + .ok_or_else(|| rejected(format!("malformed `{command}` input")))?; + let status = match event { + Lifecycle::Joined(_) | Lifecycle::Reconnected(_) => "online", + Lifecycle::ConnectionLost(_) => "away", + Lifecycle::Left(_) | Lifecycle::Disconnected(_) => "offline", + }; + let repo = ctx.repo(); + let mut presence = match repo.get(&principal).await? { + Some(presence) => presence, + None => repo.create(), + }; + let newer = presence.current.as_ref().is_none_or(|current| { + current.stamp.stamp().emission_order(&stamp) == std::cmp::Ordering::Less + }); + if newer { + presence + .changed( + principal, + PresenceChange { + zone: event.zone().to_string(), + status: status.to_string(), + since_ms: stamp.emitted_at_ms, + stamp: StampKey::of(&stamp), + }, + ) + .map_err(rejected)?; + } + repo.commit(presence)? + .succeeded(PresenceApplied { applied: newer }) +} + +// ---------------------------------------------------------------- Hosting + +async fn dispatch( + env: &Env, + cell: &AggregateCell, + command: &str, + req: &mut Request, +) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => return error_response(CellDispatchError::BadRequest("invalid JSON".into())), + }; + let parsed = match CellWaitPathRequest::parse(body) { + Ok(parsed) => parsed, + Err(error) => return error_response(CellDispatchError::BadRequest(error)), + }; + let header = |name: &str| req.headers().get(name).ok().flatten(); + let identity = match ( + header(CELL_SERVICE_ID_HEADER), + header(CELL_PRINCIPAL_PARTITION_HEADER), + ) { + (Some(service), Some(partition)) => { + match CellCommandIdentity::new(service, partition, &parsed.command_id) { + Ok(identity) => identity, + Err(error) => return error_response(error), + } + } + _ => return error_response(CellDispatchError::Unauthorized), + }; + // Only facts run here: the principal and the stamp, from the trusted + // headers of the authenticated internal path. + let Some(session) = fact_session_from_headers(header) else { + return error_response(CellDispatchError::Forbidden); + }; + let result = cell + .dispatch_idempotent(command, &identity, parsed.input, session) + .await; + if let Err(error) = cell.drain_outbox(env).await { + worker::console_error!("post-commit drain deferred: {error}"); + } + match result { + Ok(dispatch) => json_status( + json!({ + "payload": dispatch.payload(), + "receipt": { "commandId": dispatch.command_id(), "replayed": dispatch.replayed() }, + }), + 200, + ), + Err(error) => error_response(error), + } +} + +/// `GET /{kind}/{shard}` → `read(cell)`; `POST /{kind}/{shard}/{command}`. +async fn route( + env: &Env, + cell: &AggregateCell, + mut req: Request, + read: F, +) -> Result +where + A: Aggregate + Send + Sync + 'static, + F: AsyncFnOnce(&AggregateCell) -> std::result::Result, +{ + if let Err(error) = authenticate(&req, env) { + return error_response(error); + } + let url = req.url()?; + let parts: Vec = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .map(str::to_string) + .collect(); + match (req.method(), parts.get(2)) { + (Method::Get, None) => match read(cell).await { + Ok(body) => json_status(body, 200), + Err(error) => error_response(CellDispatchError::Internal(error)), + }, + (Method::Post, Some(command)) => { + let command = command.clone(); + dispatch(env, cell, &command, &mut req).await + } + _ => json_status(json!({ "error": "not found" }), 404), + } +} + +async fn read_crawler(cell: &AggregateCell) -> std::result::Result { + let state = cell.load().await.map_err(|error| error.to_string())?; + let mut kills = Vec::new(); + for record in cell.history().await.map_err(|error| error.to_string())? { + if record.event_name != MOB_KILLED_EVENT { + continue; + } + let (crawler, name, mob, tier, xp): MobKilledArgs = + record.decode().map_err(|error| error.to_string())?; + let at_ms = record + .timestamp + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis() as u64); + kills.push(json!({ + "sequence": record.sequence, "atMs": at_ms, "crawler": crawler, + "name": name, "mob": mob, "tier": tier, "xp": xp, + })); + } + Ok(json!({ + "state": state.map(|c| json!({ + "name": c.name, "xp": c.xp, "kills": c.kills, "level": c.level, + "version": c.version(), + })), + "kills": kills, + })) +} + +async fn read_roster(cell: &AggregateCell) -> std::result::Result { + let roster = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ + "principals": roster.map(|r| r.principals.into_iter().collect::>()).unwrap_or_default(), + })) +} + +async fn read_presence(cell: &AggregateCell) -> std::result::Result { + let presence = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ "current": presence.and_then(|p| p.current) })) +} + +fn outbox(env: &Env) -> CelldOutbox { + CelldOutbox::from_env(env, "OUTBOX").expect("OUTBOX Queue binding") +} + +#[durable_object] +pub struct CrawlerCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for CrawlerCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 100) + .expect("crawler cell identity") + .mount(RecordKill) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_crawler).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +#[durable_object] +pub struct CrawlerRosterCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for CrawlerRosterCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 50) + .expect("roster cell identity") + .mount(Enlist) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_roster).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +#[durable_object] +pub struct PresenceCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for PresenceCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 50) + .expect("presence cell identity") + .mount(Joined) + .mount(Lost) + .mount(Reconnected) + .mount(Disconnected) + .mount(Left) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_presence).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +async fn alarm(env: &Env, cell: &AggregateCell) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let wake = cell + .alarm(env) + .await + .map_err(|error| Error::RustError(error.to_string()))?; + for error in &wake.drain.deferred { + worker::console_error!("drain deferred: {error}"); + } + Response::ok("ok") +} + +#[event(fetch)] +async fn main(req: Request, env: Env, _ctx: Context) -> Result { + console_error_panic_hook::set_once(); + let url = req.url()?; + let path = url.path(); + if path == "/" || path == "/health" { + return Response::ok("mobpit cells\n"); + } + if let Err(error) = authenticate(&req, &env) { + return error_response(error); + } + let parts: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + let binding = match parts.first().copied() { + Some(CRAWLER_KIND) => "CRAWLER", + Some(ROSTER_KIND) => "CRAWLER_ROSTER", + Some(PRESENCE_KIND) => "PRESENCE", + _ => return json_status(json!({ "error": "unknown cell kind" }), 404), + }; + let Some(shard) = parts.get(1) else { + return json_status(json!({ "error": "missing shard" }), 400); + }; + let shard = percent_decode(shard).ok_or_else(|| Error::RustError("bad shard".into()))?; + if binding == "CRAWLER_ROSTER" && shard != ROSTER_SHARD { + return json_status(json!({ "error": "unknown roster" }), 404); + } + let stub = env + .durable_object(binding)? + .id_from_name(&shard)? + .get_stub()?; + stub.fetch_with_request(req).await +} + +/// Outbox Queue consumer: relay each canonical message to the zone server. +#[event(queue)] +pub async fn queue(batch: MessageBatch, env: Env, _ctx: Context) -> Result<()> { + console_error_panic_hook::set_once(); + let relay = match relay(&env) { + Ok(relay) => relay, + Err(error) => { + // Keep the messages: they are retried until a relay is configured. + worker::console_error!("mobpit outbox relay unavailable: {error}"); + batch.retry_all(); + return Ok(()); + } + }; + for delivery in batch.messages()? { + match relay.relay(delivery.body().clone()).await { + Ok(()) => delivery.ack(), + Err(error) => { + worker::console_error!( + "mobpit outbox relay failed for {}: {}", + delivery.id(), + error + ); + delivery.retry(); + } + } + } + Ok(()) +} + +fn relay(env: &Env) -> std::result::Result, String> { + let var = |name: &str| { + env.var(name) + .map(|value| value.to_string()) + .or_else(|_| env.secret(name).map(|value| value.to_string())) + .map_err(|_| format!("{name} is required")) + }; + let endpoint = var("CELLD_QUEUE_RELAY_URL")?; + let secret = var(CELL_INTERNAL_SECRET_ENV)?; + // `http://127.0.0.1` relays are dev-only; anything else must be HTTPS. + let local = + endpoint.starts_with("http://127.0.0.1") || endpoint.starts_with("http://localhost"); + let publisher = if local { + CelldQueueHttpPublisher::new_local_test(endpoint, &secret) + } else { + CelldQueueHttpPublisher::new(endpoint) + } + .map_err(|error| error.to_string())? + .with_header(CELL_INTERNAL_SECRET_HEADER, secret); + Ok(CelldQueueRelay::new(publisher)) +} + +fn percent_decode(segment: &str) -> Option { + let bytes = segment.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + out.push(u8::from_str_radix(segment.get(i + 1..i + 3)?, 16).ok()?); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).ok() +} + +fn authenticate(req: &Request, env: &Env) -> std::result::Result<(), CellDispatchError> { + let configured = env + .secret(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + .or_else(|_| { + env.var(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + }) + .map_err(|_| CellDispatchError::Internal("internal secret is required".into()))?; + let secret = InternalHttpSecret::new(configured).map_err(CellDispatchError::Internal)?; + let candidate = req + .headers() + .get(CELL_INTERNAL_SECRET_HEADER) + .ok() + .flatten() + .ok_or(CellDispatchError::Unauthorized)?; + if secret.matches(&candidate) { + Ok(()) + } else { + Err(CellDispatchError::Unauthorized) + } +} + +fn error_response(error: CellDispatchError) -> Result { + json_status( + json!({ "error": error.client_message(), "code": error.code() }), + error.status_code(), + ) +} + +fn json_status(body: Value, status: u16) -> Result { + Ok(Response::from_json(&body)?.with_status(status)) +} diff --git a/examples/game/cells/wrangler.jsonc b/examples/game/cells/wrangler.jsonc new file mode 100644 index 000000000..680324798 --- /dev/null +++ b/examples/game/cells/wrangler.jsonc @@ -0,0 +1,37 @@ +// Mob Pit cells for `celld dev` (and `celld deploy`). The launcher +// (`examples/game/run.sh`, the test harnesses) writes `.dev.vars` beside a +// copy of this file with DISTRIBUTED_INTERNAL_SECRET and +// CELLD_QUEUE_RELAY_URL (the zone server's `/internal/celld-queue/relay`). +{ + "name": "mobpit-cells", + "main": "build/worker/shim.mjs", + "compatibility_date": "2026-01-01", + "durable_objects": { + "bindings": [ + { "name": "CRAWLER", "class_name": "CrawlerCell" }, + { "name": "CRAWLER_ROSTER", "class_name": "CrawlerRosterCell" }, + { "name": "PRESENCE", "class_name": "PresenceCell" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["CrawlerCell", "CrawlerRosterCell", "PresenceCell"] } + ], + "queues": { + "producers": [ + { "binding": "OUTBOX", "queue": "mobpit-outbox" } + ], + "consumers": [ + { + "queue": "mobpit-outbox", + "max_batch_size": 10, + "max_batch_timeout": 1, + "max_retries": 100, + "dead_letter_queue": "mobpit-outbox-dlq", + "retry_delay": 2 + } + ] + }, + "vars": { + "OUTBOX_DRAIN_INTERVAL_MS": "5000" + } +} diff --git a/examples/game/client/tests/helpers/celld.mjs b/examples/game/client/tests/helpers/celld.mjs new file mode 100644 index 000000000..ac3442345 --- /dev/null +++ b/examples/game/client/tests/helpers/celld.mjs @@ -0,0 +1,111 @@ +/** + * `celld dev` serving Mob Pit's cells (`examples/game/cells`) for one Node + * test: its own temp project, port and internal secret; `stop()` kills the + * process tree this helper started (nothing else). + * + * celld comes from CELLD_BIN (examples/game/run.sh sets it; there is no + * native cell mode to fall back to). The worker must be built + * (`worker-build --release` in examples/game/cells; run.sh does it), and + * CELLD_ESBUILD is passed through for celld's bundler. + */ +import { spawn, execFileSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { cpSync, existsSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const CELLS = fileURLToPath(new URL('../../../cells/', import.meta.url)); +const RELAY_PATH = '/internal/celld-queue/relay'; + +/** A free local port (bound, then released). */ +export function freePort() { + return new Promise((resolve, reject) => { + const server = createServer(); + server.unref(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); +} + +/** + * Start celld. `relayPort`: the Mob Pit server port its outbox Queue relays + * domain events to (`/internal/celld-queue/relay`). + */ +export async function startCelld({ relayPort } = {}) { + const bin = process.env.CELLD_BIN; + if (!bin) throw new Error('CELLD_BIN is not set: Mob Pit\'s cells run on celld (0.6.2+); point CELLD_BIN at the celld binary'); + if (!existsSync(join(CELLS, 'build/worker/shim.mjs'))) { + throw new Error(`the Mob Pit cells worker is not built: run \`worker-build --release\` in ${CELLS}`); + } + const project = mkdtempSync(join(tmpdir(), 'mobpit-celld-js-')); + cpSync(join(CELLS, 'build'), join(project, 'build'), { recursive: true }); + cpSync(join(CELLS, 'wrangler.jsonc'), join(project, 'wrangler.jsonc')); + const secret = `test-only-${randomBytes(24).toString('hex')}`; + let vars = `DISTRIBUTED_INTERNAL_SECRET=${secret}\n`; + if (relayPort) vars += `CELLD_QUEUE_RELAY_URL=http://127.0.0.1:${relayPort}${RELAY_PATH}\n`; + writeFileSync(join(project, '.dev.vars'), vars); + const port = await freePort(); + const child = spawn(bin, ['dev', project, '--host', '127.0.0.1', '--port', String(port), '--no-watch', '--logs'], { + env: { ...process.env, RUST_LOG: 'warn' }, + stdio: ['ignore', 'pipe', 'pipe'] + }); + let output = ''; + child.stdout.on('data', (chunk) => (output += chunk)); + child.stderr.on('data', (chunk) => (output += chunk)); + const url = `http://127.0.0.1:${port}`; + const deadline = Date.now() + 90_000; + for (;;) { + if (child.exitCode !== null) throw new Error(`celld dev exited with ${child.exitCode}:\n${output}`); + try { + if ((await fetch(`${url}/health`)).ok) break; + } catch { + // not listening yet + } + if (Date.now() > deadline) throw new Error(`celld dev did not become ready:\n${output}`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return { + url, + secret, + project, + /** Env for `game-demos mobpit`. */ + env: { CELLD_URL: url, DISTRIBUTED_INTERNAL_SECRET: secret }, + output: () => output, + async stop() { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((resolve) => child.on('exit', resolve)); + killTree(child.pid); + await exited; + } + }; +} + +/** SIGKILL `root` and every descendant (`celld dev` starts its node in a + * session of its own, so the tree is collected while the root is stopped). */ +function killTree(root) { + try { + process.kill(root, 'SIGSTOP'); + } catch { + return; + } + const pairs = execFileSync('ps', ['-axo', 'pid=,ppid='], { encoding: 'utf8' }) + .trim() + .split('\n') + .map((line) => line.trim().split(/\s+/).map(Number)); + const doomed = [root]; + for (let i = 0; i < doomed.length; i++) { + for (const [pid, ppid] of pairs) if (ppid === doomed[i]) doomed.push(pid); + } + for (const pid of doomed.reverse()) { + try { + process.kill(pid, 'SIGKILL'); + } catch { + // already gone + } + } +} diff --git a/examples/game/client/tests/helpers/demo-server.mjs b/examples/game/client/tests/helpers/demo-server.mjs index 051e51897..5f95bc205 100644 --- a/examples/game/client/tests/helpers/demo-server.mjs +++ b/examples/game/client/tests/helpers/demo-server.mjs @@ -1,5 +1,6 @@ /** - * Start a `game-demos ` server for a Node test (port 0, dev routes on) + * Start a `game-demos ` server for a Node test (port 0 unless given, + * dev routes on) * and stop it afterwards. Uses GAME_DEMOS_BIN when set (run.sh builds it), * else `cargo run` in ../server (compiles on first use). */ @@ -8,9 +9,9 @@ import { fileURLToPath } from 'node:url'; const SERVER_DIR = fileURLToPath(new URL('../../../server/', import.meta.url)); -export async function startDemo(demo, { env = {} } = {}) { +export async function startDemo(demo, { env = {}, port = 0 } = {}) { const bin = process.env.GAME_DEMOS_BIN; - const args = [demo, '--port', '0', '--dev']; + const args = [demo, '--port', String(port), '--dev']; const child = bin ? spawn(bin, args, { env: { ...process.env, ...env }, stdio: ['ignore', 'pipe', 'inherit'] }) : spawn('cargo', ['run', '-q', '--', ...args], { diff --git a/examples/game/client/tests/helpers/mobpit.mjs b/examples/game/client/tests/helpers/mobpit.mjs index c930d07d8..cac3d558e 100644 --- a/examples/game/client/tests/helpers/mobpit.mjs +++ b/examples/game/client/tests/helpers/mobpit.mjs @@ -1,18 +1,52 @@ /** - * Headless Mob Pit helpers: a throwaway data dir and a scripted hunter that - * walks to the nearest mob, faces it and swings until `done()` is true. + * Headless Mob Pit helpers: a server on its own `celld dev` (the Crawler, + * roster and presence cells), a throwaway data dir and a scripted hunter + * that walks to the nearest mob, faces it and swings until `done()` is true. */ import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { BUTTON, isMob, isPlayer, mobFields, playerFields } from '../../src/generated/mobpit/index.ts'; -import { sleep } from './demo-server.mjs'; +import { freePort, startCelld } from './celld.mjs'; +import { sleep, startDemo } from './demo-server.mjs'; export function tempDataDir() { return mkdtempSync(join(tmpdir(), 'mobpit-js-')); } +/** + * celld + `game-demos ` (mobpit / leaderboard) on a fixed port that + * celld's outbox relay targets. `restart()` restarts only the game server + * (same port, data dir and cells); `stop()` stops both. + */ +export async function startPit(demo = 'mobpit') { + const port = await freePort(); + const celld = await startCelld({ relayPort: port }); + const env = { ...celld.env, MOBPIT_DATA_DIR: tempDataDir() }; + let server; + try { + server = await startDemo(demo, { env, port }); + } catch (error) { + await celld.stop(); + throw error; + } + return { + celld, + get httpUrl() { + return server.httpUrl; + }, + async restart() { + await server.stop(); + server = await startDemo(demo, { env, port }); + }, + async stop() { + await server.stop(); + await celld.stop(); + } + }; +} + /** Hunt mobs (one command frame per 30 Hz tick) until `done()` or timeout. */ export async function hunt(pit, done, timeoutMs = 45_000) { const { client } = pit; diff --git a/examples/game/client/tests/leaderboard-headless.test.mjs b/examples/game/client/tests/leaderboard-headless.test.mjs index d6510a5d5..68f248ae0 100644 --- a/examples/game/client/tests/leaderboard-headless.test.mjs +++ b/examples/game/client/tests/leaderboard-headless.test.mjs @@ -1,6 +1,6 @@ /** * Headless Leaderboard: a hunter scores kills in the pit, an observer - * receives pushed board snapshots whose scores equal the Crawler cells; + * receives pushed board snapshots whose scores equal the celld Crawler cells; * rebuilding from the cells' events yields the identical board. */ import assert from 'node:assert/strict'; @@ -8,11 +8,11 @@ import test from 'node:test'; import { createBoardWatcher, createMobPitClient, MobPitApi } from '../src/mobpit/client.ts'; import { DevApi } from '../src/shared/dev-api.ts'; -import { memoryStorage, startDemo, until } from './helpers/demo-server.mjs'; -import { hunt, tempDataDir } from './helpers/mobpit.mjs'; +import { memoryStorage, until } from './helpers/demo-server.mjs'; +import { hunt, startPit } from './helpers/mobpit.mjs'; test('board pushes match the cells and rebuild is identical', { timeout: 600_000 }, async () => { - const server = await startDemo('leaderboard', { env: { MOBPIT_DATA_DIR: tempDataDir() } }); + const server = await startPit('leaderboard'); const clients = []; try { const storage = memoryStorage(); diff --git a/examples/game/client/tests/mobpit-headless.test.mjs b/examples/game/client/tests/mobpit-headless.test.mjs index 3257296bd..231d3d189 100644 --- a/examples/game/client/tests/mobpit-headless.test.mjs +++ b/examples/game/client/tests/mobpit-headless.test.mjs @@ -1,8 +1,9 @@ /** - * Headless Mob Pit (Node, real `game-demos mobpit` binary): join → profile - * read from the Crawler cell → kill a mob → the fact settles ("saved") with - * server-decided XP → slow storage makes the settlement wait while the tick - * stays flat → restart the server on the same data dir → XP is still there. + * Headless Mob Pit (Node, real `game-demos mobpit` binary on its own + * `celld dev`): join → profile read from the celld Crawler cell → kill a mob + * → the fact settles ("saved") with server-decided XP → slow storage makes + * the settlement wait while the tick stays flat → restart the server → XP is + * still there. */ import assert from 'node:assert/strict'; import test from 'node:test'; @@ -10,16 +11,14 @@ import test from 'node:test'; import { isPit, pitFields } from '../src/generated/mobpit/index.ts'; import { createMobPitClient, MobPitApi } from '../src/mobpit/client.ts'; import { DevApi } from '../src/shared/dev-api.ts'; -import { memoryStorage, startDemo, until } from './helpers/demo-server.mjs'; -import { hunt, tempDataDir } from './helpers/mobpit.mjs'; +import { memoryStorage, until } from './helpers/demo-server.mjs'; +import { hunt, startPit } from './helpers/mobpit.mjs'; const XP = new Set([10, 25, 60]); test('kills become durable facts without stalling the tick, and survive a restart', { timeout: 600_000 }, async () => { - const dataDir = tempDataDir(); - const env = { MOBPIT_DATA_DIR: dataDir }; const storage = memoryStorage(); - let server = await startDemo('mobpit', { env }); + const server = await startPit('mobpit'); const clients = []; try { const api = new DevApi({ base: server.httpUrl, name: () => 'Hunter', storage }); @@ -77,8 +76,7 @@ test('kills become durable facts without stalling the tick, and survive a restar // the guest token stays valid (secret persisted in the data dir) and // the profile is read back from the cell. pit.client.close(); - await server.stop(); - server = await startDemo('mobpit', { env }); + await server.restart(); const again = createMobPitClient({ api: new DevApi({ base: server.httpUrl, storage }), base: server.httpUrl }); clients.push(again.client); await again.client.connect('pit-1'); diff --git a/examples/game/run.sh b/examples/game/run.sh index c3fbd8c0c..a8710be99 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -26,10 +26,17 @@ # Env: GAME_DEMOS_PORT (server, default 2567), GAME_DEMOS_CLIENT_PORT (Vite, # default 5173), DG_NET_LATENCY_MS / DG_NET_JITTER_MS / DG_NET_LOSS (initial # server NetSim; the HUD sliders change it live), MOBPIT_DATA_DIR (Mob Pit / -# Leaderboard cells, default examples/game/.data/mobpit), REDIS_URL (Redis -# leaderboard instead of in-memory), SVELTEKIT_PORT (SvelteKit demo, default -# 5573; its game server uses GAME_DEMOS_PORT, default 2967 there, and its -# read-model service MOBPIT_SERVICE_PORT, default GAME_DEMOS_PORT + 2). +# Leaderboard local files and their `celld dev` state, default +# examples/game/.data/mobpit), REDIS_URL (Redis leaderboard instead of +# in-memory), SVELTEKIT_PORT (SvelteKit demo, default 5573; its game server +# uses GAME_DEMOS_PORT, default 2967 there, and its read-model service +# MOBPIT_SERVICE_PORT, default GAME_DEMOS_PORT + 2). +# +# celld: Mob Pit's durable cells (examples/game/cells) run on celld, never +# natively. CELLD_BIN names the binary (default: `celld` on PATH; 0.6.2 or +# newer is required), CELLD_PORT its port (default GAME_DEMOS_PORT + 3), +# CELLD_ESBUILD its bundler (default: the client's esbuild). The worker is +# built with worker-build (`cargo install worker-build --locked`). set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -39,7 +46,11 @@ CLIENT="$HERE/client" SVELTEKIT="$HERE/sveltekit" BIN="$SERVER/target/debug/game-demos" SERVICE_BIN="$SERVER/target/debug/mobpit-service" +CELLS="$HERE/cells" DEMOS=(arena mobpit leaderboard gallery freeze grid ripple) +# Demos whose durable state lives in celld cells. +CELLD_DEMOS=(mobpit leaderboard) +CELLD_MIN_VERSION="0.6.2" export MOBPIT_DATA_DIR="${MOBPIT_DATA_DIR:-$HERE/.data/mobpit}" # Multi-host demos: `game-demos --host-id A|B --peer …`. CLUSTER_DEMOS=(grid ripple) @@ -79,6 +90,92 @@ ensure_sveltekit() { fi } +# ensure_celld: resolve CELLD_BIN (>= CELLD_MIN_VERSION) and CELLD_ESBUILD. +ensure_celld() { + CELLD_BIN="${CELLD_BIN:-$(command -v celld || true)}" + if [[ -z "$CELLD_BIN" || ! -x "$CELLD_BIN" ]]; then + echo "celld not found: set CELLD_BIN to a celld $CELLD_MIN_VERSION+ binary" >&2 + echo " (install: curl -fsSL https://celld.dev/install.sh | CELLD_VERSION=v$CELLD_MIN_VERSION sh)" >&2 + exit 1 + fi + local version + version="$("$CELLD_BIN" --version 2>/dev/null | sed -n 's/^celld \([0-9][0-9.]*\).*/\1/p' | head -1)" + if [[ -z "$version" ]] || [[ "$(printf '%s\n%s\n' "$CELLD_MIN_VERSION" "$version" | sort -V | head -1)" != "$CELLD_MIN_VERSION" ]]; then + echo "$CELLD_BIN is celld ${version:-(unknown)}; Mob Pit needs celld $CELLD_MIN_VERSION or newer (set CELLD_BIN)" >&2 + exit 1 + fi + if [[ -z "${CELLD_ESBUILD:-}" ]]; then + ensure_client + CELLD_ESBUILD="$CLIENT/node_modules/.bin/esbuild" + fi + export CELLD_BIN CELLD_ESBUILD +} + +# ensure_cells: build the Mob Pit cells worker when its sources changed. +ensure_cells() { + local marker="$CELLS/build/worker/shim.mjs" + if [[ ! -f "$marker" ]] || [[ -n "$(find "$CELLS/src" "$CELLS/Cargo.toml" "$SERVER/crawler" "$REPO/src" "$REPO/distributed_game/src" -newer "$marker" -name '*.rs' -print -quit 2>/dev/null)" ]] || + [[ "$CELLS/Cargo.toml" -nt "$marker" ]]; then + command -v worker-build >/dev/null || { + echo "worker-build not found: cargo install worker-build --locked" >&2 + exit 1 + } + log "building the Mob Pit cells worker (worker-build)" + (cd "$CELLS" && worker-build --release >/dev/null) + fi +} + +# kill_tree : stop a process and every descendant (celld dev starts its +# node in a session of its own, so collect the tree while the root is stopped). +kill_tree() { + local root="$1" pids=() frontier=("$1") next pid + kill -STOP "$root" 2>/dev/null || return 0 + while ((${#frontier[@]})); do + next=() + for pid in "${frontier[@]}"; do + pids+=("$pid") + while read -r child; do next+=("$child"); done < <(pgrep -P "$pid" 2>/dev/null || true) + done + frontier=("${next[@]+"${next[@]}"}") + done + kill -KILL "${pids[@]}" 2>/dev/null || true +} + +# start_celld : `celld dev` with the cells +# worker in (its state persists in /.celld/dev), relaying +# the outbox Queue to the game server on . Sets CELLD_PID, +# CELLD_URL and DISTRIBUTED_INTERNAL_SECRET (persisted in the project). +start_celld() { + local project="$1" port="$2" relay_port="$3" + ensure_celld + ensure_cells + mkdir -p "$project" + rm -rf "$project/build" + cp -R "$CELLS/build" "$project/build" + cp "$CELLS/wrangler.jsonc" "$project/wrangler.jsonc" + if [[ ! -s "$project/internal-secret" ]]; then + (umask 077 && printf 'test-only-%s' "$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')" >"$project/internal-secret") + fi + DISTRIBUTED_INTERNAL_SECRET="$(cat "$project/internal-secret")" + printf 'DISTRIBUTED_INTERNAL_SECRET=%s\nCELLD_QUEUE_RELAY_URL=http://127.0.0.1:%s/internal/celld-queue/relay\n' \ + "$DISTRIBUTED_INTERNAL_SECRET" "$relay_port" >"$project/.dev.vars" + log "starting celld dev on :$port ($("$CELLD_BIN" --version 2>/dev/null | tail -1); state in $project/.celld/dev)" + RUST_LOG=warn "$CELLD_BIN" dev "$project" --host 127.0.0.1 --port "$port" --no-watch --logs >"$project/celld.log" 2>&1 & + CELLD_PID=$! + CELLD_URL="http://127.0.0.1:$port" + for _ in $(seq 1 300); do + curl -fsS "$CELLD_URL/health" >/dev/null 2>&1 && break + if ! kill -0 "$CELLD_PID" 2>/dev/null; then + tail -20 "$project/celld.log" + echo "celld dev failed to start" >&2 + exit 1 + fi + sleep 0.2 + done + curl -fsS "$CELLD_URL/health" >/dev/null || { tail -20 "$project/celld.log"; echo "celld dev not ready" >&2; exit 1; } + export CELLD_URL DISTRIBUTED_INTERNAL_SECRET +} + ensure_server() { log "building game-demos server" cargo build --quiet --manifest-path "$SERVER/Cargo.toml" @@ -147,11 +244,15 @@ run_demo() { ensure_server ensure_wasm + CELLD_PID="" + if [[ " ${CELLD_DEMOS[*]} " == *" $demo "* ]]; then + start_celld "$MOBPIT_DATA_DIR/celld" "${CELLD_PORT:-$((port + 3))}" "$port" + fi local log_file log_file="$(mktemp -t game-demos.XXXXXX)" "$BIN" "$demo" --dev --port "$port" >"$log_file" 2>&1 & local server_pid=$! - trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; rm -f "$log_file"' EXIT INT TERM + trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; [[ -z "$CELLD_PID" ]] || kill_tree "$CELLD_PID"; rm -f "$log_file"' EXIT INT TERM for _ in $(seq 1 100); do grep -q '^LISTENING' "$log_file" && break if ! kill -0 "$server_pid" 2>/dev/null; then @@ -247,13 +348,14 @@ run_sveltekit() { ensure_server ensure_cli mkdir -p "$data" + start_celld "$data/celld" "${CELLD_PORT:-$((port + 3))}" "$port" local log_file secret log_file="$(mktemp -t game-demos-sveltekit.XXXXXX)" secret="${DISTRIBUTED_GAME_SECRET:-$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')}" DISTRIBUTED_GAME_SECRET="$secret" "$BIN" mobpit --port "$port" \ --data-dir "$data" --projection-bus "sqlite:$data/bus.db" >"$log_file" 2>&1 & local server_pid=$! - trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; rm -f "$log_file"' EXIT INT TERM + trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; kill_tree "$CELLD_PID"; rm -f "$log_file"' EXIT INT TERM wait_listening "$log_file" "$server_pid" "game-demos mobpit" api_keys echo @@ -275,11 +377,16 @@ run_sveltekit() { smoke() { ensure_sdk ensure_client + # Mob Pit's tests start their own `celld dev` (CELLD_BIN) on the worker. + ensure_celld + ensure_cells log "generated clients are current; predicted steps and pure functions compiled to WASM" gen --check log "server: fmt, clippy, tests (bots smoke)" cargo fmt --manifest-path "$SERVER/Cargo.toml" --check -p game_demos -p game_sim -p mobpit_crawler -p mobpit_service + cargo fmt --manifest-path "$CELLS/Cargo.toml" --check cargo clippy --quiet --manifest-path "$SERVER/Cargo.toml" --workspace --all-targets -- -D warnings + cargo clippy --quiet --manifest-path "$CELLS/Cargo.toml" --target wasm32-unknown-unknown -- -D warnings cargo test --quiet --manifest-path "$SERVER/Cargo.toml" --workspace cargo build --quiet --manifest-path "$SERVER/Cargo.toml" --workspace --bins log "client: type-check, build, tests (WASM steps + pure functions exact match, headless SDK)" @@ -302,7 +409,7 @@ case "${1:-}" in gen) gen "${2:-}" ;; sveltekit) run_sveltekit ;; list) printf '%s\n' "${DEMOS[@]}" sveltekit ;; - "" | -h | --help | help) sed -n '2,32p' "$0" ;; + "" | -h | --help | help) sed -n '2,39p' "$0" ;; *) if [[ " ${CLUSTER_DEMOS[*]} " == *" $1 "* ]]; then run_cluster "$1" diff --git a/examples/game/server/Cargo.lock b/examples/game/server/Cargo.lock index 8f89cca84..376e311c5 100644 --- a/examples/game/server/Cargo.lock +++ b/examples/game/server/Cargo.lock @@ -825,7 +825,6 @@ dependencies = [ "jsonwebtoken", "redis", "reqwest", - "rusqlite", "serde", "serde_json", "sha2 0.10.9", @@ -1133,6 +1132,7 @@ dependencies = [ "game_sim", "http-body-util", "mobpit_crawler", + "reqwest", "serde", "serde_json", "sqlx", diff --git a/examples/game/server/Cargo.toml b/examples/game/server/Cargo.toml index ee5ab2c11..e62fdc72d 100644 --- a/examples/game/server/Cargo.toml +++ b/examples/game/server/Cargo.toml @@ -30,10 +30,13 @@ game_sim = { path = "sim" } # The Crawler aggregate + RecordKill (shared with the read-model service). mobpit_crawler = { path = "crawler" } distributed_game = { path = "../../../distributed_game", features = ["net", "cell", "celld", "sqlite"] } -# Mob Pit: native durable SQLite cells + sorted-set leaderboards (Redis optional -# at runtime) + the SQLite bus for the read-model service. -distributed = { path = "../../..", features = ["cell-sqlite", "redis", "sqlite"] } -# The SQLite bus the cells' domain events are drained onto (`--projection-bus`). +# Mob Pit: sorted-set leaderboards (Redis optional at runtime) + the SQLite +# bus for the read-model service. Its durable cells run on celld +# (`examples/game/cells`), reached with `distributed_game`'s `celld` feature. +distributed = { path = "../../..", features = ["redis", "sqlite"] } +# Reads of the celld cells over the internal path. +reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] } +# The SQLite bus the cells' domain events are relayed onto (`--projection-bus`). sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "sqlite"] } axum = { version = "0.8", default-features = false, features = ["json", "query", "tokio", "http1"] } clap = { version = "4", features = ["derive", "env"] } diff --git a/examples/game/server/crawler/src/lib.rs b/examples/game/server/crawler/src/lib.rs index 0a933ca9b..e3dab6627 100644 --- a/examples/game/server/crawler/src/lib.rs +++ b/examples/game/server/crawler/src/lib.rs @@ -1,11 +1,11 @@ //! The durable side of Mob Pit: one `Crawler` aggregate per player, each in -//! its own native SQLite cell (`distributed` feature `cell-sqlite`, opened by -//! `game_demos`). +//! its own celld cell (`examples/game/cells`, kind [`CRAWLER_KIND`]), fed by +//! the zone server's `distributed_game::celld::CelldFactSink`. //! //! The only command is [`RECORD_KILL`], declared on the aggregate method //! [`Crawler::record_kill`] with `#[command]` (which generates the //! [`RecordKill`] mount and its handler). It is dispatched by the zone's fact -//! worker through `distributed_game::cell::CellFactSink`, never by a client: +//! worker through `distributed_game::celld::CelldFactSink`, never by a client: //! //! - the **principal** comes from the trusted server-built session //! (`x-user-id`, bound by the host from the player's verified ticket) into @@ -18,16 +18,28 @@ //! replays the stored receipt through the cell's command ledger instead of //! crediting twice; //! - `crawler.mob_killed` is a **domain event**: the commit publishes the -//! post-kill [`CrawlerState`] into the cell's outbox in the same SQLite -//! transaction ([`CrawlerMobKilledDomainEvent`]). The zone server drains -//! that outbox onto a bus and the read-model service (`mobpit_service`) -//! projects it into the `player_stats` table behind GraphQL `@live`. +//! post-kill [`CrawlerState`] into the cell's outbox in the same cell (SQLite) +//! transaction ([`CrawlerMobKilledDomainEvent`]). celld drains that outbox +//! to a Queue whose consumer relays it to the zone server, which publishes +//! it onto a bus; the read-model service (`mobpit_service`) projects it +//! into the `player_stats` table behind GraphQL `@live`. use distributed::{sourced, DomainState, Entity}; use serde::{Deserialize, Serialize}; /// Aggregate type (cell instance names are `Crawler:{principal}`). pub const AGGREGATE_TYPE: &str = "Crawler"; +/// celld cell kind (URL segment) of the Crawler cells: `crawler/{principal}`. +pub const CRAWLER_KIND: &str = "crawler"; +/// celld cell kind of the crawler roster (celld cannot enumerate cells, so +/// the principals with a Crawler cell are indexed there). +pub const ROSTER_KIND: &str = "crawler-roster"; +/// The roster's one shard. +pub const ROSTER_SHARD: &str = "all"; +/// Roster command: add the session principal (idempotent). +pub const ENLIST: &str = "crawler.enlist"; +/// celld cell kind of per-player presence (the runtime lifecycle facts). +pub const PRESENCE_KIND: &str = "presence"; /// Command dispatched for every `MobKilled` fact. pub const RECORD_KILL: &str = RecordKill::COMMAND; /// Event recorded by [`RECORD_KILL`]. diff --git a/examples/game/server/service/tests/projection.rs b/examples/game/server/service/tests/projection.rs index 81952af98..ee5bb6734 100644 --- a/examples/game/server/service/tests/projection.rs +++ b/examples/game/server/service/tests/projection.rs @@ -1,6 +1,8 @@ -//! Kill settlements → Crawler cell → outbox → SQLite bus → `player_stats` -//! → GraphQL `@live`, with the real zone-server world (cells + fact sink + -//! drain) and the service's projector and engine: +//! Kill settlements → celld Crawler cell → outbox → celld Queue → the zone +//! server's relay route → SQLite bus → `player_stats` → GraphQL `@live`, +//! with a real `celld dev` serving `examples/game/cells`, the real +//! zone-server world (fact sink + relay route) and the service's projector +//! and engine: //! //! 1. settled kills project into one row per player with the cell's totals //! (XP decided by the cell; a replayed fact changes nothing); @@ -19,6 +21,7 @@ use distributed::SqliteRepository; use distributed_game::{Fact, FactSink, FactStamp}; use futures_util::StreamExt; use game_demos::common::random_hex; +use game_demos::devcelld::DevCelld; use game_demos::mobpit::crawler::RECORD_KILL; use game_demos::mobpit::{MobPitWorld, WorldOptions}; use mobpit_service::{ @@ -74,7 +77,7 @@ async fn eventually>( let start = Instant::now(); while !check().await { assert!( - start.elapsed() < Duration::from_secs(20), + start.elapsed() < Duration::from_secs(45), "timed out waiting for {what}" ); tokio::time::sleep(Duration::from_millis(50)).await; @@ -88,16 +91,23 @@ async fn settled_kills_project_into_live_player_stats() { let bus_url = format!("sqlite:{}", dir.join("bus.db").display()); let db_url = format!("sqlite:{}?mode=rwc", dir.join("service.db").display()); - // Zone-server side: cells + sink, events drained to the bus. - let world = MobPitWorld::open(WorldOptions::new(dir.join("cells"))).unwrap(); + // Zone-server side: the relay route on a known port, celld relaying its + // outbox Queue there, the world's fact sink into the cells. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let relay = format!( + "http://{}{}", + listener.local_addr().unwrap(), + distributed::bus::CELLD_QUEUE_RELAY_PATH + ); + let celld = DevCelld::start(Some(relay)).unwrap_or_else(|error| panic!("celld dev: {error}")); + let world = MobPitWorld::open(WorldOptions::new(dir.join("local"), celld.options())).unwrap(); + let routes = game_demos::mobpit::routes::internal_routes(world.clone()); + tokio::spawn(async move { axum::serve(listener, routes).await.unwrap() }); let sink = world.fact_sink(); - // A kill settled before the bus existed waits in the cell's outbox. + // A kill settled before the bus is attached waits in the Queue (the + // relay answers 503 and the consumer retries). sink.commit(&kill("k0", "dev-ana", 2, "ana")).await.unwrap(); - let pending = world - .attach_projection_bus(&bus_url, Duration::from_millis(200)) - .await - .unwrap(); - assert_eq!(pending, 1, "startup drain publishes the waiting event"); + world.attach_projection_bus(&bus_url).await.unwrap(); // Service side. let repo = SqliteRepository::connect_and_migrate(&db_url) @@ -184,14 +194,7 @@ async fn settled_kills_project_into_live_player_stats() { sink.commit(&kill("k4", "dev-cy", 2, "cy")).await.unwrap(); eventually("third player", || async { rows(&repo).await.len() == 3 }).await; for (player, _, xp, kills, level) in rows(&repo).await { - let cell = world - .cells() - .get(&player) - .unwrap() - .load() - .await - .unwrap() - .unwrap(); + let cell = world.cells().crawler(&player).await.unwrap().state.unwrap(); assert_eq!( (xp, kills, level), (cell.xp as i64, i64::from(cell.kills), i64::from(cell.level)) diff --git a/examples/game/server/src/common/mod.rs b/examples/game/server/src/common/mod.rs index b89ad0834..d02b94ee6 100644 --- a/examples/game/server/src/common/mod.rs +++ b/examples/game/server/src/common/mod.rs @@ -102,6 +102,17 @@ impl RunningDemo { } } +/// Routes a demo adds to its server: `dev` only with `--dev` (next to +/// `/dev/ticket`), `internal` always (each route authenticates itself, e.g. +/// Mob Pit's secret-protected celld Queue relay). +#[derive(Default)] +pub struct DemoRoutes { + /// Dev-only routes. + pub dev: Router, + /// Always-mounted internal routes. + pub internal: Router, +} + /// Server config every demo starts from. pub fn server_config(key: TicketKey, net_sim: Option) -> ServerConfig { let mut config = ServerConfig::new(key); @@ -120,30 +131,32 @@ pub fn spawn( spawn_with::( listener, options, - |config, guests| (D::server(config, guests), Router::new()), + |config, guests| (D::server(config, guests), DemoRoutes::default()), signal, ) } -/// Like [`spawn`], but `build` makes the server and the demo's own dev -/// routes (merged next to `/dev/ticket`, only with `--dev`). For demos whose -/// zones and routes share state (e.g. a durable store) built by the caller. +/// Like [`spawn`], but `build` makes the server and the demo's own routes +/// ([`DemoRoutes`]). For demos whose zones and routes share state (e.g. a +/// durable store) built by the caller. pub fn spawn_with( listener: tokio::net::TcpListener, options: DemoOptions, - build: impl FnOnce(ServerConfig, Guests) -> (GameServer, Router), + build: impl FnOnce(ServerConfig, Guests) -> (GameServer, DemoRoutes), signal: impl Future + Send + 'static, ) -> std::io::Result> { let addr = listener.local_addr()?; let guests = Guests::default(); - let (server, demo_routes) = build( + let (server, routes) = build( server_config(options.key.clone(), options.net_sim), guests.clone(), ); let extra = if options.dev { - dev::routes::(server.clone(), options.key, guests.clone()).merge(demo_routes) + dev::routes::(server.clone(), options.key, guests.clone()) + .merge(routes.dev) + .merge(routes.internal) } else { - Router::new() + routes.internal }; let task = tokio::spawn(server.clone().serve_with(listener, extra, signal)); Ok(RunningDemo { @@ -157,15 +170,15 @@ pub fn spawn_with( /// CLI entry: bind, print the address, serve until Ctrl-C / SIGTERM. pub async fn run(args: DemoArgs) -> std::io::Result<()> { run_with::(args, |config, guests| { - (D::server(config, guests), Router::new()) + (D::server(config, guests), DemoRoutes::default()) }) .await } -/// [`run`] with a caller-built server and dev routes (see [`spawn_with`]). +/// [`run`] with a caller-built server and routes (see [`spawn_with`]). pub async fn run_with( args: DemoArgs, - build: impl FnOnce(ServerConfig, Guests) -> (GameServer, Router), + build: impl FnOnce(ServerConfig, Guests) -> (GameServer, DemoRoutes), ) -> std::io::Result<()> { let listener = tokio::net::TcpListener::bind((args.host.as_str(), args.port)).await?; let secret = args.secret.unwrap_or_else(|| guests::random_hex(32)); diff --git a/examples/game/server/src/devcelld.rs b/examples/game/server/src/devcelld.rs new file mode 100644 index 000000000..cf264f5ad --- /dev/null +++ b/examples/game/server/src/devcelld.rs @@ -0,0 +1,272 @@ +//! `celld dev` serving Mob Pit's cells (`examples/game/cells`) for tests: +//! its own project directory, port and process tree (killed on drop; only +//! the processes this harness started). +//! +//! - celld is found through `CELLD_BIN` (`examples/game/run.sh` sets it); +//! there is no fallback and no native cell mode, so a missing binary or an +//! unbuilt worker is an error with the fix in its message. +//! - `CELLD_ESBUILD` (celld's bundler) is inherited from the environment. +//! - State lives in `{project}/.celld/dev` and survives [`DevCelld::restart`] +//! (crash-recovery tests). + +use std::io::Write as _; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use crate::mobpit::store::CelldOptions; + +/// Env var naming the celld binary. +pub const CELLD_BIN_ENV: &str = "CELLD_BIN"; + +/// The built cells worker (`worker-build --release` in `examples/game/cells`). +pub fn cells_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../cells") +} + +/// A free local port (bound then released). +pub fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .and_then(|listener| listener.local_addr()) + .map(|addr| addr.port()) + .expect("free port") +} + +/// A random `test-only-…` internal secret (the Queue relay accepts plain +/// HTTP to localhost only with a `test-only-` secret). +pub fn test_secret() -> String { + format!("test-only-{}", crate::common::random_hex(32)) +} + +/// One `celld dev` process tree. +pub struct DevCelld { + project: PathBuf, + port: u16, + secret: String, + relay_url: Option, + child: Option, + generation: u32, + keep: bool, +} + +impl DevCelld { + /// Start celld on a fresh temp project. `relay_url` is where the outbox + /// Queue consumer relays domain events (a Mob Pit server's + /// `/internal/celld-queue/relay`); `None` keeps them queued. + pub fn start(relay_url: Option) -> Result { + let project = std::env::temp_dir().join(format!( + "mobpit-celld-{}-{}", + std::process::id(), + crate::common::random_hex(8) + )); + let mut celld = Self::in_project(&project, relay_url)?; + celld.keep = false; + Ok(celld) + } + + /// Start celld on `project` (created; existing state is reused). + pub fn in_project(project: &Path, relay_url: Option) -> Result { + let build = cells_dir().join("build"); + if !build.join("worker/shim.mjs").exists() { + return Err(format!( + "the Mob Pit cells worker is not built: run `worker-build --release` in {} \ + (examples/game/run.sh does)", + cells_dir().display() + )); + } + std::fs::create_dir_all(project).map_err(|e| format!("{}: {e}", project.display()))?; + copy_dir(&build, &project.join("build"))?; + std::fs::copy( + cells_dir().join("wrangler.jsonc"), + project.join("wrangler.jsonc"), + ) + .map_err(|e| format!("wrangler.jsonc: {e}"))?; + let mut celld = Self { + project: project.to_path_buf(), + port: free_port(), + secret: test_secret(), + relay_url, + child: None, + generation: 0, + keep: true, + }; + celld.launch()?; + Ok(celld) + } + + /// `http://127.0.0.1:{port}`. + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Options for a Mob Pit world on this celld. + pub fn options(&self) -> CelldOptions { + CelldOptions::new(self.url(), self.secret.clone()).expect("celld options") + } + + /// The internal secret. + pub fn secret(&self) -> &str { + &self.secret + } + + /// Project directory (holds `.celld/dev` and the logs). + pub fn project(&self) -> &Path { + &self.project + } + + fn launch(&mut self) -> Result<(), String> { + self.generation += 1; + let binary = std::env::var(CELLD_BIN_ENV).map_err(|_| { + format!( + "{CELLD_BIN_ENV} is not set: Mob Pit's cells run on celld (0.6.2+); \ + point {CELLD_BIN_ENV} at the celld binary" + ) + })?; + let mut vars = format!("DISTRIBUTED_INTERNAL_SECRET={}\n", self.secret); + if let Some(relay) = &self.relay_url { + vars.push_str(&format!("CELLD_QUEUE_RELAY_URL={relay}\n")); + } + std::fs::write(self.project.join(".dev.vars"), vars) + .map_err(|e| format!(".dev.vars: {e}"))?; + let log_path = self.project.join(format!("celld-{}.log", self.generation)); + let log = std::fs::File::create(&log_path).map_err(|e| e.to_string())?; + let mut command = Command::new(&binary); + command + .arg("dev") + .arg(&self.project) + .args(["--host", "127.0.0.1", "--port", &self.port.to_string()]) + .args(["--no-watch", "--logs"]) + .env("RUST_LOG", "warn") + .stdin(Stdio::null()) + .stdout(log.try_clone().map_err(|e| e.to_string())?) + .stderr(log); + let child = command + .spawn() + .map_err(|error| format!("cannot start `{binary} dev`: {error}"))?; + self.child = Some(child); + let deadline = Instant::now() + Duration::from_secs(90); + while Instant::now() < deadline { + if http_status(self.port, "/health") == Some(200) { + return Ok(()); + } + let exited = self + .child + .as_mut() + .and_then(|child| child.try_wait().ok().flatten()); + if let Some(status) = exited { + self.child = None; + let log = std::fs::read_to_string(&log_path).unwrap_or_default(); + return Err(format!( + "celld dev exited ({status}): {}", + log.lines().rev().take(5).collect::>().join(" | ") + )); + } + std::thread::sleep(Duration::from_millis(100)); + } + Err(format!( + "celld dev did not become ready; logs in {}", + self.project.display() + )) + } + + /// Kill the launcher and every descendant (a crash: no graceful + /// shutdown). `celld dev` starts its node in a session of its own, so + /// the tree is collected while the launcher is stopped, then killed. + pub fn crash(&mut self) { + let Some(mut child) = self.child.take() else { + return; + }; + let root = child.id(); + let _ = Command::new("kill") + .args(["-STOP", &root.to_string()]) + .status(); + let mut doomed = descendants(root); + doomed.insert(0, root); + for pid in doomed.iter().rev() { + let _ = Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status(); + } + let _ = child.wait(); + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline + && std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() + { + std::thread::sleep(Duration::from_millis(50)); + } + } + + /// Crash and start again on the same project, port and secret. + pub fn restart(&mut self) -> Result<(), String> { + self.crash(); + self.launch() + } +} + +impl Drop for DevCelld { + fn drop(&mut self) { + self.crash(); + if std::thread::panicking() { + let _ = writeln!( + std::io::stderr(), + "celld artifacts kept in {}", + self.project.display() + ); + } else if !self.keep { + let _ = std::fs::remove_dir_all(&self.project); + } + } +} + +/// Every descendant of `root` (breadth first). +fn descendants(root: u32) -> Vec { + let Ok(table) = Command::new("ps").args(["-axo", "pid=,ppid="]).output() else { + return Vec::new(); + }; + let pairs: Vec<(u32, u32)> = String::from_utf8_lossy(&table.stdout) + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?)) + }) + .collect(); + let mut out = Vec::new(); + let mut frontier = vec![root]; + while let Some(parent) = frontier.pop() { + for (pid, _) in pairs.iter().filter(|(_, ppid)| *ppid == parent) { + out.push(*pid); + frontier.push(*pid); + } + } + out +} + +fn copy_dir(from: &Path, to: &Path) -> Result<(), String> { + std::fs::create_dir_all(to).map_err(|e| format!("{}: {e}", to.display()))?; + for entry in std::fs::read_dir(from).map_err(|e| format!("{}: {e}", from.display()))? { + let entry = entry.map_err(|e| e.to_string())?; + let target = to.join(entry.file_name()); + if entry.file_type().map_err(|e| e.to_string())?.is_dir() { + copy_dir(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), &target) + .map_err(|e| format!("{}: {e}", target.display()))?; + } + } + Ok(()) +} + +fn http_status(port: u16, path: &str) -> Option { + use std::io::Read as _; + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok()?; + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .ok()?; + let mut response = String::new(); + stream.read_to_string(&mut response).ok()?; + response.split_whitespace().nth(1)?.parse().ok() +} diff --git a/examples/game/server/src/lib.rs b/examples/game/server/src/lib.rs index 63343f6ed..fd138f036 100644 --- a/examples/game/server/src/lib.rs +++ b/examples/game/server/src/lib.rs @@ -15,6 +15,7 @@ pub mod arena; pub mod cluster; pub mod common; +pub mod devcelld; pub mod freeze; pub mod gallery; pub mod grid; diff --git a/examples/game/server/src/mobpit/board.rs b/examples/game/server/src/mobpit/board.rs index 44083b9c9..22b02da4a 100644 --- a/examples/game/server/src/mobpit/board.rs +++ b/examples/game/server/src/mobpit/board.rs @@ -34,7 +34,7 @@ use distributed_game::GameRecord; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use super::crawler::{KillReceipt, MobKilledArgs, AGGREGATE_TYPE, MOB_KILLED_EVENT}; +use super::crawler::{KillReceipt, AGGREGATE_TYPE}; use super::store::CrawlerCells; /// Kills board. @@ -436,33 +436,24 @@ impl Leaderboards { let before = self.top_rows().await.map_err(|e| e.to_string())?; let mut occurrences = Vec::new(); let mut expected: BTreeMap = BTreeMap::new(); - let principals = cells.principals(); + let principals = cells.principals().await?; for principal in &principals { - let cell = cells.get(principal).map_err(|e| e.to_string())?; - for record in cell.history().await.map_err(|e| e.to_string())? { - if record.event_name != MOB_KILLED_EVENT { - continue; - } - let (crawler, name, mob, tier, xp): MobKilledArgs = - record.decode().map_err(|e| e.to_string())?; - self.remember_name(&crawler, &name); - let at_ms = record - .timestamp - .duration_since(UNIX_EPOCH) - .map_or(0, |d| d.as_millis() as u64); + let view = cells.crawler(principal).await?; + for kill in view.kills { + self.remember_name(&kill.crawler, &kill.name); occurrences.push(kill_occurrence( principal, - record.sequence, - at_ms, + kill.sequence, + kill.at_ms, &MobKilled { - crawler, - mob, - tier, - xp, + crawler: kill.crawler, + mob: kill.mob, + tier: kill.tier, + xp: kill.xp, }, )?); } - if let Some(state) = cell.load().await.map_err(|e| e.to_string())? { + if let Some(state) = view.state { expected.insert(principal.clone(), (state.kills, state.xp)); } } diff --git a/examples/game/server/src/mobpit/mod.rs b/examples/game/server/src/mobpit/mod.rs index e3301f85e..d0a092eb9 100644 --- a/examples/game/server/src/mobpit/mod.rs +++ b/examples/game/server/src/mobpit/mod.rs @@ -6,9 +6,11 @@ //! state machine, melee cones). A kill emits a fact and keeps ticking. //! - [`crawler`]: the per-player `Crawler` aggregate and its one command; //! XP is decided there, the principal comes from the trusted session. -//! - [`store`]: one native SQLite cell per player + the `FactSink` (with the -//! "slow storage" latency knob) that routes facts into them via -//! `distributed_game::cell::CellFactSink`. +//! - [`store`]: the zone server's side of the celld cells +//! (`examples/game/cells`): the `FactSink` (with the "slow storage" +//! latency knob) that routes facts to them via +//! `distributed_game::celld::CelldFactSink`, the roster and cell reader, +//! and the projection bus fed by the outbox Queue relay. //! - [`board`]: kills/XP leaderboards (in-memory or Redis), fire-and-forget //! live updates, idempotent apply keys, rebuild from the cells' events. //! - [`presence`]: who is online where, projected from the runtime's player @@ -17,9 +19,11 @@ //! //! `game-demos mobpit` and `game-demos leaderboard` start the same server //! (one process hosts the pit and the boards); they differ only in the page -//! `run.sh` opens. Data lives in `--data-dir` / `MOBPIT_DATA_DIR`; the -//! ticket secret is persisted there too so guests keep their identity across -//! a restart (crash-recovery demo). +//! `run.sh` opens. Durable state is in celld (`--celld-url` / `CELLD_URL`, +//! secret `DISTRIBUTED_INTERNAL_SECRET`); `run.sh` and the tests start +//! `celld dev` with `examples/game/cells`. `--data-dir` / `MOBPIT_DATA_DIR` +//! keeps the persisted ticket secret (so guests keep their identity across +//! a restart, the crash-recovery demo) and the default projection bus. pub mod board; pub mod crawler; @@ -45,8 +49,13 @@ pub struct MobPitArgs { /// Shared demo options. #[command(flatten)] pub common: DemoArgs, - /// Directory of the per-player SQLite cells (and the persisted dev - /// ticket secret). Default: `$TMPDIR/game-demos-mobpit`. + /// celld serving `examples/game/cells` (the Crawler, roster and + /// presence cells). The internal secret comes from + /// `DISTRIBUTED_INTERNAL_SECRET`. + #[arg(long, env = "CELLD_URL")] + pub celld_url: String, + /// Directory of local dev files: the persisted dev ticket secret and + /// the default projection bus. Default: `$TMPDIR/game-demos-mobpit`. #[arg(long, env = "MOBPIT_DATA_DIR")] pub data_dir: Option, /// Redis for the leaderboard (`redis://127.0.0.1:6379`); default in memory. @@ -55,10 +64,10 @@ pub struct MobPitArgs { /// Start with this artificial storage latency (ms). #[arg(long, env = "MOBPIT_STORAGE_LATENCY_MS", default_value_t = 0)] pub storage_latency_ms: u64, - /// SQLite bus (`sqlite:/path/bus.db`) to publish the Crawler cells' - /// `crawler.mob_killed` domain events to; `mobpit-service` projects them - /// into its `player_stats` read model. Unset: events stay in the cells' - /// outboxes until a bus is attached. + /// SQLite bus (`sqlite:/path/bus.db`) the Crawler cells' + /// `crawler.mob_killed` domain events are published to (relayed from + /// celld's outbox Queue); `mobpit-service` projects them into its + /// `player_stats` read model. Default: `sqlite:{data-dir}/bus.db`. #[arg(long, env = "MOBPIT_PROJECTION_BUS")] pub projection_bus: Option, } @@ -76,9 +85,11 @@ impl Demo for MobPitDemo { } /// Plain `common::spawn` gets a throwaway world (temp dir, in-memory - /// board, no dev routes of its own); use [`spawn`] / [`run`] instead. + /// board, no routes of its own) on the celld named by `CELLD_URL` and + /// `DISTRIBUTED_INTERNAL_SECRET`; use [`spawn`] / [`run`] instead. fn server(config: ServerConfig, guests: Guests) -> GameServer { - let world = MobPitWorld::open(WorldOptions::temporary()) + let celld = store::CelldOptions::from_env().expect("Mob Pit needs celld"); + let world = MobPitWorld::open(WorldOptions::temporary(celld)) .expect("mobpit world (needs a Tokio runtime)"); world.server(config, guests).0 } @@ -119,13 +130,19 @@ fn persisted_secret(dir: &std::path::Path) -> std::io::Result { Ok(secret) } -/// CLI entry: open the cells, rebuild the board from them, serve. +/// CLI entry: connect to the cells, rebuild the board from them, serve. pub async fn run(mut args: MobPitArgs) -> std::io::Result<()> { let data_dir = args .data_dir .clone() .unwrap_or_else(|| std::env::temp_dir().join("game-demos-mobpit")); - let mut options = WorldOptions::new(&data_dir); + let secret = std::env::var(distributed::cell_host::CELL_INTERNAL_SECRET_ENV).map_err(|_| { + std::io::Error::other( + "DISTRIBUTED_INTERNAL_SECRET is required (shared with the celld cells worker)", + ) + })?; + let celld = store::CelldOptions::new(&args.celld_url, secret).map_err(std::io::Error::other)?; + let mut options = WorldOptions::new(&data_dir, celld); options.redis_url = args.redis_url.clone(); let world = MobPitWorld::open(options).map_err(std::io::Error::other)?; world @@ -143,15 +160,17 @@ pub async fn run(mut args: MobPitArgs) -> std::io::Result<()> { ), Err(error) => eprintln!("mobpit: leaderboard rebuild failed (serving anyway): {error}"), } - if let Some(url) = &args.projection_bus { - let published = world - .attach_projection_bus(url, std::time::Duration::from_secs(2)) - .await - .map_err(std::io::Error::other)?; - eprintln!("mobpit: projection bus {url} (published {published} pending events)"); - } + let bus = args + .projection_bus + .clone() + .unwrap_or_else(|| format!("sqlite:{}", data_dir.join("bus.db").display())); + world + .attach_projection_bus(&bus) + .await + .map_err(std::io::Error::other)?; eprintln!( - "mobpit: data dir {} · boot {} · leaderboard {}", + "mobpit: celld {} · projection bus {bus} · data dir {} · boot {} · leaderboard {}", + args.celld_url, data_dir.display(), world.boot(), world.board().backend() diff --git a/examples/game/server/src/mobpit/routes.rs b/examples/game/server/src/mobpit/routes.rs index 156636ea2..87b1b9626 100644 --- a/examples/game/server/src/mobpit/routes.rs +++ b/examples/game/server/src/mobpit/routes.rs @@ -5,22 +5,30 @@ //! artificial latency in front of every cell commit ("slow storage"). //! - `GET /dev/mobpit/stats` — boot id, sink counters, pending facts, recent //! commits (key, exact input, receipt). -//! - `GET /dev/mobpit/crawlers` — every Crawler cell on disk (read path). +//! - `GET /dev/mobpit/crawlers` — every Crawler cell on the roster (read +//! over celld's internal path). //! - `GET /dev/mobpit/leaderboard[?member=principal]` — the pushed top-N //! snapshot, plus rank / neighbours of `member` on both boards. //! - `POST /dev/mobpit/leaderboard/rebuild` — rebuild both boards from the //! cells' event history; reports whether the result matches the cells. //! - `GET /dev/mobpit/presence` — who is online / away / offline where, //! projected from the runtime's player lifecycle facts. +//! +//! [`internal_routes`] (always mounted): `POST /internal/celld-queue/relay`, +//! the celld outbox Queue consumer's hop, authenticated with +//! `DISTRIBUTED_INTERNAL_SECRET`; it publishes the Crawler domain event onto +//! the projection bus (503 until one is attached, so the Queue retries). use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use axum::extract::{Query, State}; -use axum::http::StatusCode; +use axum::http::{HeaderMap, StatusCode}; use axum::routing::{get, post}; use axum::{Json, Router}; +use distributed::bus::{CelldQueueEnvelope, CELLD_QUEUE_RELAY_PATH}; +use distributed::cell_host::CELL_INTERNAL_SECRET_HEADER; use serde::Deserialize; use serde_json::{json, Value}; @@ -47,6 +55,47 @@ pub fn routes(world: Arc) -> Router { .with_state(world) } +/// Internal routes (see the module docs). +pub fn internal_routes(world: Arc) -> Router { + Router::new() + .route(CELLD_QUEUE_RELAY_PATH, post(relay)) + .with_state(world) +} + +async fn relay( + State(world): State>, + headers: HeaderMap, + Json(envelope): Json, +) -> (StatusCode, Json) { + let authorized = headers + .get(CELL_INTERNAL_SECRET_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| world.celld().secret.matches(candidate)); + if !authorized { + return ( + StatusCode::UNAUTHORIZED, + Json(json!({ "code": "UNAUTHORIZED", "error": "unauthorized" })), + ); + } + let Some(bus) = world.projection_bus() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "code": "UNAVAILABLE", "error": "no projection bus attached" })), + ); + }; + match bus.relay_handler()(envelope).await { + Ok(()) => (StatusCode::ACCEPTED, Json(json!({ "ok": true }))), + Err(error) if error.is_retryable() => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "code": "UNAVAILABLE", "error": error.message() })), + ), + Err(error) => ( + StatusCode::BAD_REQUEST, + Json(json!({ "code": "BAD_REQUEST", "error": error.message() })), + ), + } +} + /// `POST /dev/mobpit/storage` body. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -76,6 +125,7 @@ async fn stats(State(world): State>) -> Json { Json(json!({ "boot": world.boot(), "dataDir": world.data_dir().display().to_string(), + "celld": world.cells().url(), "latencyMs": world.sink_state().latency().as_millis() as u64, "pending": world.zone_pending(), "sink": world.sink_state().stats(), diff --git a/examples/game/server/src/mobpit/store.rs b/examples/game/server/src/mobpit/store.rs index e4c6dec17..9799a4354 100644 --- a/examples/game/server/src/mobpit/store.rs +++ b/examples/game/server/src/mobpit/store.rs @@ -1,112 +1,224 @@ -//! Per-player durable cells and the fact sink that routes into them. +//! Mob Pit's durable state lives in celld cells (`examples/game/cells`); +//! this module is the zone server's side of them. //! -//! [`CrawlerCells`] keeps one native SQLite cell per principal under a data -//! directory (`Crawler%3A{principal}.sqlite3`, see `CellSqlitePath`). Every -//! command commits its event, the command-ledger receipt and snapshot rows in -//! one SQLite transaction with `synchronous=FULL`, so an acknowledged fact -//! survives `kill -9` (and power loss), and a half-applied one cannot exist. -//! -//! [`CrawlerFactSink`] is the game's `FactSink`: optional artificial latency -//! (the "slow storage" toggle), then `CellFactSink::commit` into the killer's -//! cell, then a fire-and-forget leaderboard update. -//! -//! Each commit also writes the `crawler.mob_killed` domain event into the -//! cell's outbox (same transaction). With a [`ProjectionBus`] attached -//! (`--projection-bus`), the sink drains that outbox onto the SQLite bus -//! after the commit, and a sweep drains every cell at startup and -//! periodically (a crash between commit and drain, a bus outage). Delivery -//! is at-least-once with stable message ids; the read-model service's -//! projector deduplicates. Without a bus the rows wait in the cells. - -use std::collections::{BTreeMap, VecDeque}; -use std::path::{Path, PathBuf}; +//! - [`MobPitFactSink`] is the game's `FactSink`: optional artificial +//! latency (the "slow storage" toggle), then +//! `distributed_game::celld::CelldFactSink` sends the fact over the +//! internal path to its cell — `crawler.record_kill` to +//! `crawler/{principal}` (`CelldFactRoute::by_principal`), the runtime's +//! player lifecycle facts to `presence/{principal}` (`lifecycle_route`) — +//! then a fire-and-forget leaderboard update. The command id comes from +//! the fact key, so a retry replays through the cell's command ledger +//! instead of crediting twice; celld slow or down keeps facts in the +//! zone's queue (`RetryPolicy::until_settled`), never in a local store. +//! - celld cannot enumerate cells, so before a principal's first kill +//! commits the sink enlists it in the `crawler-roster/all` cell +//! (`crawler.enlist`, idempotent by principal). Every committed Crawler +//! cell is therefore on the roster; [`CrawlerCells`] reads both over the +//! internal path (profiles, `/dev/mobpit/crawlers`, the board rebuild). +//! - Each kill commit also writes the `crawler.mob_killed` domain event to +//! the cell's outbox in the same cell transaction. celld drains it to the +//! `mobpit-outbox` Queue; the worker's consumer relays every message to +//! this server's authenticated `/internal/celld-queue/relay`, which +//! publishes it onto the [`ProjectionBus`] (a SQLite bus read by +//! `mobpit-service`). Delivery is at-least-once with stable message ids; +//! the projector deduplicates. A relay that fails is retried by the Queue. + +use std::collections::{HashSet, VecDeque}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use distributed::bus::SqliteBus; -use distributed::cell_host::{AggregateCell, CellSqlitePath, CellStreamStore}; -use distributed::entity::EventRecord; +use distributed::bus::{celld_queue_relay_handler, CelldQueueRelayHandler, SqliteBus}; +use distributed::cell_host::{InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER}; use distributed::BusPublisher; -use distributed::{RepositoryError, StreamIdentity}; -use distributed_game::cell::CellFactSink; +use distributed_game::celld::{CelldFactRoute, CelldFactSink}; use distributed_game::lifecycle::Lifecycle; use distributed_game::{BoxFuture, Fact, FactError, FactReceipt, FactSink}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; use super::board::Leaderboards; -use super::crawler::{Crawler, KillReceipt, RecordKill, AGGREGATE_TYPE, RECORD_KILL}; +use super::crawler::{ + KillReceipt, RecordKillInput, CRAWLER_KIND, ENLIST, PRESENCE_KIND, RECORD_KILL, ROSTER_KIND, + ROSTER_SHARD, +}; use super::presence::Presence; /// Command-ledger service id for game facts. pub const SERVICE_ID: &str = "mobpit"; /// Principal for facts without a player (none are emitted today). pub const SYSTEM_PRINCIPAL: &str = "mobpit-zone"; -const FILE_PREFIX: &str = "Crawler%3A"; - -/// One player's open cell. -pub struct CrawlerCell { - cell: Arc>, - sink: CellFactSink, - /// One drain at a time per cell keeps the published order = commit order. - draining: tokio::sync::Mutex<()>, +/// Env var of the celld base URL. +pub const CELLD_URL_ENV: &str = "CELLD_URL"; + +/// Where the cells are: the celld base URL and the internal secret shared +/// with the cells worker (`DISTRIBUTED_INTERNAL_SECRET`). +#[derive(Clone)] +pub struct CelldOptions { + /// `http(s)://host:port` of celld serving `examples/game/cells`. + pub url: String, + /// Internal-path secret (also authenticates the Queue relay route). + pub secret: InternalHttpSecret, } -impl CrawlerCell { - /// Current state (`None` before the first kill). - pub async fn load(&self) -> Result, RepositoryError> { - self.cell.load().await +impl CelldOptions { + /// Options for `url` and `secret`. + pub fn new(url: impl Into, secret: impl Into) -> Result { + let url = url.into().trim_end_matches('/').to_string(); + if !(url.starts_with("http://") || url.starts_with("https://")) { + return Err(format!("celld URL must be http(s): {url}")); + } + Ok(Self { + url, + secret: InternalHttpSecret::new(secret.into())?, + }) + } + + /// `CELLD_URL` and `DISTRIBUTED_INTERNAL_SECRET` from the environment. + pub fn from_env() -> Result { + let var = |name: &str| { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| format!("{name} is required (Mob Pit's cells run on celld)")) + }; + Self::new( + var(CELLD_URL_ENV)?, + var(distributed::cell_host::CELL_INTERNAL_SECRET_ENV)?, + ) } +} + +/// A Crawler cell's state as the cells worker reports it. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct CrawlerSummary { + /// Display name at the last kill. + pub name: String, + /// Total XP. + pub xp: u64, + /// Mobs killed. + pub kills: u32, + /// Level for `xp`. + pub level: u32, + /// Committed stream version. + pub version: u64, +} + +/// One `crawler.mob_killed` event of a Crawler cell's history. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct KillRecord { + /// Stream position (the leaderboard occurrence). + pub sequence: u64, + /// Cell time of the commit (Unix ms). + pub at_ms: u64, + /// Principal. + pub crawler: String, + /// Display name at the kill. + pub name: String, + /// Mob instance. + pub mob: String, + /// Mob tier. + pub tier: u8, + /// XP credited. + pub xp: u32, +} + +/// `GET crawler/{principal}`: state (None before the first kill) and the +/// kill history, oldest first. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct CrawlerView { + /// Current state. + pub state: Option, + /// Kill events. + pub kills: Vec, +} + +/// Read side of the Crawler cells and the roster, over the internal path. +#[derive(Clone)] +pub struct CrawlerCells { + client: reqwest::Client, + base: String, + secret: InternalHttpSecret, +} - /// The full durable event history of this cell, oldest first. - pub async fn history(&self) -> Result, RepositoryError> { - self.cell.history().await +impl CrawlerCells { + /// Reader for the cells at `celld`. + pub fn new(celld: &CelldOptions) -> Result { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("celld client: {error}"))?; + Ok(Self { + client, + base: celld.url.clone(), + secret: celld.secret.clone(), + }) } - /// The cell (tests dispatch into it directly). - pub fn cell(&self) -> &Arc> { - &self.cell + /// celld base URL. + pub fn url(&self) -> &str { + &self.base } - /// Publish this cell's pending outbox rows (its domain events) to `bus`, - /// oldest first; returns how many were published. - pub async fn drain_to(&self, bus: &ProjectionBus) -> Result { - let _turn = self.draining.lock().await; - let mut published = 0; - loop { - let outcome = self - .cell - .drain_outbox_with( - bus.publisher.clone(), - Duration::from_secs(30), - DRAIN_MAX_ATTEMPTS, - DRAIN_BATCH, - ) - .await - .map_err(|e| e.to_string())?; - published += outcome.published; - if outcome.released > 0 || outcome.failed > 0 { - return Err(format!( - "projection bus publish failed ({} released, {} failed)", - outcome.released, outcome.failed - )); - } - if outcome.claimed < DRAIN_BATCH { - return Ok(published); - } + async fn get(&self, kind: &str, shard: &str) -> Result { + let url = format!("{}/{}/{}", self.base, kind, encode_segment(shard)); + let response = self + .client + .get(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .send() + .await + .map_err(|error| format!("celld unreachable at {url}: {error}"))?; + let status = response.status(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if !status.is_success() { + return Err(format!("GET {url}: HTTP {status}: {body}")); } + Ok(body) + } + + /// Principals with a Crawler cell (sorted), from the roster. + pub async fn principals(&self) -> Result, String> { + let body = self.get(ROSTER_KIND, ROSTER_SHARD).await?; + let mut out: Vec = serde_json::from_value(body["principals"].clone()) + .map_err(|error| format!("roster: {error}"))?; + out.sort(); + Ok(out) + } + + /// The durable presence entry of `principal` (`null` when none). + pub async fn presence(&self, principal: &str) -> Result { + Ok(self.get(PRESENCE_KIND, principal).await?["current"].take()) + } + + /// State and kill history of `principal`'s cell. + pub async fn crawler(&self, principal: &str) -> Result { + let body = self.get(CRAWLER_KIND, principal).await?; + serde_json::from_value(body).map_err(|error| format!("crawler {principal}: {error}")) } } -const DRAIN_BATCH: usize = 32; -/// Rows are retried by later drains rather than dropped. -const DRAIN_MAX_ATTEMPTS: u32 = 10_000; +/// Like `distributed_game::celld`: a shard cannot add path segments. +fn encode_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b':') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} /// The SQLite bus Crawler domain events are published to (consumed by /// `mobpit-service`'s projector). #[derive(Clone)] pub struct ProjectionBus { - publisher: BusPublisher, + relay: CelldQueueRelayHandler, } impl ProjectionBus { @@ -128,114 +240,16 @@ impl ProjectionBus { .await .map_err(|e| format!("projection bus tables: {e}"))?; Ok(Self { - publisher: BusPublisher::new(Arc::new(bus)), + relay: celld_queue_relay_handler(BusPublisher::new(Arc::new(bus))), }) } -} -/// Lazily opened `Crawler` cells under one directory. -pub struct CrawlerCells { - root: PathBuf, - open: Mutex>>, -} - -impl CrawlerCells { - /// Cells under `root` (created on first write). - pub fn new(root: impl Into) -> Self { - Self { - root: root.into(), - open: Mutex::new(BTreeMap::new()), - } - } - - /// Data directory. - pub fn root(&self) -> &Path { - &self.root - } - - /// Open (once per process) the cell of `principal`. Opening the same file - /// twice in one process would give two connections; the cache prevents it. - pub fn get(&self, principal: &str) -> Result, RepositoryError> { - let mut open = self.open.lock().expect("cells lock"); - if let Some(cell) = open.get(principal) { - return Ok(cell.clone()); - } - let identity = StreamIdentity::new(AGGREGATE_TYPE, principal.to_string())?; - let store = CellStreamStore::open_sqlite(&CellSqlitePath::root(&self.root), identity)?; - let cell = Arc::new(AggregateCell::from_store(store)?.mount(RecordKill)); - let entry = Arc::new(CrawlerCell { - sink: CellFactSink::new(cell.clone(), SERVICE_ID, SYSTEM_PRINCIPAL), - cell, - draining: tokio::sync::Mutex::new(()), - }); - open.insert(principal.to_string(), entry.clone()); - Ok(entry) - } - - /// Whether `principal` already has a cell file (reads never create one). - pub fn exists(&self, principal: &str) -> bool { - CellSqlitePath::root(&self.root) - .resolve(&format!("{AGGREGATE_TYPE}:{principal}")) - .exists() - } - - /// Principals with a cell file on disk (sorted). - pub fn principals(&self) -> Vec { - let Ok(dir) = std::fs::read_dir(&self.root) else { - return Vec::new(); - }; - let suffix = format!(".{}", distributed::cell_host::NATIVE_CELL_SQLITE_EXTENSION); - let mut out: Vec = dir - .filter_map(|entry| entry.ok()?.file_name().into_string().ok()) - .filter_map(|name| { - let stem = name.strip_prefix(FILE_PREFIX)?.strip_suffix(&suffix)?; - // Long names are hashed (`~`) and cannot be inverted; guest - // and bot principals never are. - percent_decode(stem) - }) - .collect(); - out.sort(); - out.dedup(); - out - } - - /// Drain every cell on disk to `bus` (startup recovery and the periodic - /// sweep). Returns rows published; per-cell errors are reported and - /// retried by the next sweep. - pub async fn drain_all(&self, bus: &ProjectionBus) -> usize { - let mut published = 0; - for principal in self.principals() { - match self.get(&principal) { - Ok(cell) => match cell.drain_to(bus).await { - Ok(n) => published += n, - Err(error) => eprintln!("mobpit: drain {principal}: {error}"), - }, - Err(error) => eprintln!("mobpit: open {principal}: {error}"), - } - } - published + /// The Queue relay handler publishing onto this bus. + pub fn relay_handler(&self) -> &CelldQueueRelayHandler { + &self.relay } } -fn percent_decode(stem: &str) -> Option { - let bytes = stem.as_bytes(); - let mut out = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'%' { - let hex = stem.get(i + 1..i + 3)?; - out.push(u8::from_str_radix(hex, 16).ok()?); - i += 3; - } else if bytes[i] == b'~' { - return None; - } else { - out.push(bytes[i]); - i += 1; - } - } - String::from_utf8(out).ok() -} - /// A committed kill as the dev routes report it. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -264,7 +278,7 @@ pub struct SinkStats { pub errors: u64, /// Facts currently inside `commit` (incl. the artificial latency). pub in_sink: u64, - /// Lifecycle facts projected into presence (not counted above). + /// Lifecycle facts settled in presence cells (not counted above). pub presence: u64, } @@ -311,38 +325,68 @@ impl SinkState { /// The game's fact sink (cloned into every zone's fact worker). #[derive(Clone)] -pub struct CrawlerFactSink { - cells: Arc, +pub struct MobPitFactSink { + celld: CelldFactSink, state: Arc, board: Arc, - projection: Arc>, presence: Arc, + /// Principals enlisted on the roster by this process. + enlisted: Arc>>, } -impl CrawlerFactSink { - /// Sink over `cells`, reporting into `state`, feeding `board`. +impl MobPitFactSink { + /// Sink into the cells at `celld`, reporting into `state`, feeding + /// `board`, mirroring settled lifecycle facts into `presence`. pub fn new( - cells: Arc, + celld: &CelldOptions, state: Arc, board: Arc, - projection: Arc>, - ) -> Self { - Self { - cells, + presence: Arc, + ) -> Result { + let sink = CelldFactSink::new( + celld.url.clone(), + celld.secret.clone(), + SERVICE_ID, + SYSTEM_PRINCIPAL, + )? + .route(CelldFactRoute::by_principal::( + CRAWLER_KIND, + )) + .route(CelldFactRoute::new(ENLIST, ROSTER_KIND, |_, _| { + Some(ROSTER_SHARD.to_string()) + })) + .lifecycle_route(PRESENCE_KIND); + Ok(Self { + celld: sink, state, board, - projection, - presence: Arc::default(), - } + presence, + enlisted: Arc::default(), + }) } - /// Project lifecycle facts into `presence` (instead of a private one). - pub fn with_presence(mut self, presence: Arc) -> Self { - self.presence = presence; - self + /// Enlist `principal` on the roster before its first kill commits (once + /// per process; the ledger makes a repeat a replay). + async fn enlist(&self, principal: &str, fact: &Fact) -> Result<(), FactError> { + if self.enlisted.lock().expect("enlisted").contains(principal) { + return Ok(()); + } + let enlist = Fact::trusted( + format!("{ENLIST}:{principal}"), + ENLIST, + json!({}), + Some(principal.to_string()), + fact.stamp().clone(), + ); + self.celld.commit(&enlist).await?; + self.enlisted + .lock() + .expect("enlisted") + .insert(principal.to_string()); + Ok(()) } - async fn commit_inner(&self, fact: &Fact) -> Result { + async fn commit_kill(&self, fact: &Fact) -> Result { let latency = self.state.latency(); if !latency.is_zero() { tokio::time::sleep(latency).await; @@ -350,22 +394,8 @@ impl CrawlerFactSink { let principal = fact .principal() .ok_or_else(|| FactError::Permanent("kill facts need a player principal".into()))?; - - let cell = self - .cells - .get(principal) - .map_err(|e| FactError::Retryable(format!("open cell: {e}")))?; - let receipt = cell.sink.commit(fact).await?; - if let Some(bus) = self.projection.get().cloned() { - // Off the settlement path: the zone hears "saved" when the cell - // commits; the read model follows from the outbox. - let cell = cell.clone(); - tokio::spawn(async move { - if let Err(error) = cell.drain_to(&bus).await { - eprintln!("mobpit: projection drain (retried by the sweep): {error}"); - } - }); - } + self.enlist(principal, fact).await?; + let receipt = self.celld.commit(fact).await?; if fact.command() == RECORD_KILL { if let Ok(kill) = serde_json::from_value::(receipt.payload.clone()) { // Fire-and-forget: never awaits the leaderboard store. @@ -386,24 +416,23 @@ impl CrawlerFactSink { } } -impl FactSink for CrawlerFactSink { +impl FactSink for MobPitFactSink { fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { Box::pin(async move { - // Runtime lifecycle facts feed the presence read model, not a - // cell (and not the kill counters). + // Runtime lifecycle facts go to the player's presence cell, not + // a Crawler cell (and not the kill counters); the settled ones + // are mirrored into the in-process presence view. if let (Some(event), Some(principal)) = (Lifecycle::of(fact), fact.principal()) { - let applied = self.presence.apply(principal, fact.stamp(), &event); + let receipt = self.celld.commit(fact).await?; + self.presence.apply(principal, fact.stamp(), &event); self.state.update(|s| s.presence += 1); - return Ok(FactReceipt { - payload: serde_json::json!({ "presence": applied }), - replayed: !applied, - }); + return Ok(receipt); } self.state.update(|s| { s.attempts += 1; s.in_sink += 1; }); - let result = self.commit_inner(fact).await; + let result = self.commit_kill(fact).await; self.state.update(|s| { s.in_sink -= 1; match &result { @@ -422,14 +451,18 @@ mod tests { use super::*; #[test] - fn file_names_round_trip_to_principals() { - let dir = - std::env::temp_dir().join(format!("mobpit-cells-{}", crate::common::random_hex(12))); - let cells = CrawlerCells::new(&dir); - for p in ["guest-0a1b", "bot-3", "odd:name"] { - cells.get(p).unwrap(); - } - assert_eq!(cells.principals(), vec!["bot-3", "guest-0a1b", "odd:name"]); - let _ = std::fs::remove_dir_all(dir); + fn shards_cannot_add_path_segments() { + assert_eq!(encode_segment("guest-0a1b"), "guest-0a1b"); + assert_eq!(encode_segment("odd:name"), "odd:name"); + assert_eq!(encode_segment("a/b c"), "a%2Fb%20c"); + } + + #[test] + fn options_require_http_and_a_strong_secret() { + assert!(CelldOptions::new("ftp://x", "test-only-secret-0123456789abcdef").is_err()); + assert!(CelldOptions::new("http://127.0.0.1:1", "short").is_err()); + let ok = + CelldOptions::new("http://127.0.0.1:1/", "test-only-secret-0123456789abcdef").unwrap(); + assert_eq!(ok.url, "http://127.0.0.1:1"); } } diff --git a/examples/game/server/src/mobpit/world.rs b/examples/game/server/src/mobpit/world.rs index 5656bc7be..22fa5d717 100644 --- a/examples/game/server/src/mobpit/world.rs +++ b/examples/game/server/src/mobpit/world.rs @@ -1,32 +1,34 @@ //! Process-wide Mob Pit state shared by the zones, the fact worker and the -//! dev routes: the durable cells, the sink's latency/counters and the -//! leaderboards. +//! dev routes: the celld cells (fact sink and reader), the sink's +//! latency/counters, the leaderboards and the projection bus. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use axum::Router; use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; use distributed_game::RetryPolicy; use serde_json::{json, Value}; use super::board::Leaderboards; use super::presence::Presence; -use super::store::{CrawlerCells, CrawlerFactSink, ProjectionBus, SinkState}; +use super::store::{CelldOptions, CrawlerCells, MobPitFactSink, ProjectionBus, SinkState}; use super::zone::{MobPit, MobPitConfig}; use super::{routes, MobPitDemo}; -use crate::common::{random_hex, Demo, Guests}; +use crate::common::{random_hex, Demo, DemoRoutes, Guests}; /// Unsettled facts one zone may hold before `emit_fact` reports backpressure. pub const FACT_CAPACITY: usize = 256; /// How to open the world. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct WorldOptions { - /// Directory of the per-player cell files. + /// Directory of local dev files (the persisted ticket secret, the + /// default projection bus). Durable game state is in celld. pub data_dir: PathBuf, + /// celld serving `examples/game/cells`. + pub celld: CelldOptions, /// `redis://…` for a Redis leaderboard; None = in memory. pub redis_url: Option, /// Redis key prefix (tests isolate themselves with it). @@ -34,18 +36,22 @@ pub struct WorldOptions { } impl WorldOptions { - /// Cells in `data_dir`, in-memory leaderboard. - pub fn new(data_dir: impl Into) -> Self { + /// Cells on `celld`, local files in `data_dir`, in-memory leaderboard. + pub fn new(data_dir: impl Into, celld: CelldOptions) -> Self { Self { data_dir: data_dir.into(), + celld, redis_url: None, redis_prefix: "mobpit".into(), } } - /// A fresh directory under the system temp dir. - pub fn temporary() -> Self { - Self::new(std::env::temp_dir().join(format!("game-demos-mobpit-{}", random_hex(12)))) + /// A fresh local directory under the system temp dir. + pub fn temporary(celld: CelldOptions) -> Self { + Self::new( + std::env::temp_dir().join(format!("game-demos-mobpit-{}", random_hex(12))), + celld, + ) } } @@ -55,8 +61,11 @@ pub type LoadedProfile = (String, Option<(u64, u32, u32)>); /// See the module docs. pub struct MobPitWorld { boot: String, + data_dir: PathBuf, + celld: CelldOptions, cells: Arc, sink: Arc, + fact_sink: MobPitFactSink, board: Arc, zone_pending: AtomicUsize, profiles: Arc>>, @@ -65,21 +74,33 @@ pub struct MobPitWorld { } impl MobPitWorld { - /// Open the cells directory and the leaderboard. Must be called inside a - /// Tokio runtime. Does not rebuild the board: call [`Self::rebuild_board`]. + /// Open the world over celld. Must be called inside a Tokio runtime. + /// Does not contact celld (cells are reached on first use) and does not + /// rebuild the board: call [`Self::rebuild_board`]. pub fn open(options: WorldOptions) -> Result, String> { std::fs::create_dir_all(&options.data_dir) .map_err(|e| format!("create {}: {e}", options.data_dir.display()))?; let board = Leaderboards::open(options.redis_url.as_deref(), &options.redis_prefix)?; + let sink = Arc::new(SinkState::default()); + let presence = Arc::new(Presence::default()); + let fact_sink = MobPitFactSink::new( + &options.celld, + sink.clone(), + board.clone(), + presence.clone(), + )?; Ok(Arc::new(Self { boot: random_hex(8), - cells: Arc::new(CrawlerCells::new(options.data_dir)), - sink: Arc::new(SinkState::default()), + data_dir: options.data_dir, + cells: Arc::new(CrawlerCells::new(&options.celld)?), + celld: options.celld, + sink, + fact_sink, board, zone_pending: AtomicUsize::new(0), profiles: Arc::default(), projection: Arc::default(), - presence: Arc::default(), + presence, })) } @@ -89,12 +110,17 @@ impl MobPitWorld { &self.boot } - /// Cell files directory. + /// Local dev files directory. pub fn data_dir(&self) -> &Path { - self.cells.root() + &self.data_dir + } + + /// Where the cells are. + pub fn celld(&self) -> &CelldOptions { + &self.celld } - /// The cells. + /// Read side of the cells. pub fn cells(&self) -> &Arc { &self.cells } @@ -109,42 +135,25 @@ impl MobPitWorld { &self.board } - /// Presence projected from the zones' lifecycle facts. + /// Presence mirrored from the lifecycle facts settled in presence cells. pub fn presence(&self) -> &Arc { &self.presence } /// The game's fact sink. - pub fn fact_sink(&self) -> CrawlerFactSink { - CrawlerFactSink::new( - self.cells.clone(), - self.sink.clone(), - self.board.clone(), - self.projection.clone(), - ) - .with_presence(self.presence.clone()) + pub fn fact_sink(&self) -> MobPitFactSink { + self.fact_sink.clone() } - /// Publish the cells' Crawler domain events to the SQLite bus at `url` - /// (the read-model service consumes it): drain what is pending now, then - /// after every commit, and sweep every `sweep` for anything a crash or a - /// bus outage left behind. Call once, inside a Tokio runtime. - pub async fn attach_projection_bus(&self, url: &str, sweep: Duration) -> Result { + /// Publish the Crawler domain events that celld's outbox Queue relays + /// to `/internal/celld-queue/relay` onto the SQLite bus at `url` (the + /// read-model service consumes it). Until a bus is attached the relay + /// answers 503 and the Queue keeps the messages. Call once. + pub async fn attach_projection_bus(&self, url: &str) -> Result<(), String> { let bus = ProjectionBus::open(url).await?; - let published = self.cells.drain_all(&bus).await; self.projection - .set(bus.clone()) - .map_err(|_| "projection bus already attached".to_string())?; - let cells = self.cells.clone(); - tokio::spawn(async move { - let mut every = tokio::time::interval(sweep); - every.tick().await; - loop { - every.tick().await; - cells.drain_all(&bus).await; - } - }); - Ok(published) + .set(bus) + .map_err(|_| "projection bus already attached".to_string()) } /// The attached projection bus, if any. @@ -171,10 +180,12 @@ impl MobPitWorld { return; }; handle.spawn(async move { - // A first-time player has no cell yet; do not create one to read. - let state = match cells.exists(&principal).then(|| cells.get(&principal)) { - Some(Ok(cell)) => cell.load().await.ok().flatten(), - _ => None, + let state = match cells.crawler(&principal).await { + Ok(view) => view.state, + Err(error) => { + eprintln!("mobpit: load profile {principal}: {error}"); + None + } }; profiles .lock() @@ -192,31 +203,30 @@ impl MobPitWorld { self.board.rebuild(&self.cells).await } - /// Every crawler on disk: `{principal, name, xp, kills, level, version}`. + /// Every crawler on the roster: `{principal, name, xp, kills, level, version}`. pub async fn crawlers(&self) -> Result, String> { let mut out = Vec::new(); - for principal in self.cells.principals() { - let cell = self.cells.get(&principal).map_err(|e| e.to_string())?; - if let Some(c) = cell.load().await.map_err(|e| e.to_string())? { + for principal in self.cells.principals().await? { + if let Some(c) = self.cells.crawler(&principal).await?.state { out.push(json!({ "principal": principal, "name": c.name, "xp": c.xp, - "kills": c.kills, "level": c.level, "version": c.version(), + "kills": c.kills, "level": c.level, "version": c.version, })); } } Ok(out) } - /// Game server for `config` plus the Mob Pit dev routes. + /// Game server for `config` plus the Mob Pit dev and internal routes. pub fn server( self: &Arc, config: ServerConfig, guests: Guests, - ) -> (GameServer, Router) { + ) -> (GameServer, DemoRoutes) { let world = self.clone(); let mut config = config; // Joined / connection lost / reconnected / disconnected / left facts - // for the presence projection. + // for the presence cells. config.host.lifecycle_facts = true; let server = GameServer::new( config, @@ -228,7 +238,19 @@ impl MobPitWorld { }) }), ) - .with_fact_sink(self.fact_sink(), FACT_CAPACITY, RetryPolicy::default()); - (server, routes::routes(self.clone())) + // celld down or slow: facts stay in the zone's queue until they + // settle (backpressure at capacity), never in a local store. + .with_fact_sink( + self.fact_sink(), + FACT_CAPACITY, + RetryPolicy::until_settled(Duration::from_millis(50), Duration::from_secs(2)), + ); + ( + server, + DemoRoutes { + dev: routes::routes(self.clone()), + internal: routes::internal_routes(self.clone()), + }, + ) } } diff --git a/examples/game/server/src/mobpit/zone.rs b/examples/game/server/src/mobpit/zone.rs index 619de90e5..7946f5e84 100644 --- a/examples/game/server/src/mobpit/zone.rs +++ b/examples/game/server/src/mobpit/zone.rs @@ -1271,7 +1271,12 @@ mod projection_golden { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn projected_world_and_delta_bytes_match_the_legacy_replicate() { - let world = MobPitWorld::open(WorldOptions::temporary()).unwrap(); + let celld = crate::mobpit::store::CelldOptions::new( + "http://127.0.0.1:9", + "test-only-unreachable-celld-0123456789", + ) + .unwrap(); + let world = MobPitWorld::open(WorldOptions::temporary(celld)).unwrap(); let clock = ManualClock::new(); let mut host = ZoneHost::::new( "pit-1", diff --git a/examples/game/server/tests/mobpit_smoke.rs b/examples/game/server/tests/mobpit_smoke.rs index d31089627..a87af1594 100644 --- a/examples/game/server/tests/mobpit_smoke.rs +++ b/examples/game/server/tests/mobpit_smoke.rs @@ -1,14 +1,17 @@ -//! Mob Pit + Leaderboard smoke tests. +//! Mob Pit + Leaderboard smoke tests, every one against its own `celld dev` +//! serving `examples/game/cells` (`game_demos::devcelld`; `CELLD_BIN` must +//! name celld and the worker must be built — `run.sh smoke` does both). //! -//! 1. Facts → native SQLite cells exactly once (ledger replay), server-side +//! 1. Facts → celld Crawler cells exactly once (ledger replay), server-side //! XP, trusted principal; leaderboard occurrences have the same ids live -//! and from history, so rebuild + replay never double count. +//! and from history, so rebuild + replay never double count. celld down: +//! the fact is retryable, never stored locally. //! 2. Bots fight in-process; with 1.5 s of storage latency the tick stays //! flat while pending facts climb, then everything settles and the //! boards equal the cells (and a rebuild changes nothing). //! 3. Crash recovery with the real binary: `kill -9` mid-fight with facts in -//! flight, restart on the same data dir, verify the cells and that a -//! replayed pre-crash fact key does not double-credit. +//! flight, then celld crashes too; restart both, verify the cells and +//! that a replayed pre-crash fact key does not double-credit. use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -21,6 +24,7 @@ use distributed_game::bots::{run_bots, Behaviour, BotConfig}; use distributed_game::net::TicketKey; use distributed_game::{Fact, FactError, FactSink, FactStamp, ZoneRng}; use game_demos::common::{random_hex, DemoOptions, RunningDemo}; +use game_demos::devcelld::DevCelld; use game_demos::mobpit::board::{kill_occurrence, MobKilled}; use game_demos::mobpit::crawler::RECORD_KILL; use game_demos::mobpit::{self, encode_bot_input, MobPitDemo, MobPitWorld, WorldOptions}; @@ -43,6 +47,10 @@ impl Behaviour for Spin { } } +fn celld() -> DevCelld { + DevCelld::start(None).unwrap_or_else(|error| panic!("celld dev: {error}")) +} + fn temp_dir(tag: &str) -> PathBuf { std::env::temp_dir().join(format!("mobpit-{tag}-{}", random_hex(10))) } @@ -103,7 +111,8 @@ async fn assert_board_matches_cells(world: &MobPitWorld) -> u64 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn facts_commit_once_and_board_rebuilds_identically() { let dir = temp_dir("unit"); - let world = MobPitWorld::open(WorldOptions::new(&dir)).unwrap(); + let mut celld = celld(); + let world = MobPitWorld::open(WorldOptions::new(&dir, celld.options())).unwrap(); let sink = world.fact_sink(); let input = json!({ "mob": "pit-1/m3", "tier": 2, "name": "Ana" }); @@ -157,15 +166,23 @@ async fn facts_commit_once_and_board_rebuilds_identically() { .await .unwrap(); - let cell = world.cells().get("guest-ana").unwrap(); - let ana = cell.load().await.unwrap().unwrap(); - assert_eq!((ana.kills, ana.xp, ana.level), (1, 25, 1), "credited once"); - let history = cell.history().await.unwrap(); - assert_eq!(history.len(), 1); + let ana = world.cells().crawler("guest-ana").await.unwrap(); + let state = ana.state.clone().unwrap(); assert_eq!( - history[0].sequence, + (state.kills, state.xp, state.level), + (1, 25, 1), + "credited once" + ); + assert_eq!(ana.kills.len(), 1); + assert_eq!( + ana.kills[0].sequence, first.payload["sequence"].as_u64().unwrap() ); + assert_eq!( + world.cells().principals().await.unwrap(), + vec!["guest-ana", "guest-bo"], + "enlisted on the roster before the first kill" + ); // Live updates are fire-and-forget; the replay was deduplicated. eventually("live board", Duration::from_secs(5), || async { @@ -211,6 +228,27 @@ async fn facts_commit_once_and_board_rebuilds_identically() { ); let (kills, _) = world.board().boards(); assert_eq!(world.board().store().len(kills).await.unwrap(), 2); + + // celld down: the fact is retryable (the zone keeps it queued), and it + // commits once celld is back. + celld.crash(); + let later = kill_fact( + "kill:t:pit-1:m7:0", + "guest-ana", + json!({ "mob": "pit-1/m7", "tier": 1, "name": "Ana" }), + ); + let down = sink.commit(&later).await; + assert!(matches!(down, Err(FactError::Retryable(_))), "{down:?}"); + celld.restart().unwrap(); + assert!(!sink.commit(&later).await.unwrap().replayed); + let state = world + .cells() + .crawler("guest-ana") + .await + .unwrap() + .state + .unwrap(); + assert_eq!((state.kills, state.xp), (2, 35), "durable across the crash"); let _ = std::fs::remove_dir_all(dir); } @@ -245,7 +283,8 @@ fn spin(i: usize) -> Box { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn bots_fight_slow_storage_keeps_the_tick_flat_and_boards_match_cells() { let dir = temp_dir("bots"); - let world = MobPitWorld::open(WorldOptions::new(&dir)).unwrap(); + let celld = celld(); + let world = MobPitWorld::open(WorldOptions::new(&dir, celld.options())).unwrap(); let (demo, stop) = start(world.clone()).await; // Unknown fields (e.g. an identity) are rejected on the dev routes too. @@ -380,7 +419,7 @@ struct Binary { addr: SocketAddr, } -async fn start_binary(dir: &Path) -> Binary { +async fn start_binary(dir: &Path, celld: &DevCelld) -> Binary { let mut child = Command::new(env!("CARGO_BIN_EXE_game-demos")) .args([ "mobpit", @@ -392,6 +431,8 @@ async fn start_binary(dir: &Path) -> Binary { "--data-dir", ]) .arg(dir) + .args(["--celld-url", &celld.url()]) + .env("DISTRIBUTED_INTERNAL_SECRET", celld.secret()) .env_remove("REDIS_URL") .stdout(Stdio::piped()) .stderr(Stdio::null()) @@ -418,7 +459,8 @@ async fn start_binary(dir: &Path) -> Binary { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { let dir = temp_dir("crash"); - let mut first = start_binary(&dir).await; + let mut celld = celld(); + let mut first = start_binary(&dir, &celld).await; let ws = format!("ws://{}", first.addr); // Settle some kills at full speed, then slow storage so facts are in // flight at the moment of the crash. @@ -460,9 +502,12 @@ async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { let in_flight = at_crash["pending"].as_u64().unwrap(); assert!(settled_before >= before["sink"]["committed"].as_u64().unwrap()); - // Restart on the same directory: the settled facts are all there; of the - // in-flight ones each either committed atomically or not at all. - let mut second = start_binary(&dir).await; + // celld crashes too (kill -9 of its process group) and restarts on its + // state; then the server restarts on the same directory: the settled + // facts are all there; of the in-flight ones each either committed + // atomically or not at all. + celld.restart().unwrap(); + let mut second = start_binary(&dir, &celld).await; let (_, crawlers) = http(second.addr, "GET", "/dev/mobpit/crawlers", None).await; let total: u64 = crawlers["crawlers"] .as_array() @@ -489,14 +534,23 @@ async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { second.child.kill().unwrap(); second.child.wait().unwrap(); - // Replay a pre-crash fact (same key, same input) against the files: the + // Replay a pre-crash fact (same key, same input) against the cells: the // command ledger replays it, nothing is credited twice. let recent = &before["recent"].as_array().unwrap()[0]; let key = recent["key"].as_str().unwrap(); let principal = recent["receipt"]["crawler"].as_str().unwrap(); - let world = MobPitWorld::open(WorldOptions::new(&dir)).unwrap(); - let cell = world.cells().get(principal).unwrap(); - let kills_before = cell.load().await.unwrap().unwrap().kills; + let world = MobPitWorld::open(WorldOptions::new(&dir, celld.options())).unwrap(); + let kills_of = |world: Arc| async move { + world + .cells() + .crawler(principal) + .await + .unwrap() + .state + .unwrap() + .kills + }; + let kills_before = kills_of(world.clone()).await; let replay = world .fact_sink() .commit(&kill_fact(key, principal, recent["input"].clone())) @@ -504,7 +558,7 @@ async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { .unwrap(); assert!(replay.replayed, "pre-crash key replays after the restart"); assert_eq!(replay.payload, recent["receipt"], "original receipt"); - assert_eq!(cell.load().await.unwrap().unwrap().kills, kills_before); + assert_eq!(kills_of(world.clone()).await, kills_before); let _ = std::fs::remove_dir_all(dir); } @@ -517,7 +571,8 @@ async fn redis_board_projects_and_rebuilds_when_redis_url_is_set() { return; }; let dir = temp_dir("redis"); - let mut options = WorldOptions::new(&dir); + let celld = celld(); + let mut options = WorldOptions::new(&dir, celld.options()); options.redis_url = Some(url); options.redis_prefix = format!("mobpit-test-{}", random_hex(8)); let world = MobPitWorld::open(options).unwrap(); @@ -544,7 +599,7 @@ async fn redis_board_projects_and_rebuilds_when_redis_url_is_set() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn presence_is_projected_from_the_runtime_lifecycle_facts() { +async fn presence_is_settled_in_presence_cells_from_the_lifecycle_facts() { use distributed_game::protocol::Hello; use distributed_game::{ClientMessage, ClientMirror, Role, ServerMessage, TicketClaims}; use futures_util::{SinkExt, StreamExt}; @@ -552,7 +607,8 @@ async fn presence_is_projected_from_the_runtime_lifecycle_facts() { use tokio_tungstenite::tungstenite::Message; let dir = temp_dir("presence"); - let world = MobPitWorld::open(WorldOptions::new(&dir)).unwrap(); + let celld = celld(); + let world = MobPitWorld::open(WorldOptions::new(&dir, celld.options())).unwrap(); let (demo, stop) = start(world.clone()).await; let join = |principal: &'static str| { let url = format!("{}/zones/pit-1", demo.ws_url()); @@ -632,8 +688,17 @@ async fn presence_is_projected_from_the_runtime_lifecycle_facts() { let (code, body) = http(demo.addr, "GET", "/dev/mobpit/presence", None).await; assert_eq!(code, 200); assert_eq!(body["presence"].as_array().unwrap().len(), 2, "{body}"); + // The transitions are durable in the players' presence cells. + let ana = world.cells().presence("guest-ana").await.unwrap(); + assert_eq!( + (ana["zone"].as_str(), ana["status"].as_str()), + (Some("pit-1"), Some("offline")), + "{ana}" + ); + let bo = world.cells().presence("guest-bo").await.unwrap(); + assert_eq!(bo["status"], "away", "{bo}"); // Lifecycle facts never touch the Crawler cells. - assert!(world.cells().principals().is_empty()); + assert!(world.cells().principals().await.unwrap().is_empty()); let _ = stop.send(()); let _ = std::fs::remove_dir_all(dir); } diff --git a/examples/game/sveltekit/tests/pit.test.mjs b/examples/game/sveltekit/tests/pit.test.mjs index c327df71e..8bda959fa 100644 --- a/examples/game/sveltekit/tests/pit.test.mjs +++ b/examples/game/sveltekit/tests/pit.test.mjs @@ -2,8 +2,9 @@ * Headless SvelteKit demo (production build, real `game-demos mobpit` and * real `mobpit-service`): dev session sign-in → the page joins the * server-chosen zone with a session-minted ticket → keyboard movement → - * auto-hunt kills a mob → the cell commits it, its domain event is drained to - * the bus and projected into `player_stats` → the `@live` GraphQL queries + * auto-hunt kills a mob → the celld Crawler cell commits it, its domain event + * goes through the cell outbox and celld's Queue to the zone server's relay, + * onto the bus, and is projected into `player_stats` → the `@live` GraphQL queries * (MyStats, bound to the session; Leaderboard) update without a reload, for * the killer and for a second signed-in player → the SSR HTML after a reload * has the settled numbers → no page errors. @@ -11,8 +12,10 @@ * Ports: GAME_DEMOS_PORT (default 2967), SVELTEKIT_PORT (default 5573), * MOBPIT_SERVICE_PORT (default GAME_DEMOS_PORT + 2). Needs the lifecycle * build (`distributed build examples/game/server`, which runs `vite build` - * with the generated `$distributed` client) and the binaries - * (GAME_DEMOS_BIN / MOBPIT_SERVICE_BIN, else `cargo run`). + * with the generated `$distributed` client), the binaries + * (GAME_DEMOS_BIN / MOBPIT_SERVICE_BIN, else `cargo run`), and celld + * (CELLD_BIN, with the built `examples/game/cells` worker; the test starts + * its own `celld dev`). */ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; @@ -25,6 +28,8 @@ import test from 'node:test'; import { chromium } from 'playwright'; +import { startCelld } from '../../client/tests/helpers/celld.mjs'; + const APP = fileURLToPath(new URL('..', import.meta.url)); const SERVER_DIR = fileURLToPath(new URL('../../server/', import.meta.url)); const GAME_PORT = Number(process.env.GAME_DEMOS_PORT ?? 2967); @@ -74,10 +79,14 @@ test('sign in, join, move, kill, live read model updates for two players', { tim const busUrl = `sqlite:${join(dataDir, 'bus.db')}`; const keys = apiKeys(); const bin = process.env.GAME_DEMOS_BIN; - const gameArgs = ['mobpit', '--port', String(GAME_PORT), '--data-dir', join(dataDir, 'cells'), '--projection-bus', busUrl]; + // The Crawler cells run on celld; its outbox Queue relays the domain + // events to the game server, which publishes them onto the bus. + const celld = await startCelld({ relayPort: GAME_PORT }); + const gameEnv = { ...process.env, ...celld.env, DISTRIBUTED_GAME_SECRET: secret }; + const gameArgs = ['mobpit', '--port', String(GAME_PORT), '--data-dir', join(dataDir, 'local'), '--projection-bus', busUrl]; const game = bin - ? start(bin, gameArgs, { env: { ...process.env, DISTRIBUTED_GAME_SECRET: secret } }, /LISTENING/) - : start('cargo', ['run', '-q', '--', ...gameArgs], { cwd: SERVER_DIR, env: { ...process.env, DISTRIBUTED_GAME_SECRET: secret } }, /LISTENING/); + ? start(bin, gameArgs, { env: gameEnv }, /LISTENING/) + : start('cargo', ['run', '-q', '--', ...gameArgs], { cwd: SERVER_DIR, env: gameEnv }, /LISTENING/); const serviceEnv = { ...process.env, BIND: `127.0.0.1:${SERVICE_PORT}`, @@ -245,6 +254,7 @@ test('sign in, join, move, kill, live read model updates for two players', { tim await ui.stop(); await service.stop(); await game.stop(); + await celld.stop(); rmSync(dataDir, { recursive: true, force: true }); } }); diff --git a/src/microsvc/cell_host/alarm.rs b/src/microsvc/cell_host/alarm.rs index 2c74bb608..be2cba84f 100644 --- a/src/microsvc/cell_host/alarm.rs +++ b/src/microsvc/cell_host/alarm.rs @@ -227,6 +227,7 @@ impl CellAlarm { } } + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] pub(crate) fn input_json(&self) -> Result { serde_json::to_string(&self.input) .map_err(|error| RepositoryError::Model(format!("cell alarm input: {error}"))) diff --git a/src/microsvc/cell_host/cell.rs b/src/microsvc/cell_host/cell.rs index cda5766da..488ff8b43 100644 --- a/src/microsvc/cell_host/cell.rs +++ b/src/microsvc/cell_host/cell.rs @@ -80,20 +80,6 @@ where }) } - /// Open a durable cell backed by its own SQLite file (`cell-sqlite`). - /// - /// The file is `location.resolve("{aggregate_type}:{shard_id}")`. Each - /// command commits events, snapshots, its ledger receipt and outbox rows - /// in one SQLite transaction, exactly as `from_state` does on Workers. - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn open_sqlite( - location: &super::CellSqlitePath, - shard_id: impl Into, - ) -> Result { - let shard = StreamIdentity::new(A::aggregate_type(), shard_id.into())?; - Self::from_store(CellStreamStore::open_sqlite(location, shard)?) - } - /// Host a prepared exclusive store (for example a SQLite cell configured /// with [`CellStreamStore::with_sealed_row_reducer`]). #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] @@ -380,8 +366,8 @@ where /// Durable event history of one stream this cell owns, oldest first /// (empty when the stream has no events). Reads the backing store, so it - /// works on the in-process reference host **and** on native SQLite - /// cells (`cell-sqlite`), unlike the whole-state + /// works on the in-process reference host **and** on celld (Durable + /// Object SQLite), unlike the whole-state /// [`durable_events`](Self::durable_events) export. A stream the cell /// does not own is rejected. pub async fn stream_events( @@ -673,17 +659,6 @@ where }) } - /// Open a durable SQLite cell with repository snapshot caching. - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn open_sqlite_with_snapshots( - location: &super::CellSqlitePath, - shard_id: impl Into, - frequency: u64, - ) -> Result { - let shard = StreamIdentity::new(A::aggregate_type(), shard_id.into())?; - Self::from_store_with_snapshots(CellStreamStore::open_sqlite(location, shard)?, frequency) - } - /// Host a prepared exclusive store with repository snapshot caching. #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] pub fn from_store_with_snapshots( diff --git a/src/microsvc/cell_host/mod.rs b/src/microsvc/cell_host/mod.rs index 5e3881bc7..7e5ed4cde 100644 --- a/src/microsvc/cell_host/mod.rs +++ b/src/microsvc/cell_host/mod.rs @@ -13,13 +13,12 @@ //! routes only supply kind, shard, and payload. Aggregate Workers own outbox //! delivery through celld Queue. //! -//! Durable storage is one SQLite database per cell on both targets, through -//! one shared store (`sql_store.rs`): Durable Object `storage.sql` on -//! `workers-rs`/wasm32, and a synchronous rusqlite file per cell on native -//! hosts with the `cell-sqlite` feature (`CellSqlitePath`, -//! `AggregateCell::open_sqlite`). `cell-sqlite` selects a storage backend -//! only; the cell host itself stays ungated (`PCH-DEC-005`). Without it, -//! native cells use the in-process reference host. +//! Durable storage is one SQLite database per cell: Durable Object +//! `storage.sql` on celld (`workers-rs`/wasm32, `sql_store.rs`). Durable +//! cells run on celld only, for local development and deployment alike; +//! there is no native durable cell backend. Native builds have the +//! in-process reference host, for unit tests (the cell host itself stays +//! ungated, `PCH-DEC-005`). pub(crate) mod alarm; pub(crate) mod causal; @@ -29,19 +28,9 @@ mod celld_outbox; #[cfg(feature = "graphql")] mod command; mod internal_auth; -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -mod native_sqlite; #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] mod sql_executor; -// Native durable cells reuse the Durable Object store over a rusqlite executor -// with the same `read`/`transaction` surface (see `native_sql_executor.rs`). -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -#[path = "native_sql_executor.rs"] -mod sql_executor; -#[cfg(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) -))] +#[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] mod sql_store; mod store; mod wire; @@ -73,10 +62,6 @@ pub use command::{CelldCommandHost, CelldRoute}; pub use internal_auth::{ InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, }; -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -pub use native_sqlite::{CellOutboxStore, CellSqlitePath, NATIVE_CELL_SQLITE_EXTENSION}; -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -pub use sql_store::CellSqlOutboxStore; pub use store::CellStreamStore; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] pub use store::{ diff --git a/src/microsvc/cell_host/native_sql_executor.rs b/src/microsvc/cell_host/native_sql_executor.rs deleted file mode 100644 index 3efe856e9..000000000 --- a/src/microsvc/cell_host/native_sql_executor.rs +++ /dev/null @@ -1,294 +0,0 @@ -//! Native execution of the shared cell SQL repository operations. -//! -//! This is the `cell-sqlite` counterpart of the Durable Object executor in -//! `sql_executor.rs`. Both expose the same `CellSqlConnection::{read, -//! transaction}` surface to `sql_store.rs`, so schema bootstrap, commits, -//! ledger fencing and outbox leases are one shared implementation. -//! -//! # Why a synchronous binding (rusqlite), not SQLx -//! -//! Durable Object `storage.transactionSync` runs the whole command commit in -//! one synchronous callback: nothing can yield between the domain, snapshot, -//! receipt and delivery participants. `rusqlite` gives the same shape natively: -//! `BEGIN IMMEDIATE`, the shared operations completed by `finish_sql` without -//! awaiting, then `COMMIT` (or `ROLLBACK` on any error). An async SQLx pool -//! would need a different, await-based transaction owner and would let the -//! shared store observe a different atomicity contract on each target. -//! -//! One connection per cell file is held behind a mutex; reads and transactions -//! from the same process are serialized just like Durable Object input gates. -//! SQLite calls block the calling thread for their (local, short) duration. - -use std::collections::HashMap; -use std::path::Path; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::{Duration, SystemTime}; - -use rusqlite::types::Value; -use rusqlite::Connection; - -use crate::repository::sql::{SqlBind, SqlExecutor, SqlPart, SqlRow, Statement}; -use crate::repository::{sqlite_codec, RepositoryError}; - -/// How long a writer waits for another process holding the same cell file. -const BUSY_TIMEOUT: Duration = Duration::from_secs(5); - -#[derive(Clone)] -pub(crate) struct CellSqlConnection(Arc>); - -impl CellSqlConnection { - /// Open (creating when absent) one cell's SQLite file. - pub fn open(path: &Path) -> Result { - if let Some(parent) = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - std::fs::create_dir_all(parent).map_err(|error| { - RepositoryError::permanent_storage("create cell directory", error) - })?; - } - let connection = Connection::open(path).map_err(sqlite_error)?; - connection - .busy_timeout(BUSY_TIMEOUT) - .map_err(sqlite_error)?; - // WAL keeps readers off the writer; FULL syncs every commit so an - // acknowledged command survives power loss, not only process loss. - connection - .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get::<_, String>(0)) - .map_err(sqlite_error)?; - connection - .pragma_update(None, "synchronous", "FULL") - .map_err(sqlite_error)?; - Ok(Self(Arc::new(Mutex::new(connection)))) - } - - fn lock(&self) -> MutexGuard<'_, Connection> { - match self.0.lock() { - Ok(guard) => guard, - Err(poisoned) => { - // A panic inside a transaction callback left it open. Roll it - // back so no partial command can be committed by a later one. - let guard = poisoned.into_inner(); - if !guard.is_autocommit() { - let _ = guard.execute_batch("ROLLBACK"); - } - self.0.clear_poison(); - guard - } - } - } - - /// Run reads outside an explicit transaction, serialized with commits. - pub fn read>( - &self, - operation: impl FnOnce(&mut CellSqlExecutor<'_>) -> Result, - ) -> Result { - let mut guard = self.lock(); - operation(&mut CellSqlExecutor(&mut guard)) - } - - /// Run `operation` in one `BEGIN IMMEDIATE` transaction. Any error rolls - /// back every participant; nothing is committed before `COMMIT` returns. - pub fn transaction>( - &self, - operation: impl FnOnce(&mut CellSqlExecutor<'_>) -> Result, - ) -> Result { - let mut guard = self.lock(); - guard - .execute_batch("BEGIN IMMEDIATE") - .map_err(sqlite_error)?; - match operation(&mut CellSqlExecutor(&mut guard)) { - Ok(value) => match guard.execute_batch("COMMIT") { - Ok(()) => Ok(value), - Err(error) => { - let _ = guard.execute_batch("ROLLBACK"); - Err(sqlite_error(error).into()) - } - }, - Err(error) => { - guard.execute_batch("ROLLBACK").map_err(sqlite_error)?; - Err(error) - } - } - } -} - -pub(crate) struct CellSqlExecutor<'a>(&'a mut Connection); -pub(crate) struct CellSqlRow(HashMap); - -impl CellSqlRow { - fn value(&self, column: &str) -> Result<&Value, RepositoryError> { - self.0 - .get(column) - .ok_or_else(|| RepositoryError::Model(format!("cell SQL row is missing {column}"))) - } - fn invalid(column: &str) -> RepositoryError { - RepositoryError::Model(format!( - "cell SQL column {column} has the wrong storage type" - )) - } -} - -impl SqlRow for CellSqlRow { - fn text(&self, column: &'static str) -> Result { - match self.value(column)? { - Value::Text(value) => Ok(value.clone()), - _ => Err(Self::invalid(column)), - } - } - fn integer(&self, column: &'static str) -> Result { - match self.value(column)? { - Value::Integer(value) => Ok(*value), - _ => Err(Self::invalid(column)), - } - } - fn optional_text(&self, column: &'static str) -> Result, RepositoryError> { - match self.value(column)? { - Value::Null => Ok(None), - _ => self.text(column).map(Some), - } - } - fn optional_timestamp( - &self, - column: &'static str, - ) -> Result, RepositoryError> { - match self.value(column)? { - Value::Null => Ok(None), - _ => self.timestamp(column).map(Some), - } - } - fn optional_integer(&self, column: &'static str) -> Result, RepositoryError> { - match self.value(column)? { - Value::Null => Ok(None), - _ => self.integer(column).map(Some), - } - } - fn bytes(&self, column: &'static str) -> Result, RepositoryError> { - match self.value(column)? { - Value::Blob(value) => Ok(value.clone()), - _ => Err(Self::invalid(column)), - } - } - fn timestamp(&self, column: &'static str) -> Result { - match self.value(column)? { - Value::Text(value) => sqlite_codec::decode(value), - Value::Real(value) => sqlite_codec::decode_epoch(*value), - Value::Integer(value) => sqlite_codec::decode_epoch(*value as f64), - _ => Err(Self::invalid(column)), - } - } -} - -/// Render a structural statement exactly as the Durable Object executor does: -/// same SQLite expressions, `?` placeholders and value storage classes. -fn render(statement: Statement<'_>) -> Result<(String, Vec), RepositoryError> { - let mut sql = String::new(); - let mut bindings = Vec::new(); - for part in statement.0 { - match part { - SqlPart::TimestampCompare { - column, - operator, - value, - } => { - sql.push_str(&format!( - "CAST({column} AS REAL) {operator} CAST(? AS REAL)" - )); - bindings.push(Value::Text(sqlite_codec::encode(value)?)); - } - SqlPart::LedgerNow | SqlPart::LedgerNowEpoch => { - sql.push_str("unixepoch('now','subsec')") - } - SqlPart::LedgerDeadline(duration) => { - sql.push_str("(unixepoch('now','subsec') + ?)"); - bindings.push(Value::Real(duration.as_secs_f64())); - } - SqlPart::LedgerDeadlineIsLive(deadline) => { - sql.push_str("CAST(? AS REAL) > unixepoch('now','subsec')"); - bindings.push(Value::Text(sqlite_codec::encode(deadline)?)); - } - SqlPart::LedgerJson(json) => { - sql.push('?'); - bindings.push(Value::Text(json)); - } - SqlPart::Sql(text) => sql.push_str(&text), - SqlPart::Bind(value) => { - sql.push('?'); - bindings.push(match value { - SqlBind::Text(value) | SqlBind::Metadata(value) => Value::Text(value), - SqlBind::Bytes(value) => Value::Blob(value.into_owned()), - SqlBind::Integer(value) => Value::Integer(value), - SqlBind::Timestamp(value) => Value::Text(sqlite_codec::encode(value)?), - }); - } - } - } - Ok((sql, bindings)) -} - -impl CellSqlExecutor<'_> { - fn query_rows(&mut self, statement: Statement<'_>) -> Result, RepositoryError> { - let (sql, bindings) = render(statement)?; - let mut prepared = self.0.prepare_cached(&sql).map_err(sqlite_error)?; - let names = prepared - .column_names() - .into_iter() - .map(String::from) - .collect::>(); - let mut rows = prepared - .query(rusqlite::params_from_iter(bindings)) - .map_err(sqlite_error)?; - let mut result = Vec::new(); - while let Some(row) = rows.next().map_err(sqlite_error)? { - let mut values = HashMap::with_capacity(names.len()); - for (index, name) in names.iter().enumerate() { - values.insert( - name.clone(), - row.get::<_, Value>(index).map_err(sqlite_error)?, - ); - } - result.push(CellSqlRow(values)); - } - Ok(result) - } - - fn execute_statement(&mut self, statement: Statement<'_>) -> Result { - let (sql, bindings) = render(statement)?; - if bindings.is_empty() { - // Validated migrations run as one whole script, as `storage.sql.exec` - // does; never split SQL text on semicolons. - self.0.execute_batch(&sql).map_err(sqlite_error)?; - return Ok(self.0.changes()); - } - let mut prepared = self.0.prepare_cached(&sql).map_err(sqlite_error)?; - let written = prepared - .execute(rusqlite::params_from_iter(bindings)) - .map_err(sqlite_error)?; - Ok(written as u64) - } -} - -impl SqlExecutor for CellSqlExecutor<'_> { - type Row = CellSqlRow; - const EVENT_SELECT: &'static str = sqlite_codec::EVENT_SELECT; - const SNAPSHOT_SELECT: &'static str = sqlite_codec::SNAPSHOT_SELECT; - const OUTBOX_SELECT: &'static str = sqlite_codec::OUTBOX_SELECT; - const NOW: &'static str = "CURRENT_TIMESTAMP"; - const COMMAND_LEDGER_SELECT: &'static str = sqlite_codec::COMMAND_LEDGER_SELECT; - const COMMAND_LEDGER_LOCK_SUFFIX: &'static str = ""; - const COMMAND_LEDGER_COMPACTION_LOCK_SUFFIX: &'static str = ""; - - async fn query(&mut self, statement: Statement<'_>) -> Result, RepositoryError> { - self.query_rows(statement) - } - - async fn execute(&mut self, statement: Statement<'_>) -> Result { - self.execute_statement(statement) - } -} - -fn sqlite_error(error: rusqlite::Error) -> RepositoryError { - // Match the Durable Object adapter: storage failures are never allowed to - // permanently discard a command or pending delivery. - RepositoryError::retryable_storage("cell SQL", error) -} diff --git a/src/microsvc/cell_host/native_sqlite.rs b/src/microsvc/cell_host/native_sqlite.rs deleted file mode 100644 index bc9abf8f8..000000000 --- a/src/microsvc/cell_host/native_sqlite.rs +++ /dev/null @@ -1,362 +0,0 @@ -//! Native durable cell storage (`cell-sqlite`). -//! -//! Deployment model: one SQLite file per cell instance, the native analogue of -//! one Durable Object per `{aggregate_type}:{shard}`. [`CellSqlitePath`] -//! derives the file from a root directory and the sanitized instance name, or -//! names one file explicitly. Every command commits its events, snapshots, -//! command-ledger receipt, outbox rows and (optionally) the sealed row in one -//! SQLite transaction through the store shared with Workers (`sql_store.rs`). -//! -//! The in-process reference host ([`CellStreamStore::new`] and friends) is -//! unchanged and still backs the whole-state export used by conformance -//! fixtures; the SQLite backend is selected only by the `open_sqlite*` -//! constructors. -//! -//! [`CellStreamStore::new`]: super::CellStreamStore::new - -use std::path::{Path, PathBuf}; - -use sha2::{Digest, Sha256}; - -use super::sql_store::{CellSqlOutboxStore, CellSqlRepository}; -use crate::command_ledger::{ - AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, - CausalStorageIdentity, CausalTransactionalCommit, CommandLedgerError, CommandLedgerKey, - CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, - ExternalCommandCompletion, ReservationOutcome, -}; -use crate::entity::Entity; -use crate::microsvc::HasOutboxStore; -use crate::outbox::{OutboxMessage, OutboxMessageStatus}; -use crate::outbox_worker::{ClaimOutboxMessages, OutboxBacklogStats, OutboxClaimRef, OutboxStore}; -use crate::repository::{ - CommitBatch, GetStream, RepositoryError, SnapshotStore, StreamIdentity, TransactionalCommit, -}; -use crate::snapshot::SnapshotRecord; -use crate::{InMemoryOutboxStore, InMemoryRepository}; - -/// File extension used for cell files derived from a root directory. -pub const NATIVE_CELL_SQLITE_EXTENSION: &str = "sqlite3"; - -/// Longest sanitized instance name kept verbatim in a derived file name. -const MAX_FILE_STEM: usize = 120; - -/// Where a native cell keeps its SQLite file. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum CellSqlitePath { - /// One file per cell under this directory: - /// `{root}/{sanitized instance name}.sqlite3`. - Root(PathBuf), - /// This exact file. The file is bound to the first cell that opens it. - File(PathBuf), -} - -impl CellSqlitePath { - /// Derive one file per cell instance under `dir`. - pub fn root(dir: impl Into) -> Self { - Self::Root(dir.into()) - } - - /// Use exactly `path` for one cell. - pub fn file(path: impl Into) -> Self { - Self::File(path.into()) - } - - /// The database file for `instance_name` (`{aggregate_type}:{shard}`). - pub fn resolve(&self, instance_name: &str) -> PathBuf { - match self { - Self::Root(root) => root.join(Self::file_name(instance_name)), - Self::File(path) => path.clone(), - } - } - - /// Injective, filesystem-safe file name for an instance name. - /// - /// `[A-Za-z0-9_-]` and non-leading `.` are kept; every other byte is - /// percent-encoded, so `game:a` and `game_a` never share a file. Very long - /// names keep a readable prefix plus a SHA-256 suffix of the full name. - pub fn file_name(instance_name: &str) -> String { - let mut stem = String::with_capacity(instance_name.len()); - for (index, byte) in instance_name.bytes().enumerate() { - let keep = byte.is_ascii_alphanumeric() - || byte == b'-' - || byte == b'_' - || (byte == b'.' && index != 0); - if keep { - stem.push(byte as char); - } else { - stem.push_str(&format!("%{byte:02X}")); - } - } - if stem.is_empty() { - stem.push_str("%00"); - } - if stem.len() > MAX_FILE_STEM { - let digest = format!("{:x}", Sha256::digest(instance_name.as_bytes())); - let mut cut = MAX_FILE_STEM - 33; - while !stem.is_char_boundary(cut) { - cut -= 1; - } - stem = format!("{}~{}", &stem[..cut], &digest[..32]); - } - format!("{stem}.{NATIVE_CELL_SQLITE_EXTENSION}") - } - - pub(super) fn open(&self, instance_name: &str) -> Result { - CellSqlRepository::open(Path::new(&self.resolve(instance_name)), instance_name) - } -} - -/// Native cell backend: the in-process reference host or a SQLite file. -#[derive(Clone)] -pub(super) enum NativeCellRepository { - Memory(InMemoryRepository), - Sql(CellSqlRepository), -} - -impl NativeCellRepository { - pub(super) fn memory(&self) -> Result<&InMemoryRepository, RepositoryError> { - match self { - Self::Memory(memory) => Ok(memory), - Self::Sql(_) => Err(RepositoryError::Model( - "SQLite cells persist rows directly; whole-state export/restore is only \ - available on the in-process reference host" - .into(), - )), - } - } - - pub(super) fn sql(&self) -> Option<&CellSqlRepository> { - match self { - Self::Sql(sql) => Some(sql), - Self::Memory(_) => None, - } - } - - pub(super) fn sql_mut(&mut self) -> Option<&mut CellSqlRepository> { - match self { - Self::Sql(sql) => Some(sql), - Self::Memory(_) => None, - } - } -} - -macro_rules! delegate { - ($self:ident, $inner:ident => $call:expr) => { - match $self { - NativeCellRepository::Memory($inner) => $call, - NativeCellRepository::Sql($inner) => $call, - } - }; -} - -impl GetStream for NativeCellRepository { - async fn get_stream( - &self, - identity: &StreamIdentity, - ) -> Result, RepositoryError> { - delegate!(self, inner => GetStream::get_stream(inner, identity).await) - } - - async fn get_stream_tail( - &self, - identity: &StreamIdentity, - after_version: u64, - ) -> Result, RepositoryError> { - delegate!(self, inner => GetStream::get_stream_tail(inner, identity, after_version).await) - } -} - -impl CausalGetStream for NativeCellRepository { - async fn get_causal_stream( - &self, - identity: &StreamIdentity, - ) -> Result, RepositoryError> { - delegate!(self, inner => CausalGetStream::get_causal_stream(inner, identity).await) - } - - async fn get_causal_stream_tail( - &self, - identity: &StreamIdentity, - after_version: u64, - ) -> Result, RepositoryError> { - delegate!(self, inner => inner.get_causal_stream_tail(identity, after_version).await) - } -} - -impl SnapshotStore for NativeCellRepository { - async fn get_snapshot( - &self, - identity: &StreamIdentity, - ) -> Result, RepositoryError> { - delegate!(self, inner => SnapshotStore::get_snapshot(inner, identity).await) - } - - async fn get_snapshots( - &self, - identities: &[StreamIdentity], - ) -> Result, RepositoryError> { - delegate!(self, inner => SnapshotStore::get_snapshots(inner, identities).await) - } - - async fn save_snapshot( - &self, - identity: &StreamIdentity, - record: SnapshotRecord, - ) -> Result<(), RepositoryError> { - delegate!(self, inner => SnapshotStore::save_snapshot(inner, identity, record).await) - } - - async fn delete_snapshot(&self, identity: &StreamIdentity) -> Result { - delegate!(self, inner => SnapshotStore::delete_snapshot(inner, identity).await) - } -} - -impl CausalRepositoryIdentity for NativeCellRepository { - fn causal_storage_identity(&self) -> CausalStorageIdentity { - delegate!(self, inner => CausalRepositoryIdentity::causal_storage_identity(inner)) - } -} - -impl CommandLedgerStore for NativeCellRepository { - async fn reserve_command( - &self, - reservation: CommandReservation, - ) -> Result { - delegate!(self, inner => CommandLedgerStore::reserve_command(inner, reservation).await) - } - - async fn lookup_command( - &self, - key: &CommandLedgerKey, - scope: CommandLookupScope<'_>, - ) -> Result { - delegate!(self, inner => CommandLedgerStore::lookup_command(inner, key, scope).await) - } - - async fn mark_retryable_unknown( - &self, - attempt: AttemptFence, - ) -> Result<(), CommandLedgerError> { - delegate!(self, inner => CommandLedgerStore::mark_retryable_unknown(inner, attempt).await) - } - - async fn complete_external_command( - &self, - completion: ExternalCommandCompletion, - ) -> Result<(), CommandLedgerError> { - delegate!(self, inner => CommandLedgerStore::complete_external_command(inner, completion).await) - } - - async fn compact_expired_commands(&self, limit: usize) -> Result { - delegate!(self, inner => CommandLedgerStore::compact_expired_commands(inner, limit).await) - } -} - -impl TransactionalCommit for NativeCellRepository { - async fn commit_batch(&self, batch: CommitBatch<'_>) -> Result<(), RepositoryError> { - delegate!(self, inner => TransactionalCommit::commit_batch(inner, batch).await) - } -} - -impl CausalTransactionalCommit for NativeCellRepository { - async fn commit_causal_batch( - &self, - batch: CausalCommitBatch<'_>, - ) -> Result<(), CommandLedgerError> { - delegate!(self, inner => CausalTransactionalCommit::commit_causal_batch(inner, batch).await) - } -} - -impl HasOutboxStore for NativeCellRepository { - type OutboxStore = CellOutboxStore; - - fn outbox_store(&self) -> CellOutboxStore { - match self { - Self::Memory(inner) => CellOutboxStore::Memory(inner.outbox_store()), - Self::Sql(inner) => CellOutboxStore::Sql(HasOutboxStore::outbox_store(inner)), - } - } -} - -/// Outbox of a native cell: the reference host's rows or the cell's SQLite -/// `outbox_messages` table (lease-fenced claim, publish, delete). -#[derive(Clone)] -pub enum CellOutboxStore { - Memory(InMemoryOutboxStore), - Sql(CellSqlOutboxStore), -} - -macro_rules! delegate_outbox { - ($self:ident, $inner:ident => $call:expr) => { - match $self { - CellOutboxStore::Memory($inner) => $call, - CellOutboxStore::Sql($inner) => $call, - } - }; -} - -impl OutboxStore for CellOutboxStore { - async fn messages_by_status( - &self, - status: OutboxMessageStatus, - limit: usize, - ) -> Result, RepositoryError> { - delegate_outbox!(self, inner => inner.messages_by_status(status, limit).await) - } - - async fn backlog_stats(&self) -> Result { - delegate_outbox!(self, inner => inner.backlog_stats().await) - } - - async fn claim( - &self, - request: ClaimOutboxMessages, - ) -> Result, RepositoryError> { - delegate_outbox!(self, inner => inner.claim(request).await) - } - - async fn complete(&self, claim: &OutboxClaimRef) -> Result<(), RepositoryError> { - delegate_outbox!(self, inner => inner.complete(claim).await) - } - - async fn complete_many(&self, claims: &[OutboxClaimRef]) -> Result<(), RepositoryError> { - delegate_outbox!(self, inner => inner.complete_many(claims).await) - } - - async fn release(&self, claim: &OutboxClaimRef, error: &str) -> Result<(), RepositoryError> { - delegate_outbox!(self, inner => inner.release(claim, error).await) - } - - async fn fail(&self, claim: &OutboxClaimRef, error: &str) -> Result<(), RepositoryError> { - delegate_outbox!(self, inner => inner.fail(claim, error).await) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn file_names_are_safe_and_injective() { - assert_eq!(CellSqlitePath::file_name("game:g1"), "game%3Ag1.sqlite3"); - assert_ne!( - CellSqlitePath::file_name("game:a"), - CellSqlitePath::file_name("game_a") - ); - assert_eq!(CellSqlitePath::file_name(".."), "%2E..sqlite3"); - assert_eq!(CellSqlitePath::file_name("a/../b"), "a%2F..%2Fb.sqlite3"); - let long = format!("Room:{}", "x".repeat(400)); - let name = CellSqlitePath::file_name(&long); - assert!(name.len() < 140, "{name}"); - assert_ne!(name, CellSqlitePath::file_name(&format!("{long}y"))); - assert_eq!( - CellSqlitePath::root("/data/cells").resolve("Room:r1"), - PathBuf::from("/data/cells/Room%3Ar1.sqlite3") - ); - assert_eq!( - CellSqlitePath::file("/tmp/one.db").resolve("Room:r1"), - PathBuf::from("/tmp/one.db") - ); - } -} diff --git a/src/microsvc/cell_host/sql_store.rs b/src/microsvc/cell_host/sql_store.rs index 43ba47df4..dd651a96e 100644 --- a/src/microsvc/cell_host/sql_store.rs +++ b/src/microsvc/cell_host/sql_store.rs @@ -1,11 +1,10 @@ //! Cell-local execution of the ordinary SQL event/snapshot/receipt repository. //! No in-memory working copy and no whole-cell export are used here. //! -//! Shared by both cell executors: Durable Object `storage.sql` on -//! `workers-rs`/wasm32 and a synchronous rusqlite file on native -//! `cell-sqlite`. Each executor supplies only `read` and a synchronous -//! all-or-nothing `transaction`; schema bootstrap, the command commit, ledger -//! fencing and outbox leases below are identical on both targets. +//! Runs over Durable Object `storage.sql` (celld, `workers-rs`/wasm32): the +//! executor supplies only `read` and a synchronous all-or-nothing +//! `transaction`; schema bootstrap, the command commit, ledger fencing and +//! outbox leases are below. use std::future::Future; use std::task::{Context, Poll, Waker}; @@ -57,15 +56,10 @@ pub(super) fn finish_sql>( } } -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -use super::store::SealedRowReducer; - #[derive(Clone)] pub(super) struct CellSqlRepository { connection: CellSqlConnection, identity: CausalStorageIdentity, - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - sealed_row_reducer: Option, } impl CellSqlRepository { @@ -79,38 +73,6 @@ impl CellSqlRepository { }) } - /// Open one native cell file. The file is bound to `instance_name` on - /// first open; reopening it as another cell fails closed. - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn open(path: &std::path::Path, instance_name: &str) -> Result { - let connection = CellSqlConnection::open(path)?; - Self::initialize(&connection)?; - connection - .transaction(|executor| finish_sql(native::bind_instance(executor, instance_name)))?; - Ok(Self { - connection, - identity: CausalStorageIdentity::new(), - sealed_row_reducer: None, - }) - } - - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn set_sealed_row_reducer(&mut self, reducer: SealedRowReducer) { - self.sealed_row_reducer = Some(reducer); - } - - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn sealed_row(&self) -> Result, RepositoryError> { - self.connection - .read(|executor| finish_sql(native::sealed_row(executor))) - } - - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn replace_sealed_row(&self, row: &serde_json::Value) -> Result<(), RepositoryError> { - self.connection - .transaction(|executor| finish_sql(native::write_sealed_row(executor, row))) - } - fn initialize(connection: &CellSqlConnection) -> Result<(), RepositoryError> { connection.transaction(|executor| finish_sql(async { let tables = executor.query(Statement::new("SELECT name FROM sqlite_master WHERE type = 'table'")).await?; @@ -211,17 +173,6 @@ impl CellSqlRepository { } // Native sealed read-model row: same transaction, so a reducer // error or lost ledger lease rolls the row back with the events. - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - if let Some(reducer) = &self.sealed_row_reducer { - if prepared.iter().any(|append| !append.events.is_empty()) { - let current = native::sealed_row(executor).await?; - if let Some(next) = reducer(current.as_ref(), &prepared)? { - if current.as_ref() != Some(&next) { - native::write_sealed_row(executor, &next).await?; - } - } - } - } // Same final fenced write as SQLx. A lost lease rolls back every // participant, and entities are marked committed only afterwards. if let Some(completion) = &completion { @@ -473,93 +424,6 @@ impl CellSqlOutboxStore { } } -/// Native-only cell metadata: the instance binding and the sealed row. Kept -/// out of the shared migration inventory so Durable Object schemas (and their -/// checksummed history) are unchanged by the native store. -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -mod native { - use super::*; - use serde_json::Value; - - const CREATE: &str = "CREATE TABLE IF NOT EXISTS __distributed_native_cell (id INTEGER PRIMARY KEY CHECK (id = 1), instance_name TEXT NOT NULL, sealed_row TEXT)"; - - pub(super) async fn bind_instance( - executor: &mut impl SqlExecutor, - instance_name: &str, - ) -> Result<(), RepositoryError> { - executor.execute(Statement::new(CREATE)).await?; - let rows = executor - .query(Statement::new( - "SELECT instance_name FROM __distributed_native_cell WHERE id = 1", - )) - .await?; - match rows.first() { - Some(row) => { - let bound = row.text("instance_name")?; - if bound != instance_name { - return Err(RepositoryError::Model(format!( - "cell SQLite file belongs to `{bound}`, not `{instance_name}`" - ))); - } - } - None => { - executor - .execute( - Statement::new( - "INSERT INTO __distributed_native_cell (id, instance_name) VALUES (1, ", - ) - .bind(instance_name.into()) - .sql(")"), - ) - .await?; - } - } - Ok(()) - } - - pub(super) async fn sealed_row( - executor: &mut impl SqlExecutor, - ) -> Result, RepositoryError> { - let rows = executor - .query(Statement::new( - "SELECT sealed_row FROM __distributed_native_cell WHERE id = 1", - )) - .await?; - match rows - .first() - .map(|row| row.optional_text("sealed_row")) - .transpose()? - .flatten() - { - Some(json) => serde_json::from_str(&json).map(Some).map_err(|error| { - RepositoryError::Model(format!("cell sealed row is invalid JSON: {error}")) - }), - None => Ok(None), - } - } - - pub(super) async fn write_sealed_row( - executor: &mut impl SqlExecutor, - row: &Value, - ) -> Result<(), RepositoryError> { - let json = serde_json::to_string(row) - .map_err(|error| RepositoryError::Model(format!("cell sealed row: {error}")))?; - let written = executor - .execute( - Statement::new("UPDATE __distributed_native_cell SET sealed_row = ") - .bind(json.as_str().into()) - .sql(" WHERE id = 1"), - ) - .await?; - if written != 1 { - return Err(RepositoryError::Model( - "cell SQLite file has no instance binding".into(), - )); - } - Ok(()) - } -} - /// Cell alarm rows (`cell_alarms`, migration 11). Shared by Durable Object and /// native SQLite cells; the reference host mirrors these semantics in memory. mod alarms { diff --git a/src/microsvc/cell_host/store.rs b/src/microsvc/cell_host/store.rs index 62d7bd9f2..dee9e7136 100644 --- a/src/microsvc/cell_host/store.rs +++ b/src/microsvc/cell_host/store.rs @@ -1,7 +1,7 @@ //! One cell's private command-side repository, with enforced stream ownership. -//! Workers use runtime-owned SQLite. Native hosts use the in-process reference -//! host, or with `cell-sqlite` one SQLite file per cell sharing the Workers -//! store (`open_sqlite*`). Ownership checks below apply to every backend. +//! Workers (celld) use runtime-owned SQLite. Native builds have only the +//! in-process reference host, for unit tests; durable cells run on celld. +//! Ownership checks below apply to both. use std::future::Future; use std::sync::Arc; @@ -34,13 +34,8 @@ use crate::repository::{ use crate::snapshot::SnapshotRecord; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] use crate::InMemoryRepository; -#[cfg(not(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) -)))] +#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] type CellRepository = InMemoryRepository; -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -type CellRepository = super::native_sqlite::NativeCellRepository; #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] type CellRepository = CellSqlRepository; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] @@ -219,51 +214,6 @@ impl CellStreamStore { } } - /// Open a durable exclusive cell backed by its own SQLite file. - /// - /// The file is `location.resolve("{type}:{id}")`; every command commits - /// events, snapshots, ledger receipt, outbox rows and the sealed row (see - /// [`Self::with_sealed_row_reducer`]) in one SQLite transaction. Dropping - /// every handle and reopening the same file restores the identical cell. - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn open_sqlite( - location: &super::CellSqlitePath, - identity: StreamIdentity, - ) -> Result { - let sql = location.open(&identity.to_string())?; - Ok(Self::native( - CellOwnership::Exclusive(identity), - super::native_sqlite::NativeCellRepository::Sql(sql), - )) - } - - /// Open a durable parent-shard cell (`"{parent_type}:{parent_id}"`) whose - /// owned sibling streams share one SQLite file. As on Workers, the - /// predicate never grants access to another cell's file. - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn open_sqlite_parent( - location: &super::CellSqlitePath, - parent_type: impl Into, - parent_id: impl Into, - owns: impl Fn(&StreamIdentity) -> bool + Send + Sync + 'static, - ) -> Result { - let name = StreamIdentity::new(parent_type, parent_id)?; - let sql = location.open(&name.to_string())?; - Ok(Self::native( - CellOwnership::Parent { - name, - owns: Arc::new(owns), - }, - super::native_sqlite::NativeCellRepository::Sql(sql), - )) - } - - /// Whether this cell persists to a SQLite file (`open_sqlite*`). - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - pub fn is_sqlite(&self) -> bool { - self.inner.sql().is_some() - } - /// Maintain the sealed read-model row from each commit's events, inside /// the command's transaction (SQLite cells) or alongside its in-memory /// commit (reference host). A reducer error rejects the command. @@ -276,10 +226,6 @@ impl CellStreamStore { + 'static, ) -> Self { let reducer: SealedRowReducer = Arc::new(reducer); - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - if let Some(sql) = self.inner.sql_mut() { - sql.set_sealed_row_reducer(Arc::clone(&reducer)); - } self.sealed_row_reducer = Some(reducer); self } @@ -287,9 +233,6 @@ impl CellStreamStore { /// The in-process reference repository backing whole-state export. #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] fn memory(&self) -> Result<&InMemoryRepository, RepositoryError> { - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - return self.inner.memory(); - #[cfg(not(all(feature = "cell-sqlite", not(target_arch = "wasm32"))))] Ok(&self.inner) } @@ -375,10 +318,6 @@ impl CellStreamStore { /// Sealed read-model row for GET on this cell instance. #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] pub fn sealed_row(&self) -> Result, RepositoryError> { - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - if let Some(sql) = self.inner.sql() { - return sql.sealed_row(); - } self.sealed_row .lock() .map(|guard| guard.clone()) @@ -388,10 +327,6 @@ impl CellStreamStore { /// Replace the sealed read-model row (Atomic board / Todo view). #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] pub fn replace_sealed_row(&self, row: Value) -> Result<(), RepositoryError> { - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - if let Some(sql) = self.inner.sql() { - return sql.replace_sealed_row(&row); - } let mut guard = self .sealed_row .lock() @@ -577,33 +512,21 @@ impl CellStreamStore { .map_err(|_| RepositoryError::Model("cell alarm table lock poisoned".into())) } - /// The SQL database holding alarms (Durable Object or native SQLite); - /// `None` on the reference host, which keeps them in memory. - #[cfg(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) - ))] + /// The SQL database holding alarms (the Durable Object's); `None` on + /// the reference host, which keeps them in memory. + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] fn alarm_sql(&self) -> Option<&super::sql_store::CellSqlRepository> { - #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] - return Some(&self.inner); - #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] - self.inner.sql() + Some(&self.inner) } - #[cfg(not(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) - )))] + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] fn alarm_sql(&self) -> Option<&std::convert::Infallible> { None } /// Armed alarms, earliest first. pub fn armed_alarms(&self) -> Result, RepositoryError> { - #[cfg(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) - ))] + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] if let Some(sql) = self.alarm_sql() { return sql.armed_alarms(); } @@ -619,10 +542,7 @@ impl CellStreamStore { now_ms: i64, limit: usize, ) -> Result, RepositoryError> { - #[cfg(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) - ))] + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] if let Some(sql) = self.alarm_sql() { return sql.due_alarms(now_ms, limit); } @@ -634,10 +554,7 @@ impl CellStreamStore { /// Due time (epoch ms) of the earliest armed alarm. pub(super) fn next_alarm_due_ms(&self) -> Result, RepositoryError> { - #[cfg(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) - ))] + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] if let Some(sql) = self.alarm_sql() { return sql.next_alarm_due_ms(); } @@ -653,10 +570,7 @@ impl CellStreamStore { generation: u64, settlement: &super::alarm::AlarmSettlement, ) -> Result<(), RepositoryError> { - #[cfg(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) - ))] + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] if let Some(sql) = self.alarm_sql() { return sql.settle_alarm(name, generation, settlement); } @@ -900,13 +814,8 @@ impl CausalTransactionalCommit for CellStreamStore { } impl HasOutboxStore for CellStreamStore { - #[cfg(not(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) - )))] + #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] type OutboxStore = crate::InMemoryOutboxStore; - #[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] - type OutboxStore = super::native_sqlite::CellOutboxStore; #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] type OutboxStore = super::sql_store::CellSqlOutboxStore; @@ -930,15 +839,7 @@ impl crate::microsvc::dependencies::CausalHostProjections for CellStreamStore { } } -#[cfg(not(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) -)))] +#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] fn memory_repository() -> CellRepository { InMemoryRepository::new() } - -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -fn memory_repository() -> CellRepository { - super::native_sqlite::NativeCellRepository::Memory(InMemoryRepository::new()) -} diff --git a/src/microsvc/cell_host/tests.rs b/src/microsvc/cell_host/tests.rs index 2c901143c..389367969 100644 --- a/src/microsvc/cell_host/tests.rs +++ b/src/microsvc/cell_host/tests.rs @@ -670,5 +670,3 @@ async fn reference_host_applies_the_sealed_row_reducer_on_commit() { } mod alarms; -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -mod native_sqlite; diff --git a/src/microsvc/cell_host/tests/alarms.rs b/src/microsvc/cell_host/tests/alarms.rs index 5e69293be..db2748adc 100644 --- a/src/microsvc/cell_host/tests/alarms.rs +++ b/src/microsvc/cell_host/tests/alarms.rs @@ -461,63 +461,6 @@ async fn repositories_without_an_alarm_slot_reject_alarm_writes() { assert!(error.to_string().contains("aggregate cell"), "{error}"); } -/// The SQL alarm table shared with Durable Object cells, on a native SQLite -/// file: same transaction as the events, and armed rows survive a reopen. -#[cfg(all(feature = "cell-sqlite", not(target_arch = "wasm32")))] -#[tokio::test] -async fn sqlite_cell_alarms_commit_atomically_and_survive_reopening() { - use super::super::CellSqlitePath; - - let root = - std::env::temp_dir().join(format!("distributed-cell-alarms-{}", uuid::Uuid::now_v7())); - let location = CellSqlitePath::root(&root); - let clock = clock(); - let open = |clock: &ManualClock| { - let store = - CellStreamStore::open_sqlite(&location, StreamIdentity::new("Ticker", "t1").unwrap()) - .unwrap() - .with_clock(clock.clone()); - mount(AggregateCell::from_store(store).unwrap()) - }; - - let cell = open(&clock); - let refused = cell - .dispatch_idempotent( - "ticker.start", - &identity("0190a000-0000-7000-8000-000000000002"), - json!({ "id": "t1", "limit": 3, "period_ms": 1000, "fail_after_arming": true, "tick_command": "ticker.tick" }), - owner_session(), - ) - .await; - assert!(refused.is_err()); - assert!(cell.armed_alarms().unwrap().is_empty()); - start(&cell, json!({ "id": "t1", "limit": 3, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; - drop(cell); - - let cell = open(&clock); - assert_eq!(cell.armed_alarms().unwrap().len(), 1); - clock.advance(Duration::from_secs(1)); - assert_eq!(cell.fire_due_alarms().await.unwrap().fired.len(), 1); - drop(cell); - - // Restart between ticks: the re-armed generation is still there, once. - let cell = open(&clock); - let armed = cell.armed_alarms().unwrap(); - assert_eq!(armed.len(), 1); - assert_eq!(armed[0].generation, 2); - assert!(cell.fire_due_alarms().await.unwrap().is_empty()); - for _ in 0..2 { - clock.advance(Duration::from_secs(1)); - assert_eq!(cell.fire_due_alarms().await.unwrap().fired.len(), 1); - } - assert_eq!(ticks(&cell).await, 3); - assert!(cell.armed_alarms().unwrap().is_empty()); - clock.advance(Duration::from_secs(60)); - assert!(cell.fire_due_alarms().await.unwrap().is_empty()); - drop(cell); - let _ = std::fs::remove_dir_all(root); -} - #[tokio::test] async fn handlers_see_the_cell_clock_and_callers_cannot_forge_it() { let clock = clock(); diff --git a/src/microsvc/cell_host/tests/native_sqlite.rs b/src/microsvc/cell_host/tests/native_sqlite.rs deleted file mode 100644 index bac239691..000000000 --- a/src/microsvc/cell_host/tests/native_sqlite.rs +++ /dev/null @@ -1,471 +0,0 @@ -//! Native durable cells (`cell-sqlite`): restart, replay, outbox and -//! ownership conformance against real SQLite files. - -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use serde_json::{json, Value}; - -use super::super::{ - AggregateCell, CellCommandIdentity, CellDispatchError, CellSqlitePath, CellStreamStore, -}; -use super::{owner_session, CellItem, Complete, Create, Publish}; -use crate::bus::{Message, MessagePublisher, TransportError}; -use crate::entity::{Entity, EventRecord}; -use crate::microsvc::HasOutboxStore; -use crate::outbox::{OutboxMessage, OutboxMessageStatus}; -use crate::outbox_worker::{ClaimOutboxMessages, OutboxClaimRef, OutboxStore}; -use crate::repository::{ - CommitBatch, GetStream, PreparedEventAppend, RepositoryError, SnapshotStore, StreamIdentity, - StreamWrite, TransactionalCommit, -}; -use crate::snapshot::SnapshotRecord; - -struct TempRoot(PathBuf); - -impl TempRoot { - fn new() -> Self { - Self(std::env::temp_dir().join(format!("distributed-cell-sqlite-{}", uuid::Uuid::now_v7()))) - } - - fn location(&self) -> CellSqlitePath { - CellSqlitePath::root(&self.0) - } -} - -impl Drop for TempRoot { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.0); - } -} - -#[derive(Debug, PartialEq)] -struct CellDump { - events: Vec, - snapshot: Option, - outbox: Vec, - sealed: Option, -} - -/// Read every durable concern of one exclusive cell through a fresh handle, -/// which is dropped again before returning. -async fn dump(location: &CellSqlitePath, shard: &str) -> CellDump { - let identity = StreamIdentity::new("CellItem", shard).unwrap(); - let store = CellStreamStore::open_sqlite(location, identity.clone()).unwrap(); - assert!(store.is_sqlite()); - let events = GetStream::get_stream(&store, &identity) - .await - .unwrap() - .map(|entity| entity.events().to_vec()) - .unwrap_or_default(); - let snapshot = SnapshotStore::get_snapshot(&store, &identity) - .await - .unwrap(); - let outbox_store = store.outbox_store(); - let mut outbox = Vec::new(); - for status in [ - OutboxMessageStatus::Pending, - OutboxMessageStatus::InFlight, - OutboxMessageStatus::Failed, - ] { - outbox.extend(outbox_store.messages_by_status(status, 100).await.unwrap()); - } - CellDump { - events, - snapshot, - outbox, - sealed: store.sealed_row().unwrap(), - } -} - -fn count_events( - current: Option<&Value>, - appends: &[PreparedEventAppend<'_>], -) -> Result, RepositoryError> { - let mut count = current.and_then(|row| row["events"].as_u64()).unwrap_or(0); - let mut last = current - .map(|row| row["last"].clone()) - .unwrap_or(Value::Null); - for append in appends { - for event in append.events { - count += 1; - last = Value::String(event.event_name.clone()); - } - } - Ok(Some(json!({ "events": count, "last": last }))) -} - -fn identity(command_id: &str) -> CellCommandIdentity { - CellCommandIdentity::new("cell-test-service", "principal-alice", command_id).unwrap() -} - -fn sqlite_cell(location: &CellSqlitePath, shard: &str) -> AggregateCell { - let store = - CellStreamStore::open_sqlite(location, StreamIdentity::new("CellItem", shard).unwrap()) - .unwrap() - .with_sealed_row_reducer(count_events); - AggregateCell::::from_store_with_snapshots(store, 1) - .unwrap() - .mount(Create) - .mount(Complete) - .mount(Publish) -} - -#[tokio::test] -async fn sqlite_cell_reopens_identically_and_replays_recorded_outcomes() { - let root = TempRoot::new(); - let location = root.location(); - let create_id = identity("0190a000-0000-7000-8000-000000000501"); - let complete_id = identity("0190a000-0000-7000-8000-000000000502"); - let create_input = json!({ "id": "item-1", "title": "ship" }); - - let cell = sqlite_cell(&location, "item-1"); - let created = cell - .dispatch_idempotent( - "cell_item.create", - &create_id, - create_input.clone(), - owner_session(), - ) - .await - .unwrap(); - let completed = cell - .dispatch_idempotent( - "cell_item.complete", - &complete_id, - json!({ "id": "item-1" }), - owner_session(), - ) - .await - .unwrap(); - assert!(!created.replayed() && !completed.replayed()); - // Whole-state export is a reference-host API; SQLite cells persist rows. - assert!(cell.durable_state().is_err()); - - let before = dump(&location, "item-1").await; - assert_eq!(before.events.len(), 2); - assert_eq!( - before.snapshot.as_ref().map(|record| record.version), - Some(2) - ); - assert_eq!( - before.sealed, - Some(json!({ "events": 2, "last": "cell_item.completed" })) - ); - assert!(before.outbox.is_empty(), "create/complete publish nothing"); - - // Drop every handle: nothing survives except the cell's SQLite file. - drop(cell); - assert!(location.resolve("CellItem:item-1").is_file()); - - let after = dump(&location, "item-1").await; - assert_eq!(after, before, "reopen must restore identical cell state"); - - let reopened = AggregateCell::::open_sqlite_with_snapshots(&location, "item-1", 1) - .unwrap() - .mount(Create) - .mount(Complete); - let loaded = reopened.load().await.unwrap().expect("durable aggregate"); - assert_eq!(loaded.title, "ship"); - assert!(loaded.done); - assert_eq!(loaded.entity.snapshot_version(), 2); - assert_eq!( - reopened.sealed_row().unwrap(), - Some(json!({ "events": 2, "last": "cell_item.completed" })) - ); - - // Create would be rejected ("already exists") if it ran again; the durable - // ledger must return the recorded outcome without invoking the handler. - let replay = reopened - .dispatch_idempotent( - "cell_item.create", - &create_id, - create_input, - owner_session(), - ) - .await - .expect("durable replay after reopen"); - assert!(replay.replayed()); - assert_eq!(replay.payload(), created.payload()); - assert_eq!(replay.causation_id(), created.causation_id()); - assert_eq!(replay.projection_events(), created.projection_events()); - - let reuse = reopened - .dispatch_idempotent( - "cell_item.create", - &create_id, - json!({ "id": "item-1", "title": "different" }), - owner_session(), - ) - .await - .unwrap_err(); - assert!(matches!(reuse, CellDispatchError::CommandIdReuse)); - - drop(reopened); - assert_eq!(dump(&location, "item-1").await, before, "replay re-applied"); -} - -#[tokio::test] -async fn sqlite_cell_history_reads_every_event_after_reopen() { - let root = TempRoot::new(); - let location = root.location(); - let cell = sqlite_cell(&location, "item-h"); - assert!(cell.history().await.unwrap().is_empty()); - assert!( - cell.durable_events().is_err(), - "whole-state export stays a reference-host API" - ); - cell.dispatch_idempotent( - "cell_item.create", - &identity("0190a000-0000-7000-8000-000000000601"), - json!({ "id": "item-h", "title": "ship" }), - owner_session(), - ) - .await - .unwrap(); - cell.dispatch_idempotent( - "cell_item.complete", - &identity("0190a000-0000-7000-8000-000000000602"), - json!({ "id": "item-h" }), - owner_session(), - ) - .await - .unwrap(); - let history = cell.history().await.unwrap(); - let names: Vec<_> = history.iter().map(|e| e.event_name.as_str()).collect(); - assert_eq!(names.len(), 2, "{names:?}"); - drop(cell); - - // A fresh handle (process restart) reads the same history, snapshots - // notwithstanding (snapshot every event here). - let reopened = sqlite_cell(&location, "item-h"); - assert_eq!(reopened.history().await.unwrap(), history); - let own = StreamIdentity::new("CellItem", "item-h").unwrap(); - assert_eq!(reopened.stream_events(&own).await.unwrap(), history); - assert_eq!(dump(&location, "item-h").await.events, history); -} - -#[tokio::test] -async fn sealed_row_reducer_failure_rolls_back_the_whole_command() { - let root = TempRoot::new(); - let location = root.location(); - let store = CellStreamStore::open_sqlite( - &location, - StreamIdentity::new("CellItem", "item-r").unwrap(), - ) - .unwrap() - .with_sealed_row_reducer(|current, appends| { - if appends - .iter() - .flat_map(|append| append.events) - .any(|event| event.event_name == "cell_item.completed") - { - return Err(RepositoryError::Model("sealed row rejected".into())); - } - count_events(current, appends) - }); - let cell = AggregateCell::::from_store_with_snapshots(store, 1) - .unwrap() - .mount(Create) - .mount(Complete); - cell.dispatch_idempotent( - "cell_item.create", - &identity("0190a000-0000-7000-8000-000000000511"), - json!({ "id": "item-r", "title": "keep" }), - owner_session(), - ) - .await - .unwrap(); - let before = dump(&location, "item-r").await; - - assert!(cell - .dispatch_idempotent( - "cell_item.complete", - &identity("0190a000-0000-7000-8000-000000000512"), - json!({ "id": "item-r" }), - owner_session(), - ) - .await - .is_err()); - drop(cell); - - let after = dump(&location, "item-r").await; - assert_eq!(after, before, "events, snapshot and sealed row roll back"); - assert_eq!(after.events.len(), 1); - assert_eq!(after.snapshot.map(|record| record.version), Some(1)); -} - -#[derive(Clone, Default)] -struct RecordingPublisher { - fail: Arc, - published: Arc>>>, -} - -impl MessagePublisher for RecordingPublisher { - async fn publish(&self, message: Message) -> Result<(), TransportError> { - if self.fail.load(Ordering::SeqCst) { - return Err(TransportError::retryable("transport down")); - } - self.published.lock().unwrap().push(message.id); - Ok(()) - } -} - -#[tokio::test] -async fn outbox_rows_survive_reopen_and_drain_at_least_once() { - let root = TempRoot::new(); - let location = root.location(); - let cell = sqlite_cell(&location, "pub-1"); - cell.dispatch_idempotent( - "cell_item.publish", - &identity("0190a000-0000-7000-8000-000000000521"), - json!({ "id": "pub-1", "title": "news" }), - owner_session(), - ) - .await - .unwrap(); - - // A drain that fails to publish releases the row for a later pass. - let publisher = RecordingPublisher::default(); - publisher.fail.store(true, Ordering::SeqCst); - let outcome = cell - .drain_outbox_with(publisher.clone(), Duration::from_secs(30), 10, 10) - .await - .unwrap(); - assert_eq!((outcome.claimed, outcome.released), (1, 1)); - drop(cell); - - let pending = dump(&location, "pub-1").await.outbox; - assert_eq!(pending.len(), 1); - assert_eq!(pending[0].status, OutboxMessageStatus::Pending); - let message_id = pending[0].id.clone(); - - // Simulate a crash mid-drain: claimed with a short lease, never settled. - let store = - CellStreamStore::open_sqlite(&location, StreamIdentity::new("CellItem", "pub-1").unwrap()) - .unwrap(); - let claimed = store - .outbox_store() - .claim(ClaimOutboxMessages::new( - "crashed-worker", - 10, - Duration::from_secs(2), - )) - .await - .unwrap(); - assert_eq!(claimed.len(), 1); - let stale = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - drop(store); - - // Restart. The row survives as in-flight under the dead worker's lease. - let reopened = sqlite_cell(&location, "pub-1"); - let outbox = dump(&location, "pub-1").await.outbox; - assert_eq!(outbox.len(), 1); - assert_eq!(outbox[0].status, OutboxMessageStatus::InFlight); - publisher.fail.store(false, Ordering::SeqCst); - let fenced = reopened - .drain_outbox_with(publisher.clone(), Duration::from_secs(30), 10, 10) - .await - .unwrap(); - assert_eq!(fenced.claimed, 0, "a live lease is not stolen"); - - tokio::time::sleep(Duration::from_millis(2_100)).await; - let store = - CellStreamStore::open_sqlite(&location, StreamIdentity::new("CellItem", "pub-1").unwrap()) - .unwrap(); - assert!( - store.outbox_store().complete(&stale).await.is_err(), - "an expired lease cannot delete the row" - ); - drop(store); - - let drained = reopened - .drain_outbox_with(publisher.clone(), Duration::from_secs(30), 10, 10) - .await - .unwrap(); - assert_eq!((drained.claimed, drained.published), (1, 1)); - assert_eq!( - publisher.published.lock().unwrap().as_slice(), - &[Some(message_id)] - ); - drop(reopened); - - let after = dump(&location, "pub-1").await; - assert!(after.outbox.is_empty(), "published rows are deleted"); - assert_eq!(after.events.len(), 1); -} - -#[tokio::test] -async fn sqlite_cells_enforce_ownership_and_file_binding() { - let root = TempRoot::new(); - let location = root.location(); - - let exclusive = CellStreamStore::open_sqlite( - &location, - StreamIdentity::new("CellItem", "item-1").unwrap(), - ) - .unwrap(); - let mut foreign = Entity::with_id("item-2"); - foreign.digest_empty("created").unwrap(); - let error = TransactionalCommit::commit_batch( - &exclusive, - CommitBatch::new(vec![StreamWrite::new( - StreamIdentity::new("CellItem", "item-2").unwrap(), - &mut foreign, - )]), - ) - .await - .unwrap_err(); - assert!( - matches!(error, RepositoryError::Model(message) if message.contains("cannot access stream")) - ); - - let owns = |game: &'static str| { - move |identity: &StreamIdentity| identity.aggregate_id().starts_with(game) - }; - let game_1 = CellStreamStore::open_sqlite_parent(&location, "game", "g1", owns("g1:")).unwrap(); - let mut player = Entity::with_id("g1:player:1"); - player.digest_empty("joined").unwrap(); - let mut bomb = Entity::with_id("g1:bomb:1"); - bomb.digest_empty("placed").unwrap(); - let player_id = StreamIdentity::new("Player", "g1:player:1").unwrap(); - let bomb_id = StreamIdentity::new("Bomb", "g1:bomb:1").unwrap(); - TransactionalCommit::commit_batch( - &game_1, - CommitBatch::new(vec![ - StreamWrite::new(player_id.clone(), &mut player), - StreamWrite::new(bomb_id.clone(), &mut bomb), - ]), - ) - .await - .unwrap(); - drop(game_1); - - let game_1 = CellStreamStore::open_sqlite_parent(&location, "game", "g1", owns("g1:")).unwrap(); - assert!(GetStream::get_stream(&game_1, &player_id) - .await - .unwrap() - .is_some()); - assert!(GetStream::get_stream(&game_1, &bomb_id) - .await - .unwrap() - .is_some()); - let game_2 = CellStreamStore::open_sqlite_parent(&location, "game", "g2", owns("g2:")).unwrap(); - assert!(GetStream::get_stream(&game_2, &player_id).await.is_err()); - assert_ne!( - location.resolve("game:g1"), - location.resolve("game:g2"), - "one SQLite file per cell" - ); - - // An explicit file is bound to the first cell that opens it. - let file = CellSqlitePath::file(root.0.join("explicit.db")); - CellStreamStore::open_sqlite(&file, StreamIdentity::new("CellItem", "a").unwrap()).unwrap(); - let error = CellStreamStore::open_sqlite(&file, StreamIdentity::new("CellItem", "b").unwrap()) - .err() - .expect("file bound to CellItem:a"); - assert!(error.to_string().contains("belongs to"), "{error}"); - CellStreamStore::open_sqlite(&file, StreamIdentity::new("CellItem", "a").unwrap()).unwrap(); -} diff --git a/src/repository/migrations.rs b/src/repository/migrations.rs index 188fc912a..e5dcec142 100644 --- a/src/repository/migrations.rs +++ b/src/repository/migrations.rs @@ -14,10 +14,7 @@ include!(concat!(env!("OUT_DIR"), "/migration_inventory.rs")); /// Cells host commands, not read-model projectors. These are the event, /// snapshot, delivery and command-ledger migrations from the native inventory. /// Projection-owned tables are initialized by their separate query host. -#[cfg(any( - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) -))] +#[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] pub(crate) fn cell_migrations() -> impl Iterator { SQLITE_MIGRATIONS .iter() diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 1957b2ba5..29c100c17 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -4,8 +4,7 @@ mod inbox; #[cfg(any( feature = "sqlite", feature = "postgres", - all(feature = "workers-rs", target_arch = "wasm32"), - all(feature = "cell-sqlite", not(target_arch = "wasm32")) + all(feature = "workers-rs", target_arch = "wasm32") ))] pub(crate) mod migrations; pub(crate) mod sql; diff --git a/tests/celld/README.md b/tests/celld/README.md index 5d248e439..d87295bb1 100644 --- a/tests/celld/README.md +++ b/tests/celld/README.md @@ -73,33 +73,16 @@ The whole-state export/restore and `persist_and_drain_outbox` APIs are not available on the Worker host. Native in-process conformance fixtures are not a production persistence adapter. -### Native durable cells (`cell-sqlite`) - -Native hosts get the same store with the `cell-sqlite` Cargo feature: one -SQLite file per cell, executed through a synchronous rusqlite connection that -exposes the same `read`/`transaction` surface as `storage.transactionSync`. -Schema bootstrap, the per-command transaction (events, snapshots, ledger -receipt, outbox rows) and lease-fenced outbox settlement are the shared -`sql_store.rs` code, not a copy. Native cells additionally keep the sealed -read-model row in the same file and, with -`CellStreamStore::with_sealed_row_reducer`, update it inside the command -transaction. - -```rust,ignore -let cells = CellSqlitePath::root("var/cells"); // var/cells/Todo%3At1.sqlite3 -let cell = AggregateCell::::open_sqlite_with_snapshots(&cells, "t1", 100)? - .mount(create()); -let result = cell.dispatch_idempotent(command, &identity, input, session).await?; - -// Alarm equivalent: run at startup, after dispatch and periodically, or spawn -// OutboxDrainRunner::new(cell.outbox_dispatcher(..)) on a tokio runtime. -let drained = cell.drain_outbox_with(publisher, lease, max_attempts, 100).await?; -``` - -Dropping every handle and reopening the file restores the identical cell, and -an already-applied idempotent command replays its recorded outcome. A file is -bound to the first cell instance that opens it. Whole-state export/restore -remains a reference-host (in-memory) API and is rejected for SQLite cells. +### No native durable cells + +Durable cells run on celld only, for local development (`celld dev`) and +deployment alike. The native `cell-sqlite` backend (one rusqlite file per +cell) was removed in v5: a native build has only the in-process reference +host, for unit tests (`CellStreamStore::for_identity`, whole-state +export/restore, `ManualClock` alarms). A native process that needs durable +aggregates talks to celld over the internal path (`CelldCommandHost`, +`distributed_game::celld::CelldFactSink`); see `examples/game/cells` for a +zone server's cells. Existing `cell_state` databases are rejected with an explicit migration-required error. They are not silently reset or opened through a legacy adapter. Back up From fac1cf3609b7bbd4bd56bf5a4a3727bf606d3509 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 14:06:50 -0500 Subject: [PATCH 17/40] feat(game)!: hibernate zones in celld cells, all demos on celld Zone hibernation snapshots move to celld and every demo with durable state now runs on celld; SqliteZoneStateStore is removed (same decision as cell-sqlite: no native durable store). - world::zone_state: a ZoneState:{zone} aggregate cell (zone_state.save / zone_state.delete, token fencing equal to the old SQLite store; the highest token survives a delete). Snapshots up to 64 KiB are inline; larger ones go to the worker's R2 bucket binding (celld object storage, RustFS in the Hops stack) with key/sha256/len in the cell; superseded and refused uploads are deleted. - celld::CellZoneStateStore client; world::zone_state::celld_worker route glue (feature workers-rs). celld's R2 binding is not an instance of R2Bucket, so the glue takes it unchecked (Env::bucket refuses it). - tests/celld/fact-worker and examples/game/cells mount ZoneLease and ZoneState cells; the demo worker also gets The Freeze's gem-pouch and content-log cells. - Grid and Ripple require celld: CellZoneOwnership + CellZoneStateStore (no LocalOwnership/SQLite fallback); The Freeze settles its facts in celld via FreezeFactSink. run.sh starts celld for freeze, grid and ripple; Rust and JS cluster/freeze tests start their own celld dev. - Hibernation tests (distributed_game world, celld_zone_state, grid_smoke) run against celld; CI's storage job runs them with celld pinned to v0.6.2. BREAKING CHANGE: ZoneStateStore is async and has delete(); SqliteZoneStateStore and distributed_game's `sqlite` feature are gone. game_demos: ClusterOptions.celld is required and state_dir is removed (`--state-dir` flag gone, `--celld-url`/CELLD_URL and DISTRIBUTED_INTERNAL_SECRET required for grid, ripple and freeze); World::standalone takes (host id, hibernate_after, guests). [[tasks/gr-p37-celld-zone-state]] --- .github/workflows/integration-celld.yaml | 9 +- distributed_game/Cargo.toml | 22 +- distributed_game/README.md | 10 +- distributed_game/src/celld.rs | 202 ++++++- distributed_game/src/net/mod.rs | 38 +- distributed_game/src/world/mod.rs | 10 +- distributed_game/src/world/state.rs | 348 ++++------- distributed_game/src/world/zone_state.rs | 559 ++++++++++++++++++ .../tests/celld_zone_state/main.rs | 61 ++ distributed_game/tests/support/celld.rs | 4 +- distributed_game/tests/world/main.rs | 90 ++- distributed_game/tests/zone_state/main.rs | 243 ++++++++ examples/game/README.md | 56 +- examples/game/cells/Cargo.toml | 12 +- examples/game/cells/src/lib.rs | 356 ++++++++++- examples/game/cells/wrangler.jsonc | 16 +- .../client/tests/freeze-headless.test.mjs | 26 +- .../game/client/tests/helpers/cluster.mjs | 33 +- examples/game/run.sh | 32 +- examples/game/server/Cargo.lock | 50 -- examples/game/server/Cargo.toml | 2 +- examples/game/server/src/cluster.rs | 129 ++-- examples/game/server/src/common/celld.rs | 45 ++ examples/game/server/src/common/mod.rs | 2 + examples/game/server/src/devcelld.rs | 2 +- examples/game/server/src/freeze/world.rs | 108 +++- examples/game/server/src/grid/dev.rs | 53 +- examples/game/server/src/grid/mod.rs | 9 +- examples/game/server/src/mobpit/store.rs | 40 +- examples/game/server/src/ripple/mod.rs | 5 +- examples/game/server/tests/freeze_smoke.rs | 28 +- examples/game/server/tests/grid_smoke.rs | 7 +- examples/game/server/tests/support/mod.rs | 29 +- tests/celld/fact-worker/Cargo.toml | 2 +- tests/celld/fact-worker/src/lib.rs | 34 ++ tests/celld/fact-worker/wrangler.jsonc | 9 +- 36 files changed, 2078 insertions(+), 603 deletions(-) create mode 100644 distributed_game/src/world/zone_state.rs create mode 100644 distributed_game/tests/celld_zone_state/main.rs create mode 100644 distributed_game/tests/zone_state/main.rs create mode 100644 examples/game/server/src/common/celld.rs diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index f08008562..d61c02ad4 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -52,7 +52,8 @@ jobs: sudo apt-get update && sudo apt-get install -y sqlite3 npm install -g esbuild cargo install worker-build --locked - curl -fsSL https://celld.dev/install.sh | sh + # Pinned: the zone-state R2 path and the fact routing are proven on it. + curl -fsSL https://celld.dev/install.sh | CELLD_VERSION=v0.6.2 sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Build isolated fault-probe Worker working-directory: tests/celld/worker @@ -75,6 +76,12 @@ jobs: env: CELLD_FACTS_REQUIRED: "1" run: cargo test -p distributed_game --features celld,runtime,net --test celld_zone_lease + - name: Zone hibernation state in celld cells (inline + R2 snapshots, crash, GameServer) + env: + CELLD_FACTS_REQUIRED: "1" + run: | + cargo test -p distributed_game --features celld,runtime --test celld_zone_state + cargo test -p distributed_game --features celld,net,lease --test world - name: Retain storage proof results and logs if: always() uses: actions/upload-artifact@v4 diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index 7e37b85b1..c2f118f32 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -15,10 +15,13 @@ default = [] # Tokio-driven fact worker (`spawn_fact_worker`). Core stays runtime-agnostic. runtime = ["dep:tokio"] # Adapter that commits facts into a `distributed` AggregateCell. -cell = ["dep:distributed", "dep:sha2"] +cell = ["dep:distributed", "dep:sha2", "dep:base64"] # `celld::CelldFactSink`: a zone server's facts sent to celld cells over the # internal HTTP path (routes by command; stable command ids; no local store). celld = ["cell", "dep:reqwest", "dep:uuid"] +# celld worker glue (wasm32 Durable Objects): `world::zone_state::worker`, +# the zone-state HTTP surface over a cell and an R2 bucket binding. +workers-rs = ["cell", "dep:worker", "distributed/workers-rs"] # Per-zone Prometheus metrics (dependency-free text exposition). metrics = [] # Load-test bots (library + `game-bots` binary). @@ -27,8 +30,6 @@ bots = ["net", "dep:tokio-tungstenite", "dep:futures-util", "tokio/rt-multi-thre net = ["runtime", "metrics", "dep:axum", "dep:hmac", "dep:sha2", "dep:base64", "tokio/net", "tokio/macros", "tokio/io-util"] # `world::FencedLeaseOwnership`: zone ownership over `distributed::lease`. lease = ["dep:distributed"] -# `world::SqliteZoneStateStore`: native SQLite hibernation snapshots. -sqlite = ["dep:rusqlite"] [dependencies] distributed_game_macros = { version = "0.1.0", path = "../distributed_game_macros" } @@ -42,11 +43,12 @@ sha2 = { version = "0.10", optional = true } base64 = { version = "0.23", optional = true } tokio-tungstenite = { version = "0.30", optional = true } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"], optional = true } -rusqlite = { version = "0.39", features = ["bundled"], optional = true } reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true } uuid = { version = "1", features = ["v7"], optional = true } +worker = { version = "0.8", optional = true } [dev-dependencies] +base64 = "0.23" trybuild = "=1.0.120" tokio = { version = "1", features = ["full", "test-util"] } tokio-tungstenite = "0.30" @@ -78,11 +80,21 @@ name = "zone_lease" path = "tests/zone_lease/main.rs" required-features = ["cell"] +[[test]] +name = "zone_state" +path = "tests/zone_state/main.rs" +required-features = ["cell"] + [[test]] name = "celld_zone_lease" path = "tests/celld_zone_lease/main.rs" required-features = ["celld", "runtime", "net"] +[[test]] +name = "celld_zone_state" +path = "tests/celld_zone_state/main.rs" +required-features = ["celld", "runtime"] + [[test]] name = "celld_facts" path = "tests/celld_facts/main.rs" @@ -106,7 +118,7 @@ required-features = ["cell", "runtime"] [[test]] name = "world" path = "tests/world/main.rs" -required-features = ["net", "lease", "sqlite"] +required-features = ["net", "lease", "celld"] [[example]] name = "echo_arena" diff --git a/distributed_game/README.md b/distributed_game/README.md index efe528ce0..1bf09895e 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -795,6 +795,8 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), - **On demand and asleep.** Zones start on first use. With `hibernate_after`, an empty zone is saved (`Zone::hibernate`) and unloaded, and restored (`on_restore`) the next time someone arrives. + On celld the snapshot is the zone's own `ZoneState` cell: a sleeping + zone is a sleeping Durable Object. - **Handoff.** `ctx.handoff(player, "next-zone", carried_bytes)` moves a player to another zone, on any server, with a signed transfer ticket. The target receives `on_handoff_join` with the carried state. When the @@ -937,13 +939,13 @@ trait and its ABI (`run_abi`, `call_bytes`). |---|---|---| | *(default)* | Zone/host runtime, facts queue, protocol, schema, replication, authoring derives, game surface, predicted steps (compiles to `wasm32-unknown-unknown`, where `#[predict]` exports its steps) | `serde`, `serde_json`, `distributed_game_macros` | | `runtime` | `spawn_fact_worker` on Tokio | `tokio` (rt, time, sync) | -| `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record` | `distributed`, `sha2` | -| `celld` | `celld::CelldFactSink`: facts → celld cells over the internal HTTP path, routed by command; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells | `cell` + `reqwest`, `uuid` | +| `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native, unit tests); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record`; the `world::zone_lease` and `world::zone_state` cell aggregates | `distributed`, `sha2`, `base64` | +| `celld` | `celld::CelldFactSink`: facts → celld cells over the internal HTTP path, routed by command; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells; `celld::CellZoneStateStore`: hibernation snapshots in celld `ZoneState` cells | `cell` + `reqwest`, `uuid` | +| `workers-rs` | `world::zone_state::celld_worker`: the `zone-state/{zone}` route of a celld worker (cell + R2 bucket binding for large snapshots) | `cell` + `worker` | | `metrics` | `metrics::GameMetrics` per-zone Prometheus text | – | | `net` | `net::GameServer` (axum WebSocket), HMAC tickets, `GET /metrics` (implies `metrics`), handoff, admin peer routes, `WorldCoordinator` | `axum`, `hmac`, `sha2`, `base64`, `tokio` | | `bots` | `bots` load-test client library + `game-bots` binary | `net` + `tokio-tungstenite` | | `lease` | `world::FencedLeaseOwnership` over `distributed::lease` | `distributed` | -| `sqlite` | `world::SqliteZoneStateStore` (hibernation snapshots) | `rusqlite` (bundled) | ## Durable facts @@ -1104,7 +1106,7 @@ principals commit concurrently; settlements arrive in completion order. |---|---| | `ZoneDirectory` (`StaticDirectory`, `HashRing`) | zone → owning `HostAddr`. `HashRing` uses virtual nodes; adding/removing one of N hosts moves ~1/N of the zones. A Hello for a zone owned elsewhere gets `Redirect(zone, url)`; `url` is the owner's `HostAddr::ws_url` (advertise a proxied public URL with `HostAddr::with_ws_url`) or `ServerConfig::with_redirect_url(\|host, zone\| …)`. | | `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the cell's alarm publishes `zone.lease_expired { zone_id, holder, token }` when a holder stops renewing, so services see a crashed zone server. `world::ownership_conformance` is the behaviour every implementation passes. | -| `ZoneStateStore` (`MemoryZoneStateStore`, `SqliteZoneStateStore`) | hibernation snapshots, written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | +| `ZoneStateStore` (`celld::CellZoneStateStore`; `MemoryZoneStateStore` for unit tests) | hibernation snapshots (async `load` / `save` / `delete`), written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). On celld each zone's snapshot is its own `ZoneState:{zone}` cell (`world::zone_state`): inline up to 64 KiB, larger ones in the worker's R2 bucket with the digest in the cell. `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | | `ZoneTopology` (`DottedTopology` default: `eldoria.port.docks` → `eldoria.port` → `eldoria`) | parent links for tier-up events. | **Per-zone host settings.** `GameServer::with_host_config(|zone_id, host| diff --git a/distributed_game/src/celld.rs b/distributed_game/src/celld.rs index e60dee140..54a81dc4a 100644 --- a/distributed_game/src/celld.rs +++ b/distributed_game/src/celld.rs @@ -1,5 +1,7 @@ //! [`CelldFactSink`]: a zone server's facts, sent over the network to the -//! durable aggregates in celld cells (game-services decision D3). +//! durable aggregates in celld cells (game-services decision D3). Also the +//! zone server's other celld clients: [`CellZoneOwnership`] (zone leases) +//! and [`CellZoneStateStore`] (hibernation snapshots). //! //! One sink per zone server routes every fact by its command //! ([`FactCommand::COMMAND`](crate::FactCommand::COMMAND)) to a registered @@ -461,6 +463,204 @@ impl crate::world::ZoneOwnership for CellZoneOwnership { } } +/// [`ZoneStateStore`](crate::world::ZoneStateStore) over celld +/// `ZoneState:{zone_id}` cells ([`crate::world::zone_state`]): hibernation +/// snapshots of a zone server live in one celld cell per zone (inline below +/// [`ZONE_STATE_INLINE_MAX`](crate::world::zone_state::ZONE_STATE_INLINE_MAX), +/// in celld's object storage above it), fenced by the ownership token. +/// +/// Every write uses a fresh command id: a retried save under the same token +/// is accepted again (equal tokens are allowed), so an ambiguous response is +/// safe. +#[derive(Clone)] +pub struct CellZoneStateStore { + client: reqwest::Client, + base: String, + secret: InternalHttpSecret, + service_id: String, + host: String, +} + +/// Cell kind (URL segment) of zone-state cells. +pub const ZONE_STATE_KIND: &str = "zone-state"; + +impl CellZoneStateStore { + /// Store for zone server `host` against `celld_url`, under + /// command-ledger `service_id`. + pub fn new( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + host: impl Into, + ) -> Result { + let base = celld_url.into().trim_end_matches('/').to_string(); + if !(base.starts_with("http://") || base.starts_with("https://")) { + return Err(format!("celld URL must be http(s): {base}")); + } + let service_id = service_id.into(); + let host = host.into(); + if service_id.trim().is_empty() || host.trim().is_empty() { + return Err("cell zone state needs a service id and a host identity".into()); + } + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("celld zone-state client: {error}"))?; + Ok(Self { + client, + base, + secret, + service_id, + host, + }) + } + + fn url( + &self, + zone_id: &str, + command: Option<&str>, + ) -> Result { + if !valid_segment(zone_id) { + return Err(crate::world::StateError::Backend(format!( + "invalid zone id `{zone_id}`" + ))); + } + let mut url = format!( + "{}/{}/{}", + self.base, + encode_segment(ZONE_STATE_KIND), + encode_segment(zone_id) + ); + if let Some(command) = command { + url.push('/'); + url.push_str(&encode_segment(command)); + } + Ok(url) + } + + async fn command( + &self, + zone_id: &str, + command: &str, + input: Value, + ) -> Result<(), crate::world::StateError> { + use crate::world::zone_lease::ZONE_HOST_ROLE; + use crate::world::StateError; + let url = self.url(zone_id, Some(command))?; + let token = input.get("token").and_then(Value::as_u64).unwrap_or(0); + let response = self + .client + .post(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .header(CELL_SERVICE_ID_HEADER, &self.service_id) + .header(CELL_PRINCIPAL_PARTITION_HEADER, &self.host) + .header(USER_ID_KEY, &self.host) + .header(distributed::microsvc::ROLE_KEY, ZONE_HOST_ROLE) + .json(&json!({ + "commandId": uuid::Uuid::now_v7().hyphenated().to_string(), + "input": input, + })) + .send() + .await + .map_err(|error| StateError::Backend(format!("celld unreachable: {error}")))?; + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if (200..300).contains(&status) { + return Ok(()); + } + let message = body.get("error").and_then(Value::as_str).unwrap_or(""); + match crate::world::zone_state::stale_current(message) { + Some(current) => Err(StateError::Stale { + presented: token, + current, + }), + None => Err(StateError::Backend(format!("HTTP {status}: {message}"))), + } + } +} + +impl crate::world::ZoneStateStore for CellZoneStateStore { + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, crate::world::StateError>> + { + use crate::world::StateError; + Box::pin(async move { + use base64::Engine as _; + let url = self.url(zone_id, None)?; + let response = self + .client + .get(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .send() + .await + .map_err(|error| StateError::Backend(format!("celld unreachable: {error}")))?; + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if !(200..300).contains(&status) { + return Err(StateError::Backend(format!("HTTP {status}: {body}"))); + } + let Some(state) = body.get("state").filter(|state| !state.is_null()) else { + return Ok(None); + }; + let field = |name: &str| { + state + .get(name) + .ok_or_else(|| StateError::Backend(format!("zone state without `{name}`"))) + }; + let bytes = base64::engine::general_purpose::STANDARD + .decode(field("state_b64")?.as_str().unwrap_or_default()) + .map_err(|_| StateError::Backend("zone state is not base64".into()))?; + Ok(Some(crate::world::StoredZoneState { + token: field("token")?.as_u64().unwrap_or(0), + content_version: field("content_version")? + .as_str() + .unwrap_or_default() + .to_string(), + state: bytes, + })) + }) + } + + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a crate::world::StoredZoneState, + ) -> BoxFuture<'a, Result<(), crate::world::StateError>> { + Box::pin(async move { + use base64::Engine as _; + self.command( + zone_id, + crate::world::zone_state::SAVE, + json!({ + "zone_id": zone_id, + "token": state.token, + "content_version": state.content_version, + "state_b64": base64::engine::general_purpose::STANDARD.encode(&state.state), + }), + ) + .await + }) + } + + fn delete<'a>( + &'a self, + zone_id: &'a str, + token: u64, + ) -> BoxFuture<'a, Result<(), crate::world::StateError>> { + Box::pin(async move { + self.command( + zone_id, + crate::world::zone_state::DELETE, + json!({ "zone_id": zone_id, "token": token }), + ) + .await + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index 4ef4ca156..637b5e6fb 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -1148,24 +1148,20 @@ async fn run_zone( }; // 2. Restore the hibernated snapshot, if any. let saved = match &inner.state_store { - Some(store) => { - let store = store.clone(); - let id = zone_id.clone(); - match tokio::task::spawn_blocking(move || store.load(&id)).await { - Ok(Ok(saved)) => saved, - _ => { - inner.ownership.release(&claim).await; - return refuse_activation( - &inner, - &zone_id, - activation, - rx, - error::ZONE_UNAVAILABLE, - "zone state unavailable", - ); - } + Some(store) => match store.load(&zone_id).await { + Ok(saved) => saved, + _ => { + inner.ownership.release(&claim).await; + return refuse_activation( + &inner, + &zone_id, + activation, + rx, + error::ZONE_UNAVAILABLE, + "zone state unavailable", + ); } - } + }, None => None, }; // 3. The allocation record and the game's factory, bounded together. @@ -1366,11 +1362,11 @@ async fn run_zone( }; // A stale token means a newer owner saved already: ours is // dropped, but never silently (metrics + stderr). - let saved = tokio::task::spawn_blocking(move || store.save(&id, &stored)).await; + let saved = store.save(&id, &stored).await; let outcome = match &saved { - Ok(Ok(())) => "saved", - Ok(Err(StateError::Stale { .. })) => "stale", - _ => "error", + Ok(()) => "saved", + Err(StateError::Stale { .. }) => "stale", + Err(_) => "error", }; inner.metrics.record_hibernation_save(outcome); if outcome != "saved" { diff --git a/distributed_game/src/world/mod.rs b/distributed_game/src/world/mod.rs index ef86ab8b5..839e8d22c 100644 --- a/distributed_game/src/world/mod.rs +++ b/distributed_game/src/world/mod.rs @@ -7,7 +7,8 @@ //! the same zone ([`LocalOwnership`]; `FencedLeaseOwnership` over //! `distributed::lease` with the `lease` feature). //! - [`state`]: [`ZoneStateStore`] for hibernated zone state -//! ([`MemoryZoneStateStore`]; `SqliteZoneStateStore` with `sqlite`). +//! ([`MemoryZoneStateStore`] for unit tests; on celld, +//! `celld::CellZoneStateStore` over [`zone_state`] cells). //! - [`topology`]: [`ZoneTopology`] parent links (province > city > borough //! > neighborhood) and [`ChildEvent`] tier-up propagation. //! - [`allocation`]: [`ZoneAllocation`] records written by the service that @@ -26,6 +27,9 @@ pub mod topology; // The `sourced` and `Snapshot` derives generate undocumented public items. #[allow(missing_docs)] pub mod zone_lease; +#[cfg(feature = "cell")] +#[allow(missing_docs)] +pub mod zone_state; pub use crate::facts::BoxFuture; pub use allocation::{ @@ -39,7 +43,7 @@ pub use ownership::ownership_conformance; #[cfg(feature = "lease")] pub use ownership::FencedLeaseOwnership; pub use ownership::{LocalOwnership, OwnershipError, ZoneClaim, ZoneOwnership}; -#[cfg(feature = "sqlite")] -pub use state::SqliteZoneStateStore; +#[doc(hidden)] +pub use state::state_store_conformance; pub use state::{MemoryZoneStateStore, StateError, StoredZoneState, ZoneStateStore}; pub use topology::{ChildEvent, DottedTopology, FlatTopology, ZoneTopology}; diff --git a/distributed_game/src/world/state.rs b/distributed_game/src/world/state.rs index e0667a870..098c2c1b3 100644 --- a/distributed_game/src/world/state.rs +++ b/distributed_game/src/world/state.rs @@ -5,22 +5,21 @@ //! the zone's ownership token; the next activation (on any host) loads it //! and hands it to [`Zone::on_restore`](crate::Zone::on_restore). //! -//! Writes are **fenced**: `save` with a token older than the stored one -//! fails with [`StateError::Stale`], so a host that lost ownership while -//! paused can never overwrite the state written by its successor. +//! Writes are **fenced**: `save` (and `delete`) with a token older than the +//! stored one fails with [`StateError::Stale`], so a host that lost +//! ownership while paused can never overwrite the state written by its +//! successor. An equal token is accepted (the same owner saving again). //! -//! The trait is synchronous (snapshots are small and rare); async drivers -//! call it on a blocking thread. -//! -//! Why not an `AggregateCell`: a hibernation snapshot is a whole-value -//! overwrite keyed by zone, not a stream of domain events; storing it as -//! events would grow without bound and buy nothing. Durable *outcomes* (XP, -//! loot) go through facts into cells; the snapshot only holds ephemeral -//! simulation state worth keeping across a cold start. +//! Durable snapshots live in celld: one `ZoneState:{zone_id}` cell per zone +//! ([`super::zone_state`], `celld::CellZoneStateStore`), so a sleeping zone +//! is a sleeping Durable Object. [`MemoryZoneStateStore`] is the in-process +//! store for unit tests. There is no native file store. use std::collections::HashMap; use std::sync::Mutex; +use crate::facts::BoxFuture; + /// A stored snapshot. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StoredZoneState { @@ -61,17 +60,29 @@ impl std::error::Error for StateError {} /// Fenced snapshot store. pub trait ZoneStateStore: Send + Sync + 'static { - /// Latest snapshot of `zone_id`. - fn load(&self, zone_id: &str) -> Result, StateError>; + /// Latest snapshot of `zone_id` (`None` when none, or deleted). + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, StateError>>; /// Store a snapshot if `state.token` >= the stored token. - fn save(&self, zone_id: &str, state: &StoredZoneState) -> Result<(), StateError>; + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a StoredZoneState, + ) -> BoxFuture<'a, Result<(), StateError>>; + + /// Drop the snapshot if `token` >= the stored token. The token stays + /// (a later save under an older token is still stale). + fn delete<'a>(&'a self, zone_id: &'a str, token: u64) -> BoxFuture<'a, Result<(), StateError>>; } -/// In-memory store (tests / single process). +/// In-memory store (unit tests / single process). #[derive(Debug, Default)] pub struct MemoryZoneStateStore { - states: Mutex>, + /// zone -> (highest token seen, snapshot). + states: Mutex)>>, } impl MemoryZoneStateStore { @@ -79,242 +90,123 @@ impl MemoryZoneStateStore { pub fn new() -> Self { Self::default() } -} - -impl ZoneStateStore for MemoryZoneStateStore { - fn load(&self, zone_id: &str) -> Result, StateError> { - Ok(self.states.lock().expect("states").get(zone_id).cloned()) - } - fn save(&self, zone_id: &str, state: &StoredZoneState) -> Result<(), StateError> { + fn fenced( + &self, + zone_id: &str, + token: u64, + value: Option, + ) -> Result<(), StateError> { let mut states = self.states.lock().expect("states"); - if let Some(current) = states.get(zone_id) { - if current.token > state.token { - return Err(StateError::Stale { - presented: state.token, - current: current.token, - }); - } + let entry = states.entry(zone_id.to_string()).or_insert((0, None)); + if entry.0 > token { + return Err(StateError::Stale { + presented: token, + current: entry.0, + }); } - states.insert(zone_id.to_string(), state.clone()); + *entry = (token, value); Ok(()) } } -/// Native SQLite store (feature `sqlite`): one table, one fenced upsert. -#[cfg(feature = "sqlite")] -pub struct SqliteZoneStateStore { - conn: Mutex, -} - -#[cfg(feature = "sqlite")] -impl std::fmt::Debug for SqliteZoneStateStore { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("SqliteZoneStateStore") +impl ZoneStateStore for MemoryZoneStateStore { + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, StateError>> { + let state = self + .states + .lock() + .expect("states") + .get(zone_id) + .and_then(|(_, state)| state.clone()); + Box::pin(async move { Ok(state) }) } -} - -#[cfg(feature = "sqlite")] -fn backend(e: rusqlite::Error) -> StateError { - StateError::Backend(e.to_string()) -} -#[cfg(feature = "sqlite")] -impl SqliteZoneStateStore { - /// Open (or create) the database file. - pub fn open(path: impl AsRef) -> Result { - Self::from_connection(rusqlite::Connection::open(path).map_err(backend)?) + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a StoredZoneState, + ) -> BoxFuture<'a, Result<(), StateError>> { + let result = self.fenced(zone_id, state.token, Some(state.clone())); + Box::pin(async move { result }) } - /// In-memory database. - pub fn open_in_memory() -> Result { - Self::from_connection(rusqlite::Connection::open_in_memory().map_err(backend)?) - } - - fn from_connection(conn: rusqlite::Connection) -> Result { - // Hosts sharing one file collide on writes; wait for the other writer - // instead of failing at once with "database is locked". - const BUSY: std::time::Duration = std::time::Duration::from_secs(5); - conn.busy_timeout(BUSY).map_err(backend)?; - // The WAL switch at open does not use the busy handler, so two hosts - // opening the file together retry it until the same deadline. - let deadline = std::time::Instant::now() + BUSY; - loop { - match conn.execute_batch( - "PRAGMA journal_mode=WAL; - CREATE TABLE IF NOT EXISTS game_zone_state ( - zone_id TEXT PRIMARY KEY, - token INTEGER NOT NULL, - content_version TEXT NOT NULL, - state BLOB NOT NULL - );", - ) { - Ok(()) => break, - Err(rusqlite::Error::SqliteFailure(e, _)) - if matches!( - e.code, - rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked - ) && std::time::Instant::now() < deadline => - { - std::thread::sleep(std::time::Duration::from_millis(10)); - } - Err(e) => return Err(backend(e)), - } - } - Ok(Self { - conn: Mutex::new(conn), - }) + fn delete<'a>(&'a self, zone_id: &'a str, token: u64) -> BoxFuture<'a, Result<(), StateError>> { + let result = self.fenced(zone_id, token, None); + Box::pin(async move { result }) } } -#[cfg(feature = "sqlite")] -impl ZoneStateStore for SqliteZoneStateStore { - fn load(&self, zone_id: &str) -> Result, StateError> { - use rusqlite::OptionalExtension; - let conn = self.conn.lock().expect("sqlite"); - conn.query_row( - "SELECT token, content_version, state FROM game_zone_state WHERE zone_id = ?1", - [zone_id], - |row| { - Ok(StoredZoneState { - token: row.get::<_, i64>(0)? as u64, - content_version: row.get(1)?, - state: row.get(2)?, - }) - }, - ) - .optional() - .map_err(backend) - } - - fn save(&self, zone_id: &str, state: &StoredZoneState) -> Result<(), StateError> { - let conn = self.conn.lock().expect("sqlite"); - let token = i64::try_from(state.token).map_err(|_| StateError::Backend("token".into()))?; - let changed = conn - .execute( - "INSERT INTO game_zone_state (zone_id, token, content_version, state) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(zone_id) DO UPDATE SET - token = excluded.token, - content_version = excluded.content_version, - state = excluded.state - WHERE excluded.token >= game_zone_state.token", - rusqlite::params![zone_id, token, state.content_version, state.state], - ) - .map_err(backend)?; - if changed == 0 { - let current: i64 = conn - .query_row( - "SELECT token FROM game_zone_state WHERE zone_id = ?1", - [zone_id], - |row| row.get(0), - ) - .map_err(backend)?; - return Err(StateError::Stale { - presented: state.token, - current: current as u64, - }); - } - Ok(()) - } +/// The fencing contract every [`ZoneStateStore`] must meet (memory, celld). +#[doc(hidden)] +pub async fn state_store_conformance(store: &dyn ZoneStateStore, zone: &str, payload: Vec) { + let snap = |token: u64, bytes: Vec| StoredZoneState { + token, + content_version: "c1".into(), + state: bytes, + }; + assert_eq!(store.load(zone).await.unwrap(), None); + store.save(zone, &snap(2, b"a".to_vec())).await.unwrap(); + store.save(zone, &snap(2, payload.clone())).await.unwrap(); + assert_eq!( + store.load(zone).await.unwrap(), + Some(snap(2, payload.clone())) + ); + assert_eq!( + store.save(zone, &snap(1, b"old".to_vec())).await, + Err(StateError::Stale { + presented: 1, + current: 2 + }) + ); + assert_eq!( + store.delete(zone, 1).await, + Err(StateError::Stale { + presented: 1, + current: 2 + }) + ); + store.save(zone, &snap(3, b"c".to_vec())).await.unwrap(); + assert_eq!( + store.load(zone).await.unwrap(), + Some(snap(3, b"c".to_vec())) + ); + store.save(zone, &snap(4, payload.clone())).await.unwrap(); + store.delete(zone, 4).await.unwrap(); + assert_eq!(store.load(zone).await.unwrap(), None); + assert_eq!( + store.save(zone, &snap(3, b"late".to_vec())).await, + Err(StateError::Stale { + presented: 3, + current: 4 + }), + "the token survives a delete" + ); + store.save(zone, &snap(5, payload.clone())).await.unwrap(); + assert_eq!(store.load(zone).await.unwrap(), Some(snap(5, payload))); } #[cfg(test)] mod tests { use super::*; - fn snap(token: u64, bytes: &[u8]) -> StoredZoneState { - StoredZoneState { - token, - content_version: "c1".into(), - state: bytes.to_vec(), + fn block_on(future: F) -> F::Output { + use std::task::{Context, Poll, Waker}; + let mut future = std::pin::pin!(future); + match future + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + { + Poll::Ready(output) => output, + Poll::Pending => panic!("memory store futures are ready"), } } - fn exercise(store: &dyn ZoneStateStore) { - assert_eq!(store.load("z").unwrap(), None); - store.save("z", &snap(2, b"a")).unwrap(); - store.save("z", &snap(2, b"b")).unwrap(); - assert_eq!( - store.save("z", &snap(1, b"old")), - Err(StateError::Stale { - presented: 1, - current: 2 - }) - ); - store.save("z", &snap(3, b"c")).unwrap(); - assert_eq!(store.load("z").unwrap(), Some(snap(3, b"c"))); - } - #[test] fn memory_store_is_fenced() { - exercise(&MemoryZoneStateStore::new()); - } - - #[cfg(feature = "sqlite")] - #[test] - fn sqlite_stores_sharing_a_file_wait_for_each_other() { - // Hosts sharing one file (the multi-host demos) open and write at the - // same time; a colliding writer must wait, not fail "database is locked". - let path = std::env::temp_dir().join(format!( - "dg-zone-state-shared-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let threads: Vec<_> = (0..8u64) - .map(|host| { - let path = path.clone(); - std::thread::spawn(move || { - let store = SqliteZoneStateStore::open(&path)?; - for token in 1..=50 { - store.save(&format!("zone.{host}"), &snap(token, b"s"))?; - } - Ok::<_, StateError>(()) - }) - }) - .collect(); - for thread in threads { - thread.join().unwrap().unwrap(); - } - let store = SqliteZoneStateStore::open(&path).unwrap(); - for host in 0..8 { - assert_eq!( - store.load(&format!("zone.{host}")).unwrap(), - Some(snap(50, b"s")) - ); - } - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } - } - - #[cfg(feature = "sqlite")] - #[test] - fn sqlite_store_is_fenced_and_durable() { - exercise(&SqliteZoneStateStore::open_in_memory().unwrap()); - let path = std::env::temp_dir().join(format!( - "dg-zone-state-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - { - let store = SqliteZoneStateStore::open(&path).unwrap(); - store.save("realm.east", &snap(7, b"boss:dead")).unwrap(); - } - let reopened = SqliteZoneStateStore::open(&path).unwrap(); - assert_eq!( - reopened.load("realm.east").unwrap(), - Some(snap(7, b"boss:dead")) - ); - drop(reopened); - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } + let store = MemoryZoneStateStore::new(); + block_on(state_store_conformance(&store, "z", vec![7; 300_000])); } } diff --git a/distributed_game/src/world/zone_state.rs b/distributed_game/src/world/zone_state.rs new file mode 100644 index 000000000..aa6e93ed2 --- /dev/null +++ b/distributed_game/src/world/zone_state.rs @@ -0,0 +1,559 @@ +//! Hibernated zone state as a cell: `ZoneState:{zone_id}`. +//! +//! A sleeping zone is a sleeping Durable Object: its last snapshot lives in +//! its own aggregate cell on celld (the in-process reference host in unit +//! tests), written and read by zone servers through +//! `celld::CellZoneStateStore`. +//! +//! | Command | Effect | +//! |---|---| +//! | `zone_state.save` | store the snapshot if `token` >= the stored token | +//! | `zone_state.delete` | drop the snapshot if `token` >= the stored token | +//! +//! Fencing equals the [`ZoneStateStore`](super::ZoneStateStore) contract: an +//! older token is rejected with `stale:{current}`, an equal one is accepted, +//! and the highest token survives a delete. +//! +//! **Size.** A snapshot up to [`ZONE_STATE_INLINE_MAX`] bytes is stored in +//! the cell itself (in the `zone_state.saved` event and the aggregate +//! snapshot). A larger one goes to object storage: the worker route +//! (`celld_worker::fetch`, feature `workers-rs`) writes the bytes to an R2 bucket +//! binding of celld (the fleet bucket; RustFS in the Hops stack) under +//! `zone-state/{zone}/{token}-{sha256}` *before* the command, and the cell +//! records only `{key, sha256, len}`. The blob a save or delete supersedes +//! is deleted after the commit; a save refused as stale deletes its own +//! upload. Loads verify the length and digest. +//! +//! Every save is one small event (inline bytes or a blob reference); cells +//! load from the aggregate snapshot, which is refreshed on every commit. +//! +//! Callers must hold the `zone-host` role (an internal identity on the +//! authenticated internal path). + +use distributed::cell_host::AggregateCell; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, +}; +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; + +pub use super::zone_lease::ZONE_HOST_ROLE; + +/// Aggregate type (cell `ZoneState:{zone_id}`). +pub const ZONE_STATE: &str = "ZoneState"; +/// Save command. +pub const SAVE: &str = "zone_state.save"; +/// Delete command. +pub const DELETE: &str = "zone_state.delete"; +/// Rejection prefix of a stale token (`stale:{current}`). +pub const STALE_PREFIX: &str = "stale:"; +/// Largest snapshot kept inline in the cell; larger ones go to object +/// storage. +pub const ZONE_STATE_INLINE_MAX: usize = 64 * 1024; + +/// A snapshot stored in object storage. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, distributed::CommandInput)] +pub struct BlobRef { + /// Object key. + pub key: String, + /// Lowercase hex SHA-256 of the bytes. + pub sha256: String, + /// Length in bytes. + pub len: u64, +} + +/// Where a snapshot's bytes are. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum SnapshotBytes { + /// In the cell. + Inline(Vec), + /// In object storage. + Blob(BlobRef), +} + +impl SnapshotBytes { + fn blob_key(&self) -> Option<&str> { + match self { + Self::Blob(blob) => Some(&blob.key), + Self::Inline(_) => None, + } + } +} + +/// The hibernated state of one zone. +#[derive(Clone, Debug, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct ZoneState { + entity: Entity, + token: u64, + content_version: String, + bytes: Option, +} + +#[sourced(entity, aggregate_type = "ZoneState", events = "ZoneStateEvent")] +impl ZoneState { + #[event("zone_state.saved", version = 1)] + fn saved( + &mut self, + zone_id: String, + token: u64, + content_version: String, + bytes: SnapshotBytes, + ) { + if self.entity.id().is_empty() { + self.entity.set_id(zone_id); + } + self.token = token; + self.content_version = content_version; + self.bytes = Some(bytes); + } + + #[event("zone_state.deleted", version = 1)] + fn deleted(&mut self, zone_id: String, token: u64) { + if self.entity.id().is_empty() { + self.entity.set_id(zone_id); + } + self.token = token; + self.bytes = None; + } +} + +impl ZoneState { + /// Highest token seen (kept after a delete). + pub fn token(&self) -> u64 { + self.token + } + + /// Content version of the stored snapshot. + pub fn content_version(&self) -> &str { + &self.content_version + } + + /// The stored snapshot, if any. + pub fn bytes(&self) -> Option<&SnapshotBytes> { + self.bytes.as_ref() + } +} + +/// `zone_state.save` input: exactly one of `inline_b64` (standard base64, +/// at most [`ZONE_STATE_INLINE_MAX`] bytes decoded) and `blob`. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct SaveInput { + pub zone_id: String, + pub token: u64, + pub content_version: String, + pub inline_b64: Option, + pub blob: Option, +} + +/// `zone_state.delete` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct DeleteInput { + pub zone_id: String, + pub token: u64, +} + +/// Outcome of a save or delete. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, distributed::CommandOutput)] +pub struct StatePayload { + pub zone_id: String, + pub token: u64, + /// Object key the command superseded (delete it after the commit). + pub superseded_blob: Option, +} + +/// Map a rejected command's message to a stale token, if it is one. +pub fn stale_current(message: &str) -> Option { + let message = message.strip_prefix("rejected: ").unwrap_or(message); + message.strip_prefix(STALE_PREFIX)?.parse().ok() +} + +fn rejected(message: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(message.to_string()) +} + +fn is_zone_host(ctx: &CausalCommandContext<'_, ZoneState>) -> bool { + ctx.session().has_role(ZONE_HOST_ROLE) +} + +async fn fenced( + ctx: &CausalCommandContext<'_, ZoneState>, + zone_id: &str, + token: u64, +) -> Result, HandlerError> { + if zone_id.is_empty() { + return Err(rejected("zone_id must not be empty")); + } + let repo = ctx.repo(); + let state = match repo.get(zone_id).await? { + Some(state) => state, + None => repo.create(), + }; + if state.token > token { + return Err(rejected(format!("{STALE_PREFIX}{}", state.token))); + } + Ok(state) +} + +async fn handle_save( + ctx: &CausalCommandContext<'_, ZoneState>, + input: SaveInput, +) -> Result>, HandlerError> { + let bytes = match (input.inline_b64, input.blob) { + (Some(inline), None) => { + use base64::Engine as _; + let bytes = base64::engine::general_purpose::STANDARD + .decode(inline) + .map_err(|_| rejected("inline_b64 is not base64"))?; + if bytes.len() > ZONE_STATE_INLINE_MAX { + return Err(rejected(format!( + "inline snapshots are at most {ZONE_STATE_INLINE_MAX} bytes" + ))); + } + SnapshotBytes::Inline(bytes) + } + (None, Some(blob)) => { + let hex = blob.sha256.len() == 64 && blob.sha256.bytes().all(|b| b.is_ascii_hexdigit()); + if blob.key.is_empty() || !hex { + return Err(rejected("blob needs a key and a SHA-256")); + } + SnapshotBytes::Blob(blob) + } + _ => return Err(rejected("exactly one of inline_b64 and blob")), + }; + let mut state = fenced(ctx, &input.zone_id, input.token).await?; + let previous = state + .bytes + .as_ref() + .and_then(|b| b.blob_key()) + .map(str::to_string); + let superseded = previous.filter(|key| Some(key.as_str()) != bytes.blob_key()); + state + .saved( + input.zone_id.clone(), + input.token, + input.content_version, + bytes, + ) + .map_err(rejected)?; + ctx.repo().commit(state)?.succeeded(StatePayload { + zone_id: input.zone_id, + token: input.token, + superseded_blob: superseded, + }) +} + +async fn handle_delete( + ctx: &CausalCommandContext<'_, ZoneState>, + input: DeleteInput, +) -> Result>, HandlerError> { + let mut state = fenced(ctx, &input.zone_id, input.token).await?; + let superseded = state + .bytes + .as_ref() + .and_then(|b| b.blob_key()) + .map(str::to_string); + state + .deleted(input.zone_id.clone(), input.token) + .map_err(rejected)?; + ctx.repo().commit(state)?.succeeded(StatePayload { + zone_id: input.zone_id, + token: input.token, + superseded_blob: superseded, + }) +} + +struct Save; +impl PortableCommand for Save +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(SAVE)) + .guarded(is_zone_host, handle_save) + } +} + +struct Delete; +impl PortableCommand for Delete +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + DELETE, + )) + .guarded(is_zone_host, handle_delete) + } +} + +/// Mount both zone-state commands on a `ZoneState` cell. +pub fn mount_zone_state(cell: AggregateCell) -> AggregateCell { + cell.mount(Save).mount(Delete) +} + +/// Lowercase hex SHA-256. +pub fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// Object key of a large snapshot. +pub fn blob_key(zone_id: &str, token: u64, sha256: &str) -> String { + format!("zone-state/{zone_id}/{token}-{sha256}") +} + +/// celld worker glue: the `zone-state/{zone}` HTTP surface over a +/// `ZoneState` cell and an R2 bucket binding (see the module docs). The +/// caller authenticates the internal secret first. +#[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] +pub mod celld_worker { + use super::*; + use ::worker::{Bucket, Method, Request, Response, Result}; + use base64::Engine as _; + use distributed::cell_host::{ + CellCommandIdentity, CellDispatchError, CellWaitPathRequest, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, + }; + use distributed::microsvc::Session; + use serde_json::{json, Value}; + + /// Wire input of a save from `celld::CellZoneStateStore`. + #[derive(Deserialize)] + struct WireSave { + zone_id: String, + token: u64, + content_version: String, + state_b64: String, + } + + fn json_status(body: Value, status: u16) -> Result { + Ok(Response::from_json(&body)?.with_status(status)) + } + + fn error_response(error: CellDispatchError) -> Result { + json_status( + json!({ "error": error.client_message(), "code": error.code() }), + error.status_code(), + ) + } + + /// The R2 bucket binding `name`. celld's R2 binding is not an instance + /// of the Workers runtime's `R2Bucket` class, which `Env::bucket` + /// checks for, but it has the same methods; take it unchecked. + pub fn bucket(env: &::worker::Env, name: &str) -> Result { + use ::worker::wasm_bindgen::{JsCast, JsValue}; + let env: &JsValue = env.as_ref(); + let binding = ::worker::js_sys::Reflect::get(env, &JsValue::from_str(name)) + .map_err(|_| ::worker::Error::RustError(format!("no binding `{name}`")))?; + if binding.is_undefined() { + return Err(::worker::Error::RustError(format!( + "R2 bucket binding `{name}` is not configured" + ))); + } + Ok(binding.unchecked_into::()) + } + + /// `GET /zone-state/{zone}` → `{state: null | {token, content_version, + /// state_b64}}`; `POST /zone-state/{zone}/zone_state.save|delete`. + pub async fn fetch( + cell: &AggregateCell, + bucket: &Bucket, + mut req: Request, + ) -> Result { + let url = req.url()?; + let command = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .nth(2) + .map(str::to_string); + match (req.method(), command.as_deref()) { + (Method::Get, None) => load(cell, bucket).await, + (Method::Post, Some(SAVE)) | (Method::Post, Some(DELETE)) => { + let command = command.unwrap(); + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => { + return error_response(CellDispatchError::BadRequest("invalid JSON".into())) + } + }; + let parsed = match CellWaitPathRequest::parse(body) { + Ok(parsed) => parsed, + Err(error) => return error_response(CellDispatchError::BadRequest(error)), + }; + let header = |name: &str| req.headers().get(name).ok().flatten(); + let identity = match ( + header(CELL_SERVICE_ID_HEADER), + header(CELL_PRINCIPAL_PARTITION_HEADER), + ) { + (Some(service), Some(partition)) => { + match CellCommandIdentity::new(service, partition, &parsed.command_id) { + Ok(identity) => identity, + Err(error) => return error_response(error), + } + } + _ => return error_response(CellDispatchError::Unauthorized), + }; + let mut session = Session::new(); + for key in [distributed::USER_ID_KEY, distributed::microsvc::ROLE_KEY] { + if let Some(value) = header(key) { + session.set(key, value); + } + } + if command == SAVE { + save(cell, bucket, &identity, parsed.input, session).await + } else { + dispatch(cell, bucket, DELETE, &identity, parsed.input, session, None).await + } + } + _ => json_status(json!({ "error": "not found" }), 404), + } + } + + async fn load(cell: &AggregateCell, bucket: &Bucket) -> Result { + let state = cell + .load() + .await + .map_err(|error| ::worker::Error::RustError(error.to_string()))?; + let Some(state) = state else { + return json_status(json!({ "state": null }), 200); + }; + let bytes = match state.bytes() { + None => return json_status(json!({ "state": null, "token": state.token() }), 200), + Some(SnapshotBytes::Inline(bytes)) => bytes.clone(), + Some(SnapshotBytes::Blob(blob)) => { + let object = bucket.get(blob.key.clone()).execute().await?; + let Some(body) = object.as_ref().and_then(|object| object.body()) else { + return error_response(CellDispatchError::Internal(format!( + "zone-state blob {} is missing", + blob.key + ))); + }; + let bytes = body.bytes().await?; + if bytes.len() as u64 != blob.len || sha256_hex(&bytes) != blob.sha256 { + return error_response(CellDispatchError::Internal(format!( + "zone-state blob {} does not match its digest", + blob.key + ))); + } + bytes + } + }; + json_status( + json!({ "state": { + "token": state.token(), + "content_version": state.content_version(), + "state_b64": base64::engine::general_purpose::STANDARD.encode(bytes), + }}), + 200, + ) + } + + async fn save( + cell: &AggregateCell, + bucket: &Bucket, + identity: &CellCommandIdentity, + input: Value, + session: Session, + ) -> Result { + let wire: WireSave = match serde_json::from_value(input) { + Ok(wire) => wire, + Err(error) => { + return error_response(CellDispatchError::BadRequest(format!( + "zone_state.save input: {error}" + ))) + } + }; + let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(&wire.state_b64) else { + return error_response(CellDispatchError::BadRequest( + "state_b64 is not base64".into(), + )); + }; + let (command_input, uploaded) = if bytes.len() <= ZONE_STATE_INLINE_MAX { + ( + json!({ + "zone_id": wire.zone_id, "token": wire.token, + "content_version": wire.content_version, + "inline_b64": wire.state_b64, "blob": null, + }), + None, + ) + } else { + let sha256 = sha256_hex(&bytes); + let key = blob_key(&wire.zone_id, wire.token, &sha256); + let len = bytes.len() as u64; + bucket.put(key.clone(), bytes).execute().await?; + ( + json!({ + "zone_id": wire.zone_id, "token": wire.token, + "content_version": wire.content_version, "inline_b64": null, + "blob": { "key": key, "sha256": sha256, "len": len }, + }), + Some(key), + ) + }; + dispatch( + cell, + bucket, + SAVE, + identity, + command_input, + session, + uploaded, + ) + .await + } + + async fn dispatch( + cell: &AggregateCell, + bucket: &Bucket, + command: &str, + identity: &CellCommandIdentity, + input: Value, + session: Session, + uploaded: Option, + ) -> Result { + match cell + .dispatch_idempotent(command, identity, input, session) + .await + { + Ok(dispatch) => { + let superseded = dispatch + .payload() + .get("superseded_blob") + .and_then(Value::as_str) + .map(str::to_string); + if let Some(key) = superseded { + if let Err(error) = bucket.delete(key.clone()).await { + ::worker::console_error!("superseded zone-state blob {key}: {error}"); + } + } + json_status( + json!({ + "payload": dispatch.payload(), + "receipt": { "commandId": dispatch.command_id(), "replayed": dispatch.replayed() }, + }), + 200, + ) + } + Err(error) => { + if let CellDispatchError::Internal(message) = &error { + ::worker::console_error!("zone-state {command}: {message}"); + } + // Refused (stale or invalid): the upload is nobody's. + if let Some(key) = uploaded { + if let Err(error) = bucket.delete(key.clone()).await { + ::worker::console_error!("refused zone-state blob {key}: {error}"); + } + } + error_response(error) + } + } + } +} diff --git a/distributed_game/tests/celld_zone_state/main.rs b/distributed_game/tests/celld_zone_state/main.rs new file mode 100644 index 000000000..bf1d2b1b8 --- /dev/null +++ b/distributed_game/tests/celld_zone_state/main.rs @@ -0,0 +1,61 @@ +//! Hibernation snapshots in celld cells (`CellZoneStateStore` → +//! `ZoneState:{zone}`) on an isolated `celld dev`: the zone-state fencing +//! contract for an inline snapshot and for one above the inline limit (in +//! celld's R2 object storage), and both surviving a celld crash. +//! +//! (cd tests/celld/fact-worker && worker-build --release) +//! cargo test -p distributed_game --features celld,runtime --test celld_zone_state + +use distributed::cell_host::InternalHttpSecret; +use distributed_game::celld::CellZoneStateStore; +use distributed_game::world::zone_state::ZONE_STATE_INLINE_MAX; +use distributed_game::world::{state_store_conformance, StoredZoneState, ZoneStateStore}; + +#[path = "../support/celld.rs"] +mod celld_support; +use celld_support::{Celld, SECRET}; + +fn store(base: &str) -> CellZoneStateStore { + CellZoneStateStore::new( + base, + InternalHttpSecret::new(SECRET).unwrap(), + "zone-state-test", + "host-a", + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn snapshots_are_fenced_inline_and_in_object_storage_and_survive_a_crash() { + let Some(mut celld) = Celld::start_new() else { + return; + }; + let states = store(&celld.base()); + state_store_conformance(&states, "grid.0.0", vec![1; 512]).await; + let big: Vec = (0..ZONE_STATE_INLINE_MAX * 3) + .map(|i| (i % 253) as u8) + .collect(); + state_store_conformance(&states, "grid.0.1", big.clone()).await; + + // celld crashes (process tree killed) and restarts on its state. + celld.crash(); + celld.start().expect("celld restarts"); + let states = store(&celld.base()); + let small = states.load("grid.0.0").await.unwrap().unwrap(); + assert_eq!((small.token, small.state.len()), (5, 512)); + let large = states.load("grid.0.1").await.unwrap().unwrap(); + assert_eq!(large.token, 5); + assert_eq!(large.state, big, "the object came back byte for byte"); + // Fencing state survived too. + assert!(states + .save( + "grid.0.1", + &StoredZoneState { + token: 4, + content_version: "c1".into(), + state: vec![0; 10], + }, + ) + .await + .is_err()); +} diff --git a/distributed_game/tests/support/celld.rs b/distributed_game/tests/support/celld.rs index 12460743c..4157bdd47 100644 --- a/distributed_game/tests/support/celld.rs +++ b/distributed_game/tests/support/celld.rs @@ -79,7 +79,9 @@ impl Celld { /// celld on PATH, or no esbuild for its bundler). pub fn start(&mut self) -> Result<(), String> { self.generation += 1; - let binary = std::env::var("CELLD").unwrap_or_else(|_| "celld".into()); + let binary = std::env::var("CELLD_BIN") + .or_else(|_| std::env::var("CELLD")) + .unwrap_or_else(|_| "celld".into()); let help = Command::new(&binary) .args(["dev", "--help"]) .output() diff --git a/distributed_game/tests/world/main.rs b/distributed_game/tests/world/main.rs index 0c9adf901..165419928 100644 --- a/distributed_game/tests/world/main.rs +++ b/distributed_game/tests/world/main.rs @@ -1,7 +1,12 @@ //! Two GameServers in one process on different ports: cross-host handoff //! with a real WebSocket client, tier-up events across hosts, hibernation, -//! fenced ownership and the world content barrier. +//! fenced ownership and the world content barrier. Hibernation runs on +//! celld (`CellZoneOwnership` + `CellZoneStateStore` against +//! `tests/celld/fact-worker`; skipped unless the worker is built and celld +//! starts, required with `CELLD_FACTS_REQUIRED=1`). +#[path = "../support/celld.rs"] +mod celld_support; mod realm; use std::sync::{Arc, Mutex}; @@ -14,8 +19,8 @@ use distributed_game::net::{ }; use distributed_game::protocol::{error, kick, Hello, Notice}; use distributed_game::world::{ - FencedLeaseOwnership, HostAddr, LocalOwnership, SqliteZoneStateStore, StaticDirectory, - ZoneDirectory, ZoneOwnership, ZoneStateStore, + FencedLeaseOwnership, HostAddr, LocalOwnership, StaticDirectory, ZoneDirectory, ZoneOwnership, + ZoneStateStore, }; use distributed_game::*; use futures_util::{SinkExt, StreamExt}; @@ -503,16 +508,11 @@ async fn boss_defeat_tiers_up_to_parent_and_grandparent_across_hosts() { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn dormant_zone_hibernates_to_sqlite_and_restores() { - let path = std::env::temp_dir().join(format!( - "dg-world-hibernate-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let store = Arc::new(SqliteZoneStateStore::open(&path).unwrap()); +async fn dormant_zone_hibernates_to_celld_and_restores() { + let Some(celld) = celld_support::Celld::start_new() else { + return; + }; + let (ownership, store) = on_celld(&celld, "host-a"); let mut config = base_config(); config.host.idle_tick_rate = None; config.host.dormant_after = Duration::from_millis(50); @@ -522,6 +522,7 @@ async fn dormant_zone_hibernates_to_sqlite_and_restores() { let factory = realm_factory(&[], &[]); let server = GameServer::::new(config, ZoneFactory::sync(move |a| factory(&a.zone_id))) .with_fact_sink(sink.clone(), 64, RetryPolicy::default()) + .with_ownership(ownership) .with_state_store(store.clone()); serve(&server, listener); let url = format!("ws://{addr}/zones/realm.vault"); @@ -545,7 +546,11 @@ async fn dormant_zone_hibernates_to_sqlite_and_restores() { async move { !server.is_active("realm.vault") && hibernation_outcomes(&server) >= 1 } }) .await; - let saved = store.load("realm.vault").unwrap().expect("snapshot stored"); + let saved = store + .load("realm.vault") + .await + .unwrap() + .expect("snapshot stored"); assert_eq!(saved.state, 3u32.to_le_bytes()); // Facts emitted while disposing were still committed. eventually("dispose fact", || { @@ -578,28 +583,51 @@ async fn dormant_zone_hibernates_to_sqlite_and_restores() { } }; assert_eq!(counter, 3); - drop(store); - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } +} + +/// Zone ownership and hibernation snapshots in celld cells. +fn on_celld( + celld: &celld_support::Celld, + host: &str, +) -> ( + Arc, + Arc, +) { + let secret = || distributed::cell_host::InternalHttpSecret::new(celld_support::SECRET).unwrap(); + ( + Arc::new( + distributed_game::celld::CellZoneOwnership::new( + celld.base(), + secret(), + "world-test", + host, + ) + .unwrap(), + ), + Arc::new( + distributed_game::celld::CellZoneStateStore::new( + celld.base(), + secret(), + "world-test", + host, + ) + .unwrap(), + ), + ) } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { - let path = std::env::temp_dir().join(format!( - "dg-world-restart-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); + let Some(celld) = celld_support::Celld::start_new() else { + return; + }; let url_of = |addr: &str| format!("ws://{addr}/zones/realm.keep"); let mut epochs = Vec::new(); let mut count = 0u32; for run in 1..=2 { - // A fresh process: new store handle, new (default) LocalOwnership. - let store = Arc::new(SqliteZoneStateStore::open(&path).unwrap()); + // A fresh process: new store and ownership handles (the fencing + // tokens live in the celld lease cell, so they keep growing). + let (ownership, store) = on_celld(&celld, &format!("host-{run}")); let mut config = base_config(); config.host.idle_tick_rate = None; config.host.dormant_after = Duration::from_millis(50); @@ -608,6 +636,7 @@ async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { let factory = realm_factory(&[], &[]); let server = GameServer::::new(config, ZoneFactory::sync(move |a| factory(&a.zone_id))) + .with_ownership(ownership) .with_state_store(store.clone()); serve(&server, listener); // Two activations per run, so the first run's last token exceeds @@ -646,7 +675,7 @@ async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { cycle, "run {run}" ); - let saved = store.load("realm.keep").unwrap().expect("snapshot"); + let saved = store.load("realm.keep").await.unwrap().expect("snapshot"); assert_eq!(saved.state, count.to_le_bytes(), "run {run} saved"); } assert!(server @@ -659,9 +688,6 @@ async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { epochs.windows(2).all(|w| w[1] > w[0]), "epochs grow: {epochs:?}" ); - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/distributed_game/tests/zone_state/main.rs b/distributed_game/tests/zone_state/main.rs new file mode 100644 index 000000000..29b9665b0 --- /dev/null +++ b/distributed_game/tests/zone_state/main.rs @@ -0,0 +1,243 @@ +//! `ZoneState` cells on the in-process reference host: the zone-state +//! fencing contract (`state_store_conformance`) through the cell's +//! commands, for inline snapshots and for snapshots in object storage (an +//! in-memory blob map standing in for celld's R2 binding, split exactly as +//! the worker route splits them), plus the command's own guards. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +use base64::Engine as _; +use distributed::cell_host::{ + AggregateCell, CellCommandIdentity, CellDispatchError, CellStreamStore, +}; +use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; +use distributed::StreamIdentity; +use distributed_game::cell::command_id_for; +use distributed_game::world::zone_state::{ + blob_key, mount_zone_state, sha256_hex, stale_current, SnapshotBytes, ZoneState, DELETE, SAVE, + ZONE_HOST_ROLE, ZONE_STATE_INLINE_MAX, +}; +use distributed_game::world::{ + state_store_conformance, StateError, StoredZoneState, ZoneStateStore, +}; +use distributed_game::BoxFuture; +use serde_json::{json, Value}; + +const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; + +fn host_session() -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, "zone-server-a"); + session.set(ROLE_KEY, ZONE_HOST_ROLE); + session +} + +/// `ZoneStateStore` over one in-process cell + a blob map, as +/// `CellZoneStateStore` + the worker route are over celld. +struct InProcess { + cell: AggregateCell, + blobs: Mutex>>, + calls: AtomicU64, +} + +impl InProcess { + fn new(zone: &str) -> Self { + let store = CellStreamStore::for_identity(StreamIdentity::new("ZoneState", zone).unwrap()); + Self { + cell: mount_zone_state(AggregateCell::from_store(store).unwrap()), + blobs: Mutex::default(), + calls: AtomicU64::new(0), + } + } + + async fn dispatch( + &self, + command: &str, + input: Value, + session: Session, + ) -> Result { + let n = self.calls.fetch_add(1, Ordering::SeqCst); + let identity = CellCommandIdentity::new( + "zone-host", + "zone-server-a", + command_id_for("t", &n.to_string()), + ) + .unwrap(); + self.cell + .dispatch_idempotent(command, &identity, input, session) + .await + .map(|result| result.payload().clone()) + } + + async fn command( + &self, + command: &str, + input: Value, + uploaded: Option, + ) -> Result<(), StateError> { + let token = input["token"].as_u64().unwrap(); + match self.dispatch(command, input, host_session()).await { + Ok(payload) => { + if let Some(key) = payload["superseded_blob"].as_str() { + self.blobs.lock().unwrap().remove(key); + } + Ok(()) + } + Err(error) => { + if let Some(key) = uploaded { + self.blobs.lock().unwrap().remove(&key); + } + match error { + CellDispatchError::Rejected { message, .. } => match stale_current(&message) { + Some(current) => Err(StateError::Stale { + presented: token, + current, + }), + None => Err(StateError::Backend(message)), + }, + other => Err(StateError::Backend(format!("{other:?}"))), + } + } + } + } +} + +impl ZoneStateStore for InProcess { + fn load<'a>( + &'a self, + _zone_id: &'a str, + ) -> BoxFuture<'a, Result, StateError>> { + Box::pin(async move { + let Some(state) = self.cell.load().await.unwrap() else { + return Ok(None); + }; + let bytes = match state.bytes() { + None => return Ok(None), + Some(SnapshotBytes::Inline(bytes)) => bytes.clone(), + Some(SnapshotBytes::Blob(blob)) => { + let bytes = self + .blobs + .lock() + .unwrap() + .get(&blob.key) + .cloned() + .expect("blob"); + assert_eq!( + (bytes.len() as u64, sha256_hex(&bytes)), + (blob.len, blob.sha256.clone()) + ); + bytes + } + }; + Ok(Some(StoredZoneState { + token: state.token(), + content_version: state.content_version().to_string(), + state: bytes, + })) + }) + } + + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a StoredZoneState, + ) -> BoxFuture<'a, Result<(), StateError>> { + Box::pin(async move { + if state.state.len() <= ZONE_STATE_INLINE_MAX { + let input = json!({ + "zone_id": zone_id, "token": state.token, "content_version": state.content_version, + "inline_b64": B64.encode(&state.state), "blob": null, + }); + self.command(SAVE, input, None).await + } else { + let sha = sha256_hex(&state.state); + let key = blob_key(zone_id, state.token, &sha); + self.blobs + .lock() + .unwrap() + .insert(key.clone(), state.state.clone()); + let input = json!({ + "zone_id": zone_id, "token": state.token, "content_version": state.content_version, + "inline_b64": null, + "blob": { "key": key, "sha256": sha, "len": state.state.len() }, + }); + self.command(SAVE, input, Some(key)).await + } + }) + } + + fn delete<'a>(&'a self, zone_id: &'a str, token: u64) -> BoxFuture<'a, Result<(), StateError>> { + Box::pin(async move { + self.command(DELETE, json!({ "zone_id": zone_id, "token": token }), None) + .await + }) + } +} + +#[tokio::test] +async fn inline_snapshots_meet_the_zone_state_contract() { + let store = InProcess::new("grid.0.0"); + state_store_conformance(&store, "grid.0.0", vec![3; 1_000]).await; + assert!( + store.blobs.lock().unwrap().is_empty(), + "small snapshots stay in the cell" + ); +} + +#[tokio::test] +async fn large_snapshots_go_to_object_storage_and_superseded_blobs_are_dropped() { + let store = InProcess::new("grid.1.0"); + let big = (0..ZONE_STATE_INLINE_MAX + 4_096) + .map(|i| (i % 251) as u8) + .collect::>(); + state_store_conformance(&store, "grid.1.0", big.clone()).await; + // Only the live snapshot's object remains (the stale upload, the + // superseded saves and the deleted one are gone). + let blobs = store.blobs.lock().unwrap(); + assert_eq!(blobs.len(), 1, "{:?}", blobs.keys().collect::>()); + assert_eq!(blobs.values().next().unwrap(), &big); + drop(blobs); + let state = store.cell.load().await.unwrap().unwrap(); + assert!( + matches!(state.bytes(), Some(SnapshotBytes::Blob(blob)) if blob.len == big.len() as u64) + ); +} + +#[tokio::test] +async fn commands_need_the_zone_host_role_and_one_well_formed_payload() { + let store = InProcess::new("grid.2.0"); + let input = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", + "inline_b64": B64.encode(b"x"), "blob": null, + }); + let mut player = Session::new(); + player.set(USER_ID_KEY, "alice"); + assert!( + store.dispatch(SAVE, input.clone(), player).await.is_err(), + "players cannot write zone state" + ); + + let both = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", "inline_b64": B64.encode(b"x"), + "blob": { "key": "k", "sha256": sha256_hex(b"x"), "len": 1 }, + }); + let neither = json!({ "zone_id": "grid.2.0", "token": 1, "content_version": "c", "inline_b64": null, "blob": null }); + let oversized = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", + "inline_b64": B64.encode(vec![0u8; ZONE_STATE_INLINE_MAX + 1]), "blob": null, + }); + let bad_digest = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", "inline_b64": null, + "blob": { "key": "k", "sha256": "nope", "len": 1 }, + }); + for bad in [both, neither, oversized, bad_digest] { + assert!(matches!( + store.dispatch(SAVE, bad, host_session()).await, + Err(CellDispatchError::Rejected { .. }) + )); + } + store.dispatch(SAVE, input, host_session()).await.unwrap(); + assert_eq!(store.load("grid.2.0").await.unwrap().unwrap().state, b"x"); +} diff --git a/examples/game/README.md b/examples/game/README.md index 7744b7c4a..650321d73 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -6,16 +6,16 @@ Small playable demos of the authoritative game runtime on one mechanic. **Dev only**: the demo server mints tickets for anonymous guests and lets any page change the network simulation. -| Demo | Run | Mechanic it demonstrates | -|---|---|---| -| **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | -| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player celld cells (`cells/`, `celld dev` started by `run.sh`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence cells fed by the runtime's player lifecycle facts (`/dev/mobpit/presence`) | -| **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | -| **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | -| **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | -| **Endless Grid** | `./run.sh grid` | Seeded procedural world (terrain/biomes/props generated bit-identically in Rust and TS, nothing static replicated), 32 m cells as zones, observer connections to the 8 neighbours, handoff across borders **and hosts** (2 processes, hash ring, Redirect + transfer tickets), hibernation to SQLite and restore | -| **Mob Pit · SvelteKit** | `./run.sh sveltekit` | The same Mob Pit server behind a SvelteKit app: Auth.js session sign-in, admission tickets minted from the session (`createGameTicketHandler`), the game client and WASM step generated by the Vite plugin (`games`), a runes HUD (`useGame`) next to a plain canvas loop, and the settled kills as a business-side read model: Crawler domain events projected by `mobpit-service` into SQL and read through generated `@load @live` GraphQL (SSR, then live). See [SvelteKit](#sveltekit) | -| **Boss Ripple** | `./run.sh ripple` | Zone hierarchy realm › city › borough › hood (`DottedTopology`), per-tier tick rates (2/5/10/30 Hz), boss defeats propagating up as tier-up events (each hop on another host), parent broadcasts reaching players through observer connections | +| Demo | Run | Mechanic it demonstrates | Durable state | +|---|---|---|---| +| **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | none (stateless: positions only live in the zone) | +| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player celld cells (`cells/`, `celld dev` started by `run.sh`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence cells fed by the runtime's player lifecycle facts (`/dev/mobpit/presence`) | celld: `crawler/{player}`, `crawler-roster/all`, `presence/{player}` | +| **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | celld (the Mob Pit cells); boards are a projection (memory or Redis) | +| **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | none (stateless: score is per session) | +| **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | celld: `gem-pouch/{player}`, `content-log/{zone}` | +| **Endless Grid** | `./run.sh grid` | Seeded procedural world (terrain/biomes/props generated bit-identically in Rust and TS, nothing static replicated), 32 m cells as zones, observer connections to the 8 neighbours, handoff across borders **and hosts** (2 processes, hash ring, Redirect + transfer tickets), hibernation to celld and restore | celld: `zone-lease/{zone}`, `zone-state/{zone}` (hibernation) | +| **Mob Pit · SvelteKit** | `./run.sh sveltekit` | The same Mob Pit server behind a SvelteKit app: Auth.js session sign-in, admission tickets minted from the session (`createGameTicketHandler`), the game client and WASM step generated by the Vite plugin (`games`), a runes HUD (`useGame`) next to a plain canvas loop, and the settled kills as a business-side read model: Crawler domain events projected by `mobpit-service` into SQL and read through generated `@load @live` GraphQL (SSR, then live). See [SvelteKit](#sveltekit) | celld (the Mob Pit cells) + `mobpit-service`'s SQL read model | +| **Boss Ripple** | `./run.sh ripple` | Zone hierarchy realm › city › borough › hood (`DottedTopology`), per-tier tick rates (2/5/10/30 Hz), boss defeats propagating up as tier-up events (each hop on another host), parent broadcasts reaching players through observer connections | celld: `zone-lease/{zone}`, `zone-state/{zone}` (hibernation) | `./run.sh smoke` runs every demo's smoke tests (and checks the generated clients are current, and drives the SvelteKit demo headlessly); `./run.sh gen` regenerates the clients; `./run.sh @@ -131,8 +131,12 @@ is about a client receiving content it was not built with. Each zone switch emits a typed `ContentApplied` fact (`freeze.content_applied`) and a stamped event; gems emit `GemCollected` facts (`freeze.gem_collected`, naming the gem spot — the gem's value is a -content-pack rule for the handler, not a zone-reported amount) — the admin -panel lists them with their stamp. Display names live on the primary host: a player joining Ember +content-pack rule for the handler, not a zone-reported amount). Both hosts +settle them in celld (`run.sh` starts `celld dev`; both processes share it +through `CELLD_URL` / `DISTRIBUTED_INTERNAL_SECRET`): gems in the player's +`gem-pouch/{principal}` cell, content switches in the zone's +`content-log/{zone}` cell. The admin panel lists the last settled facts with +their stamp. Display names live on the primary host: a player joining Ember Fields directly (`&zone=freeze-ember`) shows as a guest there, names carry over portals. @@ -146,17 +150,20 @@ examples/game/run.sh grid Two `game-demos grid` processes: host **A** on `GAME_DEMOS_PORT` (Vite proxies to it) and **B** on the next port, each listing the other with `--peer`, sharing one ticket secret (the admin secret for host-to-host calls -is derived from it) and keeping hibernation snapshots under one -`--state-dir` (`grid-A.sqlite3`, `grid-B.sqlite3`). - -Zone ownership is process-local (`LocalOwnership`) by default. With -`CELLD_URL` and `DISTRIBUTED_INTERNAL_SECRET` set (or `--celld-url` / -`--internal-secret`), every host keeps its zone claims in celld -`ZoneLease:{zone}` cells (`CellZoneOwnership`; a celld project mounting -`distributed_game::world::zone_lease`, such as `tests/celld/fact-worker`), -and a host that stops renewing gets a `zone.lease_expired` from the cell's -alarm. The cluster smoke tests honour the same variables: -`CELLD_URL=… DISTRIBUTED_INTERNAL_SECRET=… cargo test --test grid_smoke`. +is derived from it) and one celld (`run.sh` starts `celld dev` with +`cells/`; `--celld-url` / `CELLD_URL` and `DISTRIBUTED_INTERNAL_SECRET` are +required, there is no local fallback): + +- **Ownership**: every zone claim is a celld `ZoneLease:{zone}` cell + (`CellZoneOwnership`); a host that stops renewing gets a + `zone.lease_expired` from the cell's alarm. +- **Hibernation**: every snapshot is a celld `ZoneState:{zone}` cell + (`CellZoneStateStore`), fenced by the lease token; a sleeping zone is a + sleeping Durable Object. Snapshots above 64 KiB go to celld's object + storage (R2 binding `ZONE_SNAPSHOTS`), the cell keeps their digest. + +The cluster smoke tests (`tests/grid_smoke.rs`, `tests/ripple_smoke.rs`) +and the headless tests start their own `celld dev` the same way. - **World.** Cell `(x, y)` is zone `grid..` (canonical integers, |x|,|y| ≤ 10⁶; the server uses `FlatTopology`, the dots are not a @@ -182,7 +189,8 @@ alarm. The cluster smoke tests honour the same variables: `cell × 32 + local` around a floating origin. - **Hibernation.** Nobody inside or observing → Idle → Dormant after 2 s → hibernated after 4 s more (`--hibernate-after-ms`): critters and markers - (E) go to SQLite and come back on the next activation; its hello says + (E) go to the zone's celld `ZoneState` cell and come back on the next + activation (on either host); its hello says `restored` and the page toasts. The minimap (`GET /dev/grid/map`, which asks the peer for its cells) shows active / hibernated / cold cells per host; borders are tinted by owning host. diff --git a/examples/game/cells/Cargo.toml b/examples/game/cells/Cargo.toml index 84c70ca40..b9213dc34 100644 --- a/examples/game/cells/Cargo.toml +++ b/examples/game/cells/Cargo.toml @@ -1,6 +1,8 @@ -# Mob Pit's durable cells on celld (workers-rs, wasm32): the per-player -# Crawler aggregate, the crawler roster, per-player presence, and the outbox -# Queue consumer that relays Crawler domain events to the zone server. +# The game demos' durable cells on celld (workers-rs, wasm32): Mob Pit's +# per-player Crawler aggregate, the crawler roster and per-player presence +# (with the outbox Queue consumer that relays Crawler domain events to the +# zone server), and the zone lease and hibernation-state cells every +# multi-host demo uses. # Own workspace (cdylib for wasm32-unknown-unknown); built with # `worker-build --release` by `examples/game/run.sh`. [workspace] @@ -12,7 +14,7 @@ name = "mobpit-cells" version = "0.1.0" edition = "2021" publish = false -description = "Mob Pit Crawler, roster and presence cells for celld (dev demo)" +description = "Game demo cells for celld: Mob Pit, zone leases and zone state (dev demo)" [lib] crate-type = ["cdylib"] @@ -20,7 +22,7 @@ crate-type = ["cdylib"] [dependencies] console_error_panic_hook = "0.1" distributed = { path = "../../..", default-features = false, features = ["workers-rs"] } -distributed_game = { path = "../../../distributed_game", default-features = false, features = ["cell"] } +distributed_game = { path = "../../../distributed_game", default-features = false, features = ["cell", "workers-rs"] } mobpit_crawler = { path = "../server/crawler" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs index 9a372491a..95ddfe272 100644 --- a/examples/game/cells/src/lib.rs +++ b/examples/game/cells/src/lib.rs @@ -1,5 +1,7 @@ -//! Mob Pit's durable cells on celld, fed by the zone server's -//! `distributed_game::celld::CelldFactSink` over the internal HTTP path. +//! The game demos' durable cells on celld. Mob Pit's are fed by the zone +//! server's `distributed_game::celld::CelldFactSink` over the internal HTTP +//! path; every multi-host demo keeps its zone ownership and hibernation +//! snapshots here. //! //! - `crawler/{principal}` (`CrawlerCell`): the `mobpit_crawler::Crawler` //! aggregate with its `#[command]` `crawler.record_kill` (XP decided here @@ -15,11 +17,23 @@ //! facts (`zone.*`). A fact emitted before the one that set the current //! entry (`FactStamp::emission_order`) is ignored, so a late "left pit-1" //! cannot mark a player who already joined pit-2 offline. +//! - `gem-pouch/{principal}` (`GemPouchCell`): The Freeze's +//! `freeze.gem_collected` facts (which gem spot, at which content +//! version; the zone reports no amount). +//! - `content-log/{zone}` (`ContentLogCell`): The Freeze's +//! `freeze.content_applied` zone facts (every content switch of a zone). +//! - `zone-lease/{zone}` (`ZoneLeaseCell`): zone ownership +//! (`distributed_game::world::zone_lease`, `CellZoneOwnership`); its +//! alarm declares `zone.lease_expired`. +//! - `zone-state/{zone}` (`ZoneStateCell`): hibernation snapshots +//! (`distributed_game::world::zone_state`, `CellZoneStateStore`); large +//! ones in the `ZONE_SNAPSHOTS` R2 bucket. //! -//! Every route requires `DISTRIBUTED_INTERNAL_SECRET`; commands also require -//! the trusted fact session (principal + `FactStamp` headers, rebuilt with -//! `fact_session_from_headers`), so nothing but a zone server's fact worker -//! can run them. +//! Every route requires `DISTRIBUTED_INTERNAL_SECRET`. Mob Pit commands also +//! require the trusted fact session (principal + `FactStamp` headers, +//! rebuilt with `fact_session_from_headers`), so nothing but a zone server's +//! fact worker can run them; zone lease and state commands require the +//! `zone-host` role. //! //! The Queue consumer relays each outbox message to `CELLD_QUEUE_RELAY_URL` //! (the zone server's authenticated `/internal/celld-queue/relay`), which @@ -36,13 +50,15 @@ use distributed::cell_host::{ }; use distributed::command::{typed_command, PreparedCommand, Succeeded}; use distributed::microsvc::{ - CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, Session, }; use distributed::{sourced, Aggregate, Entity}; use distributed_game::cell::{fact_session_from_headers, session_stamp}; use distributed_game::lifecycle::{ Lifecycle, CONNECTION_LOST, PLAYER_DISCONNECTED, PLAYER_JOINED, PLAYER_LEFT, PLAYER_RECONNECTED, }; +use distributed_game::world::zone_lease::{mount_zone_lease, ZoneLease}; +use distributed_game::world::zone_state::{celld_worker, mount_zone_state, ZoneState}; use distributed_game::FactStamp; use mobpit_crawler::{ Crawler, MobKilledArgs, RecordKill, CRAWLER_KIND, ENLIST, MOB_KILLED_EVENT, PRESENCE_KIND, @@ -274,13 +290,248 @@ async fn handle_lifecycle( .succeeded(PresenceApplied { applied: newer }) } +// ---------------------------------------------------------------- The Freeze + +/// `freeze.gem_collected`. +pub const GEM_COLLECTED: &str = "freeze.gem_collected"; +/// `freeze.content_applied`. +pub const CONTENT_APPLIED: &str = "freeze.content_applied"; + +/// One collected gem. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Gem { + pub zone: String, + pub spot: u32, + pub content_version: String, +} + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct GemPouch { + entity: Entity, + gems: u32, + last: Option, +} + +#[sourced(entity, aggregate_type = "GemPouch", events = "GemPouchEvent")] +impl GemPouch { + #[event("freeze.gem_recorded", version = 1)] + fn recorded(&mut self, principal: String, gem: Gem) { + if self.entity.id().is_empty() { + self.entity.set_id(principal); + } + self.gems += 1; + self.last = Some(gem); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct GemInput { + pub spot: u32, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct Counted { + pub count: u32, +} + +struct CollectGem; +impl PortableCommand for CollectGem +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(GEM_COLLECTED)) + .handle(handle_gem) + } +} + +async fn handle_gem( + ctx: &CausalCommandContext<'_, GemPouch>, + input: GemInput, +) -> std::result::Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + let stamp = + session_stamp(ctx.session()).ok_or_else(|| rejected("gem facts need a fact stamp"))?; + let repo = ctx.repo(); + let mut pouch = match repo.get(&principal).await? { + Some(pouch) => pouch, + None => repo.create(), + }; + pouch + .recorded( + principal, + Gem { + zone: stamp.zone_id, + spot: input.spot, + content_version: stamp.content_version, + }, + ) + .map_err(rejected)?; + let count = pouch.gems; + repo.commit(pouch)?.succeeded(Counted { count }) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ContentSwitch { + pub from: String, + pub to: String, + pub tick: u64, +} + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct ContentLog { + entity: Entity, + switches: Vec, +} + +#[sourced(entity, aggregate_type = "ContentLog", events = "ContentLogEvent")] +impl ContentLog { + #[event("freeze.content_recorded", version = 1)] + fn recorded(&mut self, zone: String, switch: ContentSwitch) { + if self.entity.id().is_empty() { + self.entity.set_id(zone); + } + self.switches.push(switch); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct ContentInput { + pub from: String, + pub to: String, +} + +struct ApplyContent; +impl PortableCommand for ApplyContent +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + CONTENT_APPLIED, + )) + .handle(handle_content) + } +} + +async fn handle_content( + ctx: &CausalCommandContext<'_, ContentLog>, + input: ContentInput, +) -> std::result::Result>, HandlerError> { + let stamp = + session_stamp(ctx.session()).ok_or_else(|| rejected("content facts need a fact stamp"))?; + let repo = ctx.repo(); + // The zone's own log: the shard is the stamped zone. + let mut log = match repo.get(&stamp.zone_id).await? { + Some(log) => log, + None => repo.create(), + }; + log.recorded( + stamp.zone_id.clone(), + ContentSwitch { + from: input.from, + to: input.to, + tick: stamp.tick, + }, + ) + .map_err(rejected)?; + let count = log.switches.len() as u32; + repo.commit(log)?.succeeded(Counted { count }) +} + +async fn read_pouch(cell: &AggregateCell) -> std::result::Result { + let pouch = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ + "gems": pouch.as_ref().map_or(0, |p| p.gems), + "last": pouch.and_then(|p| p.last), + })) +} + +async fn read_content_log(cell: &AggregateCell) -> std::result::Result { + let log = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ "switches": log.map(|l| l.switches).unwrap_or_default() })) +} + +#[durable_object] +pub struct GemPouchCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for GemPouchCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 50) + .expect("gem pouch cell identity") + .mount(CollectGem) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_pouch, fact_session).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +#[durable_object] +pub struct ContentLogCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ContentLogCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 50) + .expect("content log cell identity") + .mount(ApplyContent) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_content_log, fact_session).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + // ---------------------------------------------------------------- Hosting +/// Mob Pit cells run only facts: the principal and the stamp, from the +/// trusted headers of the authenticated internal path. +fn fact_session(header: &dyn Fn(&str) -> Option) -> Option { + fact_session_from_headers(header) +} + +/// Zone lease cells: the zone server's identity and role (the handlers +/// require `zone-host`). +fn zone_host_session(header: &dyn Fn(&str) -> Option) -> Option { + let mut session = Session::new(); + for key in [distributed::USER_ID_KEY, distributed::microsvc::ROLE_KEY] { + if let Some(value) = header(key) { + session.set(key, value); + } + } + Some(session) +} + +type Sessions = fn(&dyn Fn(&str) -> Option) -> Option; + async fn dispatch( env: &Env, cell: &AggregateCell, command: &str, req: &mut Request, + sessions: Sessions, ) -> Result where A: Aggregate + Send + Sync + 'static, @@ -306,9 +557,7 @@ where } _ => return error_response(CellDispatchError::Unauthorized), }; - // Only facts run here: the principal and the stamp, from the trusted - // headers of the authenticated internal path. - let Some(session) = fact_session_from_headers(header) else { + let Some(session) = sessions(&header) else { return error_response(CellDispatchError::Forbidden); }; let result = cell @@ -335,6 +584,7 @@ async fn route( cell: &AggregateCell, mut req: Request, read: F, + sessions: Sessions, ) -> Result where A: Aggregate + Send + Sync + 'static, @@ -357,7 +607,7 @@ where }, (Method::Post, Some(command)) => { let command = command.clone(); - dispatch(env, cell, &command, &mut req).await + dispatch(env, cell, &command, &mut req, sessions).await } _ => json_status(json!({ "error": "not found" }), 404), } @@ -423,7 +673,7 @@ impl DurableObject for CrawlerCell { } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_crawler).await + route(&self.env, &self.cell, req, read_crawler, fact_session).await } async fn alarm(&self) -> Result { @@ -448,7 +698,7 @@ impl DurableObject for CrawlerRosterCell { } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_roster).await + route(&self.env, &self.cell, req, read_roster, fact_session).await } async fn alarm(&self) -> Result { @@ -477,7 +727,7 @@ impl DurableObject for PresenceCell { } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_presence).await + route(&self.env, &self.cell, req, read_presence, fact_session).await } async fn alarm(&self) -> Result { @@ -499,6 +749,80 @@ where Response::ok("ok") } +#[durable_object] +pub struct ZoneLeaseCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ZoneLeaseCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + // Renewals are internal events: snapshots keep loads short. + let cell = mount_zone_lease( + AggregateCell::::from_state_with_snapshots(state, 50) + .expect("zone lease cell identity"), + ) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_lease, zone_host_session).await + } + + async fn alarm(&self) -> Result { + let wake = self + .cell + .alarm(&self.env) + .await + .map_err(|error| Error::RustError(error.to_string()))?; + for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + worker::console_error!("lease alarm failed: {} {}", failed.code, failed.message); + } + Response::ok("ok") + } +} + +async fn read_lease(cell: &AggregateCell) -> std::result::Result { + let lease = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ + "holder": lease.as_ref().and_then(|l| l.holder().map(str::to_string)), + "token": lease.as_ref().map(|l| l.token()), + })) +} + +#[durable_object] +pub struct ZoneStateCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ZoneStateCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + // Every save replaces the value: snapshot each commit. + let cell = mount_zone_state( + AggregateCell::::from_state_with_snapshots(state, 1) + .expect("zone state cell identity"), + ) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + if let Err(error) = authenticate(&req, &self.env) { + return error_response(error); + } + let bucket = celld_worker::bucket(&self.env, "ZONE_SNAPSHOTS")?; + celld_worker::fetch(&self.cell, &bucket, req).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + #[event(fetch)] async fn main(req: Request, env: Env, _ctx: Context) -> Result { console_error_panic_hook::set_once(); @@ -515,6 +839,10 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { Some(CRAWLER_KIND) => "CRAWLER", Some(ROSTER_KIND) => "CRAWLER_ROSTER", Some(PRESENCE_KIND) => "PRESENCE", + Some("gem-pouch") => "GEM_POUCH", + Some("content-log") => "CONTENT_LOG", + Some("zone-lease") => "ZONE_LEASE", + Some("zone-state") => "ZONE_STATE", _ => return json_status(json!({ "error": "unknown cell kind" }), 404), }; let Some(shard) = parts.get(1) else { diff --git a/examples/game/cells/wrangler.jsonc b/examples/game/cells/wrangler.jsonc index 680324798..2d06fcd8f 100644 --- a/examples/game/cells/wrangler.jsonc +++ b/examples/game/cells/wrangler.jsonc @@ -1,4 +1,4 @@ -// Mob Pit cells for `celld dev` (and `celld deploy`). The launcher +// The game demos' cells for `celld dev` (and `celld deploy`). The launcher // (`examples/game/run.sh`, the test harnesses) writes `.dev.vars` beside a // copy of this file with DISTRIBUTED_INTERNAL_SECRET and // CELLD_QUEUE_RELAY_URL (the zone server's `/internal/celld-queue/relay`). @@ -10,11 +10,21 @@ "bindings": [ { "name": "CRAWLER", "class_name": "CrawlerCell" }, { "name": "CRAWLER_ROSTER", "class_name": "CrawlerRosterCell" }, - { "name": "PRESENCE", "class_name": "PresenceCell" } + { "name": "PRESENCE", "class_name": "PresenceCell" }, + { "name": "ZONE_LEASE", "class_name": "ZoneLeaseCell" }, + { "name": "ZONE_STATE", "class_name": "ZoneStateCell" }, + { "name": "GEM_POUCH", "class_name": "GemPouchCell" }, + { "name": "CONTENT_LOG", "class_name": "ContentLogCell" } ] }, "migrations": [ - { "tag": "v1", "new_sqlite_classes": ["CrawlerCell", "CrawlerRosterCell", "PresenceCell"] } + { "tag": "v1", "new_sqlite_classes": ["CrawlerCell", "CrawlerRosterCell", "PresenceCell"] }, + { "tag": "v2", "new_sqlite_classes": ["ZoneLeaseCell", "ZoneStateCell"] }, + { "tag": "v3", "new_sqlite_classes": ["GemPouchCell", "ContentLogCell"] } + ], + // Hibernation snapshots above the inline limit (celld's object storage). + "r2_buckets": [ + { "binding": "ZONE_SNAPSHOTS", "bucket_name": "zone-snapshots" } ], "queues": { "producers": [ diff --git a/examples/game/client/tests/freeze-headless.test.mjs b/examples/game/client/tests/freeze-headless.test.mjs index a97ee65d2..457690c5b 100644 --- a/examples/game/client/tests/freeze-headless.test.mjs +++ b/examples/game/client/tests/freeze-headless.test.mjs @@ -17,6 +17,7 @@ import test from 'node:test'; import { createFreezeClient, FreezeAdmin, ZONES } from '../src/freeze/client.ts'; import { DevApi } from '../src/shared/dev-api.ts'; +import { startCelld } from './helpers/celld.mjs'; import { memoryStorage, sleep, startDemo, until } from './helpers/demo-server.mjs'; const ADMIN = 'freeze-test-admin'; @@ -45,7 +46,15 @@ async function walk(p, [x, y], done, timeoutMs = 15_000) { } test('world barrier across two hosts: rolling + safe room, hard reload, aborted rollout', { timeout: 600_000 }, async (t) => { - const server = await startDemo('freeze', { env: { DG_DEMO_ADMIN_SECRET: ADMIN, DG_FREEZE_STAGE_MS: '700' } }); + // Both Freeze hosts settle their facts in celld cells. + const celld = await startCelld(); + let server; + try { + server = await startDemo('freeze', { env: { ...celld.env, DG_DEMO_ADMIN_SECRET: ADMIN, DG_FREEZE_STAGE_MS: '700' } }); + } catch (error) { + await celld.stop(); + throw error; + } const base = server.httpUrl; const admin = new FreezeAdmin(base, ADMIN); const players = []; @@ -123,10 +132,18 @@ test('world barrier across two hosts: rolling + safe room, hard reload, aborted assert.deepEqual(bo.reloads.at(-1), { zone: ZONES.hub, version: 'v3' }); assert.equal(ana.pack().name, 'Bloom Update'); - // Facts on both hosts carry the version they were produced under. - const world = await admin.world(); + // Facts on both hosts carry the version they were produced under + // (listed once they settled in their celld cells). + const settledFacts = (world) => world.hosts.flatMap((h) => (h.facts ?? []).map((f) => ({ host: h.host, ...f }))); + let world = await admin.world(); + for (const deadline = Date.now() + 10_000; Date.now() < deadline; ) { + const applied = settledFacts(world).filter((f) => f.command === 'freeze.content_applied' && f.stamp.contentVersion === 'v3'); + if (applied.length >= 3) break; + await sleep(100); + world = await admin.world(); + } assert.equal(world.version, 'v3'); - const facts = world.hosts.flatMap((h) => (h.facts ?? []).map((f) => ({ host: h.host, ...f }))); + const facts = settledFacts(world); for (const [host, zone] of [ ['alpha', ZONES.hub], ['beta', ZONES.ember] @@ -161,5 +178,6 @@ test('world barrier across two hosts: rolling + safe room, hard reload, aborted } finally { for (const p of players) p.client.close(); await server.stop(); + await celld.stop(); } }); diff --git a/examples/game/client/tests/helpers/cluster.mjs b/examples/game/client/tests/helpers/cluster.mjs index 8b72d948d..5f1487e1b 100644 --- a/examples/game/client/tests/helpers/cluster.mjs +++ b/examples/game/client/tests/helpers/cluster.mjs @@ -1,28 +1,17 @@ /** * Start a multi-host demo (`game-demos --host-id … --peer …`) for a * Node test: one process per host on free ephemeral ports, a shared ticket - * secret, dev routes on. Uses GAME_DEMOS_BIN when set, else `cargo run`. + * secret, dev routes on, and one private `celld dev` (zone leases and + * hibernation snapshots in `examples/game/cells`). Uses GAME_DEMOS_BIN when + * set, else `cargo run`. */ import { spawn } from 'node:child_process'; -import { createServer, Socket } from 'node:net'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { Socket } from 'node:net'; import { fileURLToPath } from 'node:url'; -const SERVER_DIR = fileURLToPath(new URL('../../../server/', import.meta.url)); +import { freePort, startCelld } from './celld.mjs'; -function freePort() { - return new Promise((resolve, reject) => { - const server = createServer(); - server.unref(); - server.on('error', reject); - server.listen(0, '127.0.0.1', () => { - const { port } = server.address(); - server.close(() => resolve(port)); - }); - }); -} +const SERVER_DIR = fileURLToPath(new URL('../../../server/', import.meta.url)); /** * @param {string} demo @@ -31,7 +20,7 @@ function freePort() { export async function startCluster(demo, { hosts = ['A', 'B'], hibernateAfterMs = 4000, env = {} } = {}) { const ports = []; for (const _ of hosts) ports.push(await freePort()); - const stateDir = mkdtempSync(join(tmpdir(), `game-demos-${demo}-`)); + const celld = await startCelld(); const secret = `test-secret-${process.pid}-${Date.now()}`; const bin = process.env.GAME_DEMOS_BIN; const children = []; @@ -40,11 +29,11 @@ export async function startCluster(demo, { hosts = ['A', 'B'], hibernateAfterMs await Promise.all( hosts.map(async (id, i) => { const args = [demo, '--dev', '--port', String(ports[i]), '--host-id', id, '--secret', secret]; - args.push('--state-dir', stateDir, '--hibernate-after-ms', String(hibernateAfterMs)); + args.push('--hibernate-after-ms', String(hibernateAfterMs)); hosts.forEach((peer, j) => { if (j !== i) args.push('--peer', `${peer}=127.0.0.1:${ports[j]}`); }); - const options = { env: { ...process.env, ...env }, stdio: ['ignore', 'pipe', 'inherit'] }; + const options = { env: { ...process.env, ...celld.env, ...env }, stdio: ['ignore', 'pipe', 'inherit'] }; const child = bin ? spawn(bin, args, options) : spawn('cargo', ['run', '-q', '--', ...args], { ...options, cwd: SERVER_DIR }); @@ -62,7 +51,7 @@ export async function startCluster(demo, { hosts = ['A', 'B'], hibernateAfterMs ); } catch (error) { for (const child of children) child.kill('SIGKILL'); - rmSync(stateDir, { recursive: true, force: true }); + await celld.stop(); throw error; } return { @@ -81,7 +70,7 @@ export async function startCluster(demo, { hosts = ['A', 'B'], hibernateAfterMs clearTimeout(timer); }) ); - rmSync(stateDir, { recursive: true, force: true }); + await celld.stop(); // Node's WebSocket (undici) keeps a socket whose closing handshake // the server started (Redirect) paused until a ~60 s timeout, which // would hold the test process open: drop what still points at us. diff --git a/examples/game/run.sh b/examples/game/run.sh index a8710be99..ba84fbabd 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -32,9 +32,13 @@ # uses GAME_DEMOS_PORT, default 2967 there, and its read-model service # MOBPIT_SERVICE_PORT, default GAME_DEMOS_PORT + 2). # -# celld: Mob Pit's durable cells (examples/game/cells) run on celld, never -# natively. CELLD_BIN names the binary (default: `celld` on PATH; 0.6.2 or -# newer is required), CELLD_PORT its port (default GAME_DEMOS_PORT + 3), +# celld: the demos' durable cells (examples/game/cells: Mob Pit's Crawler, +# roster and presence cells, The Freeze's gem and content cells, every +# multi-host demo's zone leases and hibernation snapshots) run on celld, +# never natively; state persists in .data//celld (Mob Pit: +# MOBPIT_DATA_DIR/celld). CELLD_BIN names the binary (default: `celld` on +# PATH; 0.6.2 or newer is required), CELLD_PORT its port (default +# GAME_DEMOS_PORT + 3; cluster demos GAME_DEMOS_PORT + 4), # CELLD_ESBUILD its bundler (default: the client's esbuild). The worker is # built with worker-build (`cargo install worker-build --locked`). set -euo pipefail @@ -48,8 +52,9 @@ BIN="$SERVER/target/debug/game-demos" SERVICE_BIN="$SERVER/target/debug/mobpit-service" CELLS="$HERE/cells" DEMOS=(arena mobpit leaderboard gallery freeze grid ripple) -# Demos whose durable state lives in celld cells. -CELLD_DEMOS=(mobpit leaderboard) +# Single-process demos whose durable state lives in celld cells (the +# multi-host ones always run on celld too: zone leases and hibernation). +CELLD_DEMOS=(mobpit leaderboard freeze) CELLD_MIN_VERSION="0.6.2" export MOBPIT_DATA_DIR="${MOBPIT_DATA_DIR:-$HERE/.data/mobpit}" # Multi-host demos: `game-demos --host-id A|B --peer …`. @@ -159,7 +164,7 @@ start_celld() { DISTRIBUTED_INTERNAL_SECRET="$(cat "$project/internal-secret")" printf 'DISTRIBUTED_INTERNAL_SECRET=%s\nCELLD_QUEUE_RELAY_URL=http://127.0.0.1:%s/internal/celld-queue/relay\n' \ "$DISTRIBUTED_INTERNAL_SECRET" "$relay_port" >"$project/.dev.vars" - log "starting celld dev on :$port ($("$CELLD_BIN" --version 2>/dev/null | tail -1); state in $project/.celld/dev)" + log "starting celld dev on :$port ($("$CELLD_BIN" --version 2>/dev/null | grep '^celld ' | head -1); state in $project/.celld/dev)" RUST_LOG=warn "$CELLD_BIN" dev "$project" --host 127.0.0.1 --port "$port" --no-watch --logs >"$project/celld.log" 2>&1 & CELLD_PID=$! CELLD_URL="http://127.0.0.1:$port" @@ -246,7 +251,9 @@ run_demo() { CELLD_PID="" if [[ " ${CELLD_DEMOS[*]} " == *" $demo "* ]]; then - start_celld "$MOBPIT_DATA_DIR/celld" "${CELLD_PORT:-$((port + 3))}" "$port" + local project="$HERE/.data/$demo/celld" + [[ "$demo" == mobpit || "$demo" == leaderboard ]] && project="$MOBPIT_DATA_DIR/celld" + start_celld "$project" "${CELLD_PORT:-$((port + 3))}" "$port" fi local log_file log_file="$(mktemp -t game-demos.XXXXXX)" @@ -293,24 +300,27 @@ run_cluster() { ensure_server ensure_wasm + # Zone leases and hibernation snapshots live in celld (shared by both + # hosts); its state persists in .data//celld. + start_celld "$HERE/.data/$demo/celld" "${CELLD_PORT:-$((port_a + 4))}" "$port_a" local dir secret dir="$(mktemp -d "${TMPDIR:-/tmp}/game-demos-$demo.XXXXXX")" # Both hosts must share the ticket secret (and derive the admin secret). secret="${DISTRIBUTED_GAME_SECRET:-$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')}" DISTRIBUTED_GAME_SECRET="$secret" "$BIN" "$demo" --dev --port "$port_a" --host-id A \ - --peer "B=127.0.0.1:$port_b" --state-dir "$dir" >"$dir/A.log" 2>&1 & + --peer "B=127.0.0.1:$port_b" >"$dir/A.log" 2>&1 & local pid_a=$! DISTRIBUTED_GAME_SECRET="$secret" "$BIN" "$demo" --dev --port "$port_b" --host-id B \ - --peer "A=127.0.0.1:$port_a" --state-dir "$dir" >"$dir/B.log" 2>&1 & + --peer "A=127.0.0.1:$port_a" >"$dir/B.log" 2>&1 & local pid_b=$! - trap 'kill "$pid_a" "$pid_b" 2>/dev/null || true; wait "$pid_a" "$pid_b" 2>/dev/null || true; rm -rf "$dir"' EXIT INT TERM + trap 'kill "$pid_a" "$pid_b" 2>/dev/null || true; wait "$pid_a" "$pid_b" 2>/dev/null || true; kill_tree "$CELLD_PID"; rm -rf "$dir"' EXIT INT TERM wait_listening "$dir/A.log" "$pid_a" "game-demos $demo host A" wait_listening "$dir/B.log" "$pid_b" "game-demos $demo host B" sed -n '2p' "$dir/A.log" sed -n '2p' "$dir/B.log" echo log "open http://localhost:$client_port/$demo/ (hosts A :$port_a, B :$port_b, dev routes ON — local only)" - log "snapshots in $dir (removed on exit)" + log "zone leases and hibernation snapshots in celld ($CELLD_URL); host logs in $dir (removed on exit)" echo cd "$CLIENT" GAME_DEMOS_SERVER="http://127.0.0.1:$port_a" GAME_DEMOS_CLIENT_PORT="$client_port" npx vite diff --git a/examples/game/server/Cargo.lock b/examples/game/server/Cargo.lock index 376e311c5..7689c67b2 100644 --- a/examples/game/server/Cargo.lock +++ b/examples/game/server/Cargo.lock @@ -845,7 +845,6 @@ dependencies = [ "futures-util", "hmac 0.12.1", "reqwest", - "rusqlite", "serde", "serde_json", "sha2 0.10.9", @@ -956,18 +955,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fast_chemail" version = "0.9.6" @@ -2259,31 +2246,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rsqlite-vfs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" -dependencies = [ - "hashbrown 0.16.1", - "thiserror", -] - -[[package]] -name = "rusqlite" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", - "sqlite-wasm-rs", -] - [[package]] name = "rustc-hash" version = "2.1.3" @@ -2646,18 +2608,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "sqlite-wasm-rs" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" -dependencies = [ - "cc", - "js-sys", - "rsqlite-vfs", - "wasm-bindgen", -] - [[package]] name = "sqlx" version = "0.9.0" diff --git a/examples/game/server/Cargo.toml b/examples/game/server/Cargo.toml index e62fdc72d..b74143b12 100644 --- a/examples/game/server/Cargo.toml +++ b/examples/game/server/Cargo.toml @@ -29,7 +29,7 @@ path = "src/main.rs" game_sim = { path = "sim" } # The Crawler aggregate + RecordKill (shared with the read-model service). mobpit_crawler = { path = "crawler" } -distributed_game = { path = "../../../distributed_game", features = ["net", "cell", "celld", "sqlite"] } +distributed_game = { path = "../../../distributed_game", features = ["net", "cell", "celld"] } # Mob Pit: sorted-set leaderboards (Redis optional at runtime) + the SQLite # bus for the read-model service. Its durable cells run on celld # (`examples/game/cells`), reached with `distributed_game`'s `celld` feature. diff --git a/examples/game/server/src/cluster.rs b/examples/game/server/src/cluster.rs index ad3e753d0..3bf3b7223 100644 --- a/examples/game/server/src/cluster.rs +++ b/examples/game/server/src/cluster.rs @@ -10,13 +10,13 @@ //! routes (`/admin/transfer`, `/admin/child-event`); //! - the admin secret, derived from the shared ticket secret (all hosts of //! a world must be started with the same `--secret`); -//! - fenced zone claims (`LocalOwnership`, whose tokens are seeded from the -//! process start time so they outrank snapshots of an earlier run; or, -//! with `--celld-url`, `CellZoneOwnership`: one celld `ZoneLease` cell per -//! zone, whose alarm declares `zone.lease_expired` when this host stops -//! renewing) and SQLite hibernation -//! snapshots ([`SqliteZoneStateStore`], one file per host under -//! `--state-dir`, in-memory without it); +//! - zone ownership and hibernation on celld (`--celld-url` / `CELLD_URL` +//! with `DISTRIBUTED_INTERNAL_SECRET`, the `examples/game/cells` worker): +//! `CellZoneOwnership`, one `ZoneLease` cell per zone whose alarm declares +//! `zone.lease_expired` when this host stops renewing, and +//! `CellZoneStateStore`, one `ZoneState` cell per zone holding its +//! hibernation snapshot, fenced by the lease token. There is no local +//! fallback: every host of a world shares the same celld; //! - guest display names shared with every peer: `/dev/ticket` on any host //! forwards the new `principal → name` to the others before it answers, //! so a zone on another host greets the player by name. @@ -25,7 +25,6 @@ use std::future::Future; use std::net::SocketAddr; -use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -36,17 +35,17 @@ use axum::middleware::{self, Next}; use axum::response::Response; use axum::routing::post; use axum::{Json, Router}; -use distributed::cell_host::InternalHttpSecret; -use distributed_game::celld::CellZoneOwnership; +use distributed_game::celld::{CellZoneOwnership, CellZoneStateStore}; use distributed_game::net::peer::{post_json, ADMIN_SECRET_HEADER}; use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; use distributed_game::world::{ - HostAddr, LocalOwnership, SqliteZoneStateStore, ZoneDirectory, ZoneOwnership, ZoneStateStore, + HostAddr, MemoryZoneStateStore, ZoneDirectory, ZoneOwnership, ZoneStateStore, }; use serde_json::{json, Value}; use crate::common::{ - self, dev, random_hex, server_config, Demo, DemoArgs, DemoOptions, Guests, RunningDemo, + self, dev, random_hex, server_config, CelldOptions, Demo, DemoArgs, DemoOptions, Guests, + RunningDemo, }; /// CLI options of a multi-host demo. @@ -62,51 +61,31 @@ pub struct ClusterArgs { /// host must list every other host. #[arg(long = "peer", value_name = "ID=HOST:PORT")] pub peers: Vec, - /// Directory for SQLite hibernation snapshots (`-.sqlite3`). - /// Default: in-memory (lost when the process exits). - #[arg(long)] - pub state_dir: Option, /// Hibernate a zone after it has been Dormant this long (ms); it turns /// Dormant after half of this without clients. #[arg(long, default_value_t = 4000)] pub hibernate_after_ms: u64, - /// celld URL serving `ZoneLease` cells (kind `zone-lease`): zone - /// ownership lives there instead of in this process (needs - /// `DISTRIBUTED_INTERNAL_SECRET`). + /// celld serving the `zone-lease` and `zone-state` cells + /// (`examples/game/cells`): zone ownership and hibernation snapshots. #[arg(long, env = "CELLD_URL")] - pub celld_url: Option, - /// Internal secret for celld cells (with `--celld-url`). + pub celld_url: String, + /// Internal secret for the celld cells. #[arg(long, env = "DISTRIBUTED_INTERNAL_SECRET", hide_env_values = true)] - pub internal_secret: Option, + pub internal_secret: String, } /// In-process cluster options (tests and [`run_cluster`]). -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ClusterOptions { /// This host's id. pub host_id: String, /// Every other host. pub peers: Vec, - /// Snapshot directory (None = in-memory SQLite). - pub state_dir: Option, /// Dormant time before hibernation. pub hibernate_after: Duration, - /// Zone ownership in celld `ZoneLease` cells: `(url, internal secret)`. - /// `None` = process-local `LocalOwnership`. - pub celld: Option<(String, String)>, -} - -impl ClusterOptions { - /// A lone host `id` (no peers, in-memory snapshots). - pub fn single(id: &str, hibernate_after: Duration) -> Self { - Self { - host_id: id.into(), - peers: Vec::new(), - state_dir: None, - hibernate_after, - celld: None, - } - } + /// celld holding zone ownership (`ZoneLease`) and hibernation + /// snapshots (`ZoneState`). + pub celld: CelldOptions, } /// Parse `ID=HOST:PORT`. @@ -128,8 +107,8 @@ pub struct World { pub hosts: Vec, /// Zone placement. pub directory: Arc, - /// Hibernation snapshots of the zones this host ran. - pub state: Arc, + /// Hibernation snapshots (celld `ZoneState` cells). + pub state: Arc, /// Shared admin secret (peer calls). pub admin_secret: String, /// Display names (shared with peers by `/dev/ticket`). @@ -139,18 +118,23 @@ pub struct World { } impl World { - /// A world of one host that is not serving (`Demo::server`). - pub fn standalone(options: &ClusterOptions, guests: Guests) -> Self { - let host = HostAddr::new(options.host_id.clone(), "127.0.0.1:0"); + /// A world of one host that is not serving (`Demo::server`; snapshots + /// in memory). + pub fn standalone( + host_id: &str, + hibernate_after: Duration, + guests: Guests, + ) -> Self { + let host = HostAddr::new(host_id, "127.0.0.1:0"); let hosts = vec![host.clone()]; Self { directory: D::directory(&hosts), host, hosts, - state: Arc::new(SqliteZoneStateStore::open_in_memory().expect("in-memory SQLite")), + state: Arc::new(MemoryZoneStateStore::new()), admin_secret: String::new(), guests, - hibernate_after: options.hibernate_after, + hibernate_after, } } @@ -244,13 +228,13 @@ pub fn spawn_cluster( hosts.push(host.clone()); hosts.sort(); let directory = D::directory(&hosts); - let state = match &cluster.state_dir { - Some(dir) => { - std::fs::create_dir_all(dir)?; - SqliteZoneStateStore::open(dir.join(format!("{}-{}.sqlite3", D::NAME, host.id))) - } - None => SqliteZoneStateStore::open_in_memory(), - } + let service_id = format!("game-demos-{}", D::NAME); + let state = CellZoneStateStore::new( + cluster.celld.url.clone(), + cluster.celld.secret.clone(), + service_id.clone(), + host.id.clone(), + ) .map_err(std::io::Error::other)?; let guests = Guests::default(); let world = Arc::new(World { @@ -267,23 +251,20 @@ pub fn spawn_cluster( config.admin_secret = Some(world.admin_secret.clone()); config.hibernate_after = Some(cluster.hibernate_after); config.host.dormant_after = cluster.hibernate_after / 2; - let ownership: Arc = match &cluster.celld { - Some((url, secret)) => Arc::new( - CellZoneOwnership::new( - url.clone(), - InternalHttpSecret::new(secret.clone()).map_err(std::io::Error::other)?, - format!("game-demos-{}", D::NAME), - world.host.id.clone(), - ) - .map_err(std::io::Error::other)?, - ), - None => Arc::new(LocalOwnership::new()), - }; + let ownership: Arc = Arc::new( + CellZoneOwnership::new( + cluster.celld.url.clone(), + cluster.celld.secret.clone(), + service_id, + world.host.id.clone(), + ) + .map_err(std::io::Error::other)?, + ); let server = D::cluster_server(config, &world) .with_host_id(world.host.id.clone()) .with_directory(directory) .with_ownership(ownership) - .with_state_store(world.state.clone() as Arc); + .with_state_store(world.state.clone()); let extra = if options.dev { let tickets = dev::routes::(server.clone(), options.key, guests.clone()).layer( @@ -330,17 +311,9 @@ pub async fn run_cluster(args: ClusterArgs) -> std::io::Result<( let cluster = ClusterOptions { host_id: args.host_id, peers, - state_dir: args.state_dir, hibernate_after: Duration::from_millis(args.hibernate_after_ms.max(100)), - celld: match (args.celld_url, args.internal_secret) { - (Some(url), Some(secret)) => Some((url, secret)), - (Some(_), None) => { - return Err(std::io::Error::other( - "--celld-url needs DISTRIBUTED_INTERNAL_SECRET (or --internal-secret)", - )) - } - _ => None, - }, + celld: CelldOptions::new(args.celld_url, args.internal_secret) + .map_err(std::io::Error::other)?, }; let running = spawn_cluster::(listener, options, cluster, common::shutdown_signal())?; // Parsed by run.sh and the client tests: keep the format. diff --git a/examples/game/server/src/common/celld.rs b/examples/game/server/src/common/celld.rs new file mode 100644 index 000000000..fa15a8b89 --- /dev/null +++ b/examples/game/server/src/common/celld.rs @@ -0,0 +1,45 @@ +//! Where a demo's celld is: the base URL and the internal secret shared +//! with the cells worker (`examples/game/cells`). + +use distributed::cell_host::InternalHttpSecret; + +/// Env var of the celld base URL. +pub const CELLD_URL_ENV: &str = "CELLD_URL"; + +/// Where the cells are: the celld base URL and the internal secret shared +/// with the cells worker (`DISTRIBUTED_INTERNAL_SECRET`). +#[derive(Clone)] +pub struct CelldOptions { + /// `http(s)://host:port` of celld serving `examples/game/cells`. + pub url: String, + /// Internal-path secret (also authenticates the Queue relay route). + pub secret: InternalHttpSecret, +} + +impl CelldOptions { + /// Options for `url` and `secret`. + pub fn new(url: impl Into, secret: impl Into) -> Result { + let url = url.into().trim_end_matches('/').to_string(); + if !(url.starts_with("http://") || url.starts_with("https://")) { + return Err(format!("celld URL must be http(s): {url}")); + } + Ok(Self { + url, + secret: InternalHttpSecret::new(secret.into())?, + }) + } + + /// `CELLD_URL` and `DISTRIBUTED_INTERNAL_SECRET` from the environment. + pub fn from_env() -> Result { + let var = |name: &str| { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| format!("{name} is required (the demos' cells run on celld)")) + }; + Self::new( + var(CELLD_URL_ENV)?, + var(distributed::cell_host::CELL_INTERNAL_SECRET_ENV)?, + ) + } +} diff --git a/examples/game/server/src/common/mod.rs b/examples/game/server/src/common/mod.rs index d02b94ee6..bdc57e3c8 100644 --- a/examples/game/server/src/common/mod.rs +++ b/examples/game/server/src/common/mod.rs @@ -8,6 +8,7 @@ //! line in `main.rs`; everything else (ticket endpoint, NetSim control, //! `/metrics`, graceful shutdown) comes from here. +pub mod celld; pub mod dev; mod guests; @@ -18,6 +19,7 @@ use axum::Router; use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; use distributed_game::Zone; +pub use celld::{CelldOptions, CELLD_URL_ENV}; pub use guests::{random_hex, Guests}; /// One playable demo hosted by `game-demos `. diff --git a/examples/game/server/src/devcelld.rs b/examples/game/server/src/devcelld.rs index cf264f5ad..75682e971 100644 --- a/examples/game/server/src/devcelld.rs +++ b/examples/game/server/src/devcelld.rs @@ -15,7 +15,7 @@ use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::time::{Duration, Instant}; -use crate::mobpit::store::CelldOptions; +use crate::common::CelldOptions; /// Env var naming the celld binary. pub const CELLD_BIN_ENV: &str = "CELLD_BIN"; diff --git a/examples/game/server/src/freeze/world.rs b/examples/game/server/src/freeze/world.rs index 7b61dae06..4335168ff 100644 --- a/examples/game/server/src/freeze/world.rs +++ b/examples/game/server/src/freeze/world.rs @@ -8,6 +8,13 @@ //! for a zone on the other host is redirected and handoffs reserve their //! transfer with the issuing host over `/admin/transfer`. //! +//! Facts go to celld (`--celld-url` / `CELLD_URL`, `DISTRIBUTED_INTERNAL_SECRET`, +//! the `examples/game/cells` worker; the secondary inherits both through the +//! environment) with [`FreezeFactSink`]: `freeze.gem_collected` to the +//! player's `gem-pouch/{principal}` cell, `freeze.content_applied` to the +//! zone's `content-log/{zone}` cell. The host keeps the last settled facts +//! in memory only for `GET /dev/freeze/world`. +//! //! Routes (dev only, mounted with `--dev`; the mutating ones require the //! `x-game-admin-secret` header): //! @@ -29,18 +36,82 @@ use axum::extract::{Path, State}; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{get, post}; use axum::{Json, Router}; +use distributed_game::celld::{CelldFactRoute, CelldFactSink}; use distributed_game::net::peer::{post_json, ADMIN_SECRET_HEADER}; use distributed_game::net::{ GameServer, RemoteHost, RolloutError, ServerConfig, TicketKey, WorldCoordinator, ZoneFactory, }; use distributed_game::world::{HostAddr, StaticDirectory}; -use distributed_game::{BarrierMode, FreezeState, MemoryFactSink, RetryPolicy}; +use distributed_game::{ + BarrierMode, BoxFuture, Fact, FactError, FactReceipt, FactSink, FreezeState, RetryPolicy, +}; use serde::Deserialize; use serde_json::{json, Value}; use super::content::{self, ZoneKind, EMBER, HUB, MEADOW}; -use super::{freeze_config, FreezeConfig, FreezeDemo, FreezeZone, Rituals}; -use crate::common::{self, dev, random_hex, server_config, DemoArgs, Guests}; +use super::{ + freeze_config, ContentApplied, FreezeConfig, FreezeDemo, FreezeZone, GemCollected, Rituals, +}; +use crate::common::{self, dev, random_hex, server_config, CelldOptions, DemoArgs, Guests}; + +/// Command-ledger service id of Freeze facts. +pub const SERVICE_ID: &str = "freeze"; +/// Principal of zone facts (`freeze.content_applied`). +pub const SYSTEM_PRINCIPAL: &str = "freeze-zone"; + +/// The Freeze's fact sink: facts to their celld cells, the settled ones +/// mirrored (last 64) for the dev status route. +#[derive(Clone)] +pub struct FreezeFactSink { + celld: CelldFactSink, + settled: Arc>>, +} + +impl FreezeFactSink { + /// Sink into the cells at `celld`. + pub fn new(celld: &CelldOptions) -> Result { + let sink = CelldFactSink::new( + celld.url.clone(), + celld.secret.clone(), + SERVICE_ID, + SYSTEM_PRINCIPAL, + )? + .route(CelldFactRoute::by_principal::("gem-pouch")) + .route(CelldFactRoute::new( + ::COMMAND, + "content-log", + |fact, _| Some(fact.stamp().zone_id.clone()), + )); + Ok(Self { + celld: sink, + settled: Arc::default(), + }) + } + + /// Settled facts, oldest first (at most the last 64). + pub fn committed(&self) -> Vec { + self.settled + .lock() + .expect("settled") + .iter() + .cloned() + .collect() + } +} + +impl FactSink for FreezeFactSink { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + let receipt = self.celld.commit(fact).await?; + let mut settled = self.settled.lock().expect("settled"); + if settled.len() == 64 { + settled.pop_front(); + } + settled.push_back(fact.clone()); + Ok(receipt) + }) + } +} /// Default time the "content CDN" takes to stage an update while the world /// is frozen (makes the pause visible). @@ -66,6 +137,12 @@ pub struct FreezeArgs { /// Simulated content staging time during the freeze (ms). #[arg(long, env = "DG_FREEZE_STAGE_MS", default_value_t = STAGE_MS)] pub stage_ms: u64, + /// celld serving the `gem-pouch` and `content-log` cells. + #[arg(long, env = "CELLD_URL")] + pub celld_url: String, + /// Internal secret for the celld cells. + #[arg(long, env = "DISTRIBUTED_INTERNAL_SECRET", hide_env_values = true)] + pub internal_secret: String, } /// Everything one host needs. @@ -86,6 +163,8 @@ pub struct HostSpec { pub stage: Duration, /// Barrier acknowledgement timeout. pub ack_timeout: Duration, + /// celld holding the facts' cells. + pub celld: CelldOptions, } /// A running Freeze host. @@ -95,7 +174,7 @@ pub struct WorldHost { /// The game server. pub server: GameServer, /// Facts committed by this host's zones. - pub sink: MemoryFactSink, + pub sink: FreezeFactSink, /// Stuck-zone toggles. pub rituals: Rituals, /// Resolves when the server stopped. @@ -109,7 +188,9 @@ pub fn zone_server( rituals: Rituals, host: &str, ) -> GameServer { - zone_server_with_sink(config, guests, rituals, host, MemoryFactSink::new()) + let celld = CelldOptions::from_env().expect("The Freeze needs celld"); + let sink = FreezeFactSink::new(&celld).expect("freeze fact sink"); + zone_server_with_sink(config, guests, rituals, host, sink) } fn zone_server_with_sink( @@ -117,7 +198,7 @@ fn zone_server_with_sink( guests: Guests, rituals: Rituals, host: &str, - sink: MemoryFactSink, + sink: FreezeFactSink, ) -> GameServer { let host = host.to_string(); GameServer::new( @@ -132,7 +213,12 @@ fn zone_server_with_sink( }) }), ) - .with_fact_sink(sink, 256, RetryPolicy::default()) + // celld down: facts wait in the zone's queue until they settle. + .with_fact_sink( + sink, + 256, + RetryPolicy::until_settled(Duration::from_millis(50), Duration::from_secs(2)), + ) } /// The default assignment: hub + meadow on `primary`, ember on `secondary`. @@ -164,7 +250,7 @@ pub fn start_host( config.admin_secret = Some(spec.admin_secret.clone()); let guests = Guests::default(); let rituals = Rituals::default(); - let sink = MemoryFactSink::new(); + let sink = FreezeFactSink::new(&spec.celld).map_err(std::io::Error::other)?; let server = zone_server_with_sink( config, guests.clone(), @@ -222,7 +308,7 @@ pub fn start_host( #[derive(Clone)] struct FreezeRoutes { server: GameServer, - sink: MemoryFactSink, + sink: FreezeFactSink, rituals: Rituals, admin_secret: String, me: HostAddr, @@ -568,6 +654,8 @@ fn spawn_secondary( primary.admin_url.trim_start_matches("http://") )) .args(["--stage-ms", &args.stage_ms.to_string()]) + .args(["--celld-url", &args.celld_url]) + .env("DISTRIBUTED_INTERNAL_SECRET", &args.internal_secret) .env("DISTRIBUTED_GAME_SECRET", secret) .env("DG_DEMO_ADMIN_SECRET", admin) .env_remove("GAME_DEMOS_PORT") @@ -648,6 +736,8 @@ pub async fn run(args: FreezeArgs) -> std::io::Result<()> { dev: args.demo.dev, stage: Duration::from_millis(args.stage_ms), ack_timeout: ACK_TIMEOUT, + celld: CelldOptions::new(args.celld_url.clone(), args.internal_secret.clone()) + .map_err(std::io::Error::other)?, }; let host = start_host(listener, spec, shutdown_signal(secondary))?; // Parsed by run.sh and the client tests: keep the format. diff --git a/examples/game/server/src/grid/dev.rs b/examples/game/server/src/grid/dev.rs index a02c9d450..d3d066100 100644 --- a/examples/game/server/src/grid/dev.rs +++ b/examples/game/server/src/grid/dev.rs @@ -16,7 +16,6 @@ use axum::http::{HeaderMap, StatusCode, Uri}; use axum::routing::{get, post}; use axum::{Json, Router}; use distributed_game::net::GameServer; -use distributed_game::world::ZoneStateStore; use distributed_game::Tier; use serde_json::{json, Value}; @@ -58,9 +57,12 @@ fn tier_name(tier: Tier) -> &'static str { } } -/// Facts about this host's cells among `zones`, keyed by zone id. +/// Facts about this host's cells among `zones`, keyed by zone id. A cell +/// that is not active is "hibernated" when celld holds a snapshot of it +/// (the lookups run concurrently, bounded to 400 ms; "unknown" after that). async fn local_cells(state: &MapState, zones: &[String]) -> serde_json::Map { let mut out = serde_json::Map::new(); + let mut lookups = tokio::task::JoinSet::new(); for zone in zones { if state.world.owner(zone) != state.world.host.id || world::parse_zone(zone).is_none() { continue; @@ -76,16 +78,47 @@ async fn local_cells(state: &MapState, zones: &[String]) -> serde_json::Map json!({ - "state": "active", "tier": tier, "players": players, "observers": observers - }), + match live { + Some((tier, players, observers)) => { + out.insert( + zone.clone(), + json!({ + "state": "active", "tier": tier, "players": players, "observers": observers + }), + ); + } None => { - let stored = state.world.state.load(zone).ok().flatten(); - json!({ "state": if stored.is_some() { "hibernated" } else { "cold" } }) + let store = state.world.state.clone(); + let zone = zone.clone(); + lookups.spawn(async move { + let stored = store.load(&zone).await.ok().flatten(); + (zone, stored.is_some()) + }); } - }; - out.insert(zone.clone(), value); + } + } + let mut pending: std::collections::BTreeSet = zones + .iter() + .filter(|zone| !out.contains_key(zone.as_str())) + .filter(|zone| state.world.owner(zone) == state.world.host.id) + .filter(|zone| world::parse_zone(zone).is_some()) + .cloned() + .collect(); + // Bounded, so a peer asking for our cells gets an answer in time (a + // cold celld cell can take a while to activate); late ones are unknown. + let _ = tokio::time::timeout(Duration::from_millis(400), async { + while let Some(Ok((zone, stored))) = lookups.join_next().await { + pending.remove(&zone); + out.insert( + zone, + json!({ "state": if stored { "hibernated" } else { "cold" } }), + ); + } + }) + .await; + lookups.abort_all(); + for zone in pending { + out.insert(zone, json!({ "state": "unknown" })); } out } diff --git a/examples/game/server/src/grid/mod.rs b/examples/game/server/src/grid/mod.rs index 0905a0be8..362889dc6 100644 --- a/examples/game/server/src/grid/mod.rs +++ b/examples/game/server/src/grid/mod.rs @@ -55,7 +55,7 @@ use distributed_game::{ }; use serde::{Deserialize, Serialize}; -use crate::cluster::{ClusterDemo, ClusterOptions, World}; +use crate::cluster::{ClusterDemo, World}; use crate::common::{Demo, Guests}; use world::{MoveInput, Prop, CELL, POS}; @@ -626,8 +626,11 @@ impl Demo for GridDemo { /// Single host (used by `common::spawn`); `game-demos grid` runs /// [`crate::cluster::run_cluster`] instead. fn server(config: ServerConfig, guests: Guests) -> GameServer { - let options = ClusterOptions::single("local", std::time::Duration::from_secs(4)); - let world = crate::cluster::World::standalone::(&options, guests); + let world = crate::cluster::World::standalone::( + "local", + std::time::Duration::from_secs(4), + guests, + ); Self::cluster_server(config, &world) } } diff --git a/examples/game/server/src/mobpit/store.rs b/examples/game/server/src/mobpit/store.rs index 9799a4354..f13a056be 100644 --- a/examples/game/server/src/mobpit/store.rs +++ b/examples/game/server/src/mobpit/store.rs @@ -49,46 +49,8 @@ use super::presence::Presence; pub const SERVICE_ID: &str = "mobpit"; /// Principal for facts without a player (none are emitted today). pub const SYSTEM_PRINCIPAL: &str = "mobpit-zone"; -/// Env var of the celld base URL. -pub const CELLD_URL_ENV: &str = "CELLD_URL"; -/// Where the cells are: the celld base URL and the internal secret shared -/// with the cells worker (`DISTRIBUTED_INTERNAL_SECRET`). -#[derive(Clone)] -pub struct CelldOptions { - /// `http(s)://host:port` of celld serving `examples/game/cells`. - pub url: String, - /// Internal-path secret (also authenticates the Queue relay route). - pub secret: InternalHttpSecret, -} - -impl CelldOptions { - /// Options for `url` and `secret`. - pub fn new(url: impl Into, secret: impl Into) -> Result { - let url = url.into().trim_end_matches('/').to_string(); - if !(url.starts_with("http://") || url.starts_with("https://")) { - return Err(format!("celld URL must be http(s): {url}")); - } - Ok(Self { - url, - secret: InternalHttpSecret::new(secret.into())?, - }) - } - - /// `CELLD_URL` and `DISTRIBUTED_INTERNAL_SECRET` from the environment. - pub fn from_env() -> Result { - let var = |name: &str| { - std::env::var(name) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| format!("{name} is required (Mob Pit's cells run on celld)")) - }; - Self::new( - var(CELLD_URL_ENV)?, - var(distributed::cell_host::CELL_INTERNAL_SECRET_ENV)?, - ) - } -} +pub use crate::common::CelldOptions; /// A Crawler cell's state as the cells worker reports it. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] diff --git a/examples/game/server/src/ripple/mod.rs b/examples/game/server/src/ripple/mod.rs index 17f1c9cef..71c5ee80f 100644 --- a/examples/game/server/src/ripple/mod.rs +++ b/examples/game/server/src/ripple/mod.rs @@ -54,7 +54,7 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use crate::arena::movement::{self, MoveState}; -use crate::cluster::{ClusterDemo, ClusterOptions, World}; +use crate::cluster::{ClusterDemo, World}; use crate::common::{Demo, Guests}; /// Root zone. @@ -826,8 +826,7 @@ impl Demo for RippleDemo { } fn server(config: ServerConfig, guests: Guests) -> GameServer { - let options = ClusterOptions::single("local", Duration::from_secs(20)); - let world = World::standalone::(&options, guests); + let world = World::standalone::("local", Duration::from_secs(20), guests); Self::cluster_server(config, &world) } } diff --git a/examples/game/server/tests/freeze_smoke.rs b/examples/game/server/tests/freeze_smoke.rs index daf23e915..fc341304e 100644 --- a/examples/game/server/tests/freeze_smoke.rs +++ b/examples/game/server/tests/freeze_smoke.rs @@ -19,7 +19,15 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; const SECRET: &str = "freeze-smoke-secret"; const ADMIN: &str = "freeze-smoke-admin"; -async fn world() -> (WorldHost, WorldHost, Vec>) { +async fn world() -> ( + WorldHost, + WorldHost, + Vec>, + game_demos::devcelld::DevCelld, +) { + // Both hosts' facts go to one private `celld dev`. + let celld = game_demos::devcelld::DevCelld::start(None) + .unwrap_or_else(|error| panic!("celld dev: {error}")); let la = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let lb = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let a = HostAddr::new("alpha", la.local_addr().unwrap().to_string()); @@ -38,6 +46,7 @@ async fn world() -> (WorldHost, WorldHost, Vec> dev: true, stage: Duration::from_millis(600), ack_timeout: Duration::from_millis(800), + celld: celld.options(), }; hosts.push( start_host(listener, spec, async move { @@ -48,7 +57,7 @@ async fn world() -> (WorldHost, WorldHost, Vec> } let b = hosts.pop().unwrap(); let a = hosts.pop().unwrap(); - (a, b, stops) + (a, b, stops, celld) } /// Minimal HTTP/1.1 client: `(status, JSON body or Null)`. @@ -129,7 +138,7 @@ async fn dropped(host: &WorldHost, zone: &str) -> u64 { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn rollout_spans_both_hosts_drops_frozen_inputs_and_stamps_the_new_version() { - let (a, b, stops) = world().await; + let (a, b, stops, celld) = world().await; let ws_a = format!("ws://{}", a.addr); let ws_b = format!("ws://{}", b.addr); @@ -206,6 +215,17 @@ async fn rollout_spans_both_hosts_drops_frozen_inputs_and_stamps_the_new_version .any(|f| f.command() == "freeze.content_applied" && f.stamp().zone_id == zone && f.stamp().content_version == "v2")); + // Durable in the zone's celld content-log cell. + let log: Value = reqwest::Client::new() + .get(format!("{}/content-log/{zone}", celld.url())) + .header("x-distributed-internal-secret", celld.secret()) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(log["switches"][0]["to"], "v2", "{zone}: {log}"); } let (_, pack) = content(a.addr, "current").await; assert_eq!(pack.unwrap().hazard, Hazard::Ice); @@ -219,7 +239,7 @@ async fn rollout_spans_both_hosts_drops_frozen_inputs_and_stamps_the_new_version #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_stuck_zone_aborts_and_everyone_resumes_on_the_old_version() { - let (a, b, stops) = world().await; + let (a, b, stops, _celld) = world().await; // The stuck toggle is forwarded to the host owning the zone. let (status, reply) = http( a.addr, diff --git a/examples/game/server/tests/grid_smoke.rs b/examples/game/server/tests/grid_smoke.rs index 62928d557..05168ec8a 100644 --- a/examples/game/server/tests/grid_smoke.rs +++ b/examples/game/server/tests/grid_smoke.rs @@ -3,14 +3,14 @@ //! - a walker crosses several cell borders, at least one of them between //! hosts, following every Handoff (and Redirect) without losing its //! session, and ends in the right zone; -//! - a cell hibernates into SQLite and restores critters and markers; +//! - a cell hibernates into its celld `ZoneState` cell and restores +//! critters and markers; //! - dev routes: world map across hosts, names shared with peers. mod support; use std::time::Duration; -use distributed_game::world::ZoneStateStore; use distributed_game::{FieldValue, Role, ServerMessage}; use distributed_game::{GameInput, Replicated}; use game_demos::grid::world::{self, MoveInput, BUTTON_MARK, BUTTON_SPRINT}; @@ -187,8 +187,9 @@ async fn empty_cell_hibernates_and_restores_critters_and_markers() { .world .state .load(zone) + .await .unwrap() - .expect("snapshot stored"); + .expect("snapshot stored in celld"); let snapshot: Value = serde_json::from_slice(&saved.state).unwrap(); assert_eq!(snapshot["markers"].as_array().unwrap().len(), 1); let critters = snapshot["critters"].as_array().unwrap().clone(); diff --git a/examples/game/server/tests/support/mod.rs b/examples/game/server/tests/support/mod.rs index 07d620900..838faeb76 100644 --- a/examples/game/server/tests/support/mod.rs +++ b/examples/game/server/tests/support/mod.rs @@ -1,6 +1,7 @@ //! Shared helpers for the multi-host smoke tests: start N in-process hosts -//! of a cluster demo on ephemeral ports and a minimal protocol client that -//! follows Redirects. +//! of a cluster demo on ephemeral ports, sharing one private `celld dev` +//! (zone leases and hibernation snapshots; `game_demos::devcelld`), and a +//! minimal protocol client that follows Redirects. #![allow(dead_code)] use std::net::SocketAddr; @@ -15,6 +16,7 @@ use distributed_game::{ use futures_util::{SinkExt, StreamExt}; use game_demos::cluster::{spawn_cluster, ClusterDemo, ClusterOptions, RunningHost}; use game_demos::common::DemoOptions; +use game_demos::devcelld::DevCelld; use serde_json::Value; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -28,7 +30,7 @@ pub type Ws = WebSocketStream>; pub struct Cluster { pub hosts: Vec>, stops: Vec>, - pub state_dir: std::path::PathBuf, + pub celld: DevCelld, } impl Cluster { @@ -52,7 +54,7 @@ impl Cluster { for host in self.hosts { let _ = host.demo.task.await; } - let _ = std::fs::remove_dir_all(&self.state_dir); + drop(self.celld); } } @@ -68,15 +70,7 @@ pub async fn start_cluster(ids: &[&str], hibernate_after: Durati .iter() .map(|(id, addr, _)| HostAddr::new(id.clone(), addr.to_string())) .collect(); - let state_dir = std::env::temp_dir().join(format!( - "game-demos-{}-{}-{}", - D::NAME, - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); + let celld = DevCelld::start(None).unwrap_or_else(|error| panic!("celld dev: {error}")); let mut hosts = Vec::new(); let mut stops = Vec::new(); for (id, _, listener) in listeners { @@ -84,13 +78,8 @@ pub async fn start_cluster(ids: &[&str], hibernate_after: Durati let options = ClusterOptions { host_id: id, peers: all.clone(), - state_dir: Some(state_dir.clone()), hibernate_after, - // With CELLD_URL + DISTRIBUTED_INTERNAL_SECRET the hosts keep - // zone ownership in celld ZoneLease cells (same smoke tests). - celld: std::env::var("CELLD_URL") - .ok() - .zip(std::env::var("DISTRIBUTED_INTERNAL_SECRET").ok()), + celld: celld.options(), }; let host = spawn_cluster::(listener, DemoOptions::dev(SECRET), options, async move { let _ = stopped.await; @@ -102,7 +91,7 @@ pub async fn start_cluster(ids: &[&str], hibernate_after: Durati Cluster { hosts, stops, - state_dir, + celld, } } diff --git a/tests/celld/fact-worker/Cargo.toml b/tests/celld/fact-worker/Cargo.toml index 4335b0ce5..379deb850 100644 --- a/tests/celld/fact-worker/Cargo.toml +++ b/tests/celld/fact-worker/Cargo.toml @@ -15,7 +15,7 @@ crate-type = ["cdylib"] [dependencies] console_error_panic_hook = "0.1" distributed = { path = "../../..", default-features = false, features = ["workers-rs"] } -distributed_game = { path = "../../../distributed_game", default-features = false, features = ["cell"] } +distributed_game = { path = "../../../distributed_game", default-features = false, features = ["cell", "workers-rs"] } serde = { version = "1", features = ["derive"] } serde_json = "1" worker = { version = "0.8", features = ["queue"] } diff --git a/tests/celld/fact-worker/src/lib.rs b/tests/celld/fact-worker/src/lib.rs index c2b05b154..fcb695261 100644 --- a/tests/celld/fact-worker/src/lib.rs +++ b/tests/celld/fact-worker/src/lib.rs @@ -9,6 +9,9 @@ //! - `ZoneLease:{zone_id}` (kind `zone-lease`): zone ownership //! (`distributed_game::world::zone_lease`) for `CellZoneOwnership`; its //! alarm declares `zone.lease_expired`. +//! - `ZoneState:{zone_id}` (kind `zone-state`): hibernation snapshots +//! (`distributed_game::world::zone_state`) for `CellZoneStateStore`; +//! snapshots above the inline limit go to the `ZONE_SNAPSHOTS` R2 bucket. //! //! Every route requires `DISTRIBUTED_INTERNAL_SECRET`. The handler's session //! is rebuilt from the trusted fact headers (`fact_session_from_headers`): @@ -26,6 +29,7 @@ use distributed::microsvc::{ use distributed::{sourced, Aggregate, Entity}; use distributed_game::cell::{fact_session_from_headers, session_stamp}; use distributed_game::world::zone_lease::{mount_zone_lease, ZoneLease}; +use distributed_game::world::zone_state::{mount_zone_state, ZoneState}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use worker::*; @@ -460,6 +464,35 @@ impl DurableObject for ZoneLeaseCell { } } +#[durable_object] +pub struct ZoneStateCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ZoneStateCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let outbox = CelldOutbox::from_env(&env, "OUTBOX").expect("OUTBOX Queue binding"); + // Every save replaces the value: snapshot each commit. + let cell = mount_zone_state( + AggregateCell::::from_state_with_snapshots(state, 1) + .expect("zone state cell identity"), + ) + .with_celld_outbox(outbox); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + if let Err(error) = authenticate(&req, &self.env) { + return error_response(error); + } + let bucket = + distributed_game::world::zone_state::celld_worker::bucket(&self.env, "ZONE_SNAPSHOTS")?; + distributed_game::world::zone_state::celld_worker::fetch(&self.cell, &bucket, req).await + } +} + #[event(fetch)] async fn main(req: Request, env: Env, _ctx: Context) -> Result { console_error_panic_hook::set_once(); @@ -477,6 +510,7 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { Some("match") => "MATCH", Some("presence") => "PRESENCE", Some("zone-lease") => "ZONE_LEASE", + Some("zone-state") => "ZONE_STATE", _ => return json_status(json!({ "error": "unknown cell kind" }), 404), }; let Some(shard) = parts.get(1) else { diff --git a/tests/celld/fact-worker/wrangler.jsonc b/tests/celld/fact-worker/wrangler.jsonc index 8d890016c..d116119b8 100644 --- a/tests/celld/fact-worker/wrangler.jsonc +++ b/tests/celld/fact-worker/wrangler.jsonc @@ -7,12 +7,17 @@ { "name": "KILLS", "class_name": "KillsCell" }, { "name": "MATCH", "class_name": "MatchCell" }, { "name": "PRESENCE", "class_name": "PresenceCell" }, - { "name": "ZONE_LEASE", "class_name": "ZoneLeaseCell" } + { "name": "ZONE_LEASE", "class_name": "ZoneLeaseCell" }, + { "name": "ZONE_STATE", "class_name": "ZoneStateCell" } ] }, "migrations": [ { "tag": "v1", "new_sqlite_classes": ["KillsCell", "MatchCell", "PresenceCell"] }, - { "tag": "v2", "new_sqlite_classes": ["ZoneLeaseCell"] } + { "tag": "v2", "new_sqlite_classes": ["ZoneLeaseCell"] }, + { "tag": "v3", "new_sqlite_classes": ["ZoneStateCell"] } + ], + "r2_buckets": [ + { "binding": "ZONE_SNAPSHOTS", "bucket_name": "zone-snapshots" } ], "queues": { "producers": [ From 22bb63f507358662fa2561f3be8834ddc791241f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 14:25:08 -0500 Subject: [PATCH 18/40] feat(game)!: list Mob Pit crawlers from a projection, drop roster The `crawler-roster/all` cell was a global singleton every new player passed through (celld cannot enumerate cells). The crawler list is now the `player_stats` read model projected from the Crawler cells' own `crawler.mob_killed` domain events (cell outbox -> Queue -> relay -> bus -> projector): - `mobpit_crawler::stats` holds the read model, mutation and projection shared by `mobpit-service` and the zone server. - The zone server projects the bus into its own database (`mobpit/directory.rs`, bus consumer group `mobpit-zone`, `{data-dir}/read-models.db`); `/dev/mobpit/crawlers` and the board rebuild list crawlers from it, the rebuild still reads kill histories from each listed cell. - The relay route and projection bus move to `common::relay` so other demos can consume their cells' events. - `MobPitWorld::start` opens the world with its projection; `attach_projection_bus` is now `attach_projection`. - The roster cell, `crawler.enlist`, `ROSTER_KIND`/`ROSTER_SHARD`/`ENLIST` and `CrawlerCells::principals` are removed (celld 0.6.2 has no `deleted_classes`, so the class leaves the v1 migration). Refs [[tasks/gr-p32-mobpit-celld]] --- examples/game/README.md | 25 +-- examples/game/cells/Cargo.toml | 2 +- examples/game/cells/src/lib.rs | 108 +----------- examples/game/cells/wrangler.jsonc | 3 +- .../game/client/tests/helpers/demo-server.mjs | 11 ++ examples/game/client/tests/helpers/mobpit.mjs | 4 +- .../tests/leaderboard-headless.test.mjs | 12 +- .../client/tests/mobpit-headless.test.mjs | 13 +- examples/game/run.sh | 4 +- examples/game/server/Cargo.toml | 7 +- examples/game/server/crawler/src/lib.rs | 14 +- .../save_player_stats.mutation.graphql | 0 examples/game/server/crawler/src/stats.rs | 81 +++++++++ examples/game/server/service/src/lib.rs | 87 ++-------- .../game/server/service/tests/projection.rs | 9 +- examples/game/server/src/common/mod.rs | 1 + examples/game/server/src/common/relay.rs | 129 ++++++++++++++ examples/game/server/src/mobpit/board.rs | 18 +- examples/game/server/src/mobpit/directory.rs | 93 ++++++++++ examples/game/server/src/mobpit/mod.rs | 38 ++-- examples/game/server/src/mobpit/routes.rs | 58 +------ examples/game/server/src/mobpit/store.rs | 108 ++---------- examples/game/server/src/mobpit/world.rs | 132 +++++++++++--- examples/game/server/tests/mobpit_smoke.rs | 164 +++++++++++++----- 24 files changed, 662 insertions(+), 459 deletions(-) rename examples/game/server/{service => crawler}/src/mutations/save_player_stats.mutation.graphql (100%) create mode 100644 examples/game/server/crawler/src/stats.rs create mode 100644 examples/game/server/src/common/relay.rs create mode 100644 examples/game/server/src/mobpit/directory.rs diff --git a/examples/game/README.md b/examples/game/README.md index 650321d73..759262bc0 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -9,7 +9,7 @@ guests and lets any page change the network simulation. | Demo | Run | Mechanic it demonstrates | Durable state | |---|---|---|---| | **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | none (stateless: positions only live in the zone) | -| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player celld cells (`cells/`, `celld dev` started by `run.sh`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence cells fed by the runtime's player lifecycle facts (`/dev/mobpit/presence`) | celld: `crawler/{player}`, `crawler-roster/all`, `presence/{player}` | +| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player celld cells (`cells/`, `celld dev` started by `run.sh`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence cells fed by the runtime's player lifecycle facts (`/dev/mobpit/presence`) | celld: `crawler/{player}`, `presence/{player}`; crawler list = `player_stats` read model projected from the cells' events | | **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | celld (the Mob Pit cells); boards are a projection (memory or Redis) | | **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | none (stateless: score is per session) | | **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | celld: `gem-pouch/{player}`, `content-log/{zone}` | @@ -454,11 +454,11 @@ after dying. Level `n` starts at `40·(n−1)²` XP. as trusted headers) to the killer's own celld cell, `crawler/{principal}` (`cells/`, a workers-rs Durable Object per player). The command id is `sha256(service ‖ fact key)`, so the cell's command ledger turns any repeat - of the key into a replay of the stored receipt. Before a principal's first - kill the sink enlists it in the `crawler-roster/all` cell (celld cannot - enumerate cells; the roster is what the board rebuild and - `/dev/mobpit/crawlers` read). celld down: the fact stays queued in the - zone (backpressure at capacity), never in a local store. + of the key into a replay of the stored receipt. Nothing else is written + per kill: there is no index cell every player passes through (celld + cannot enumerate cells, and such a cell would be a global hot spot); the + crawler list is a read model (step 5). celld down: the fact stays queued + in the zone (backpressure at capacity), never in a local store. 3. `crawler/src/lib.rs`: `crawler.record_kill` (a `#[command]` method) takes the principal from the trusted session (`x-user-id`) and **decides the XP from the tier**; the input has no identity or XP field and unknown fields @@ -478,10 +478,15 @@ after dying. Level `n` starts at `40·(n−1)²` XP. `sqlite:{data-dir}/bus.db`). A crash between commit and drain, or a relay outage, only delays it (the cell's alarm re-drains; the Queue retries). `mobpit-service` consumes the bus with a - modeled projector (`projection!` → `upsert_player_stats`), checkpointed - and deduplicated in its own SQLite database, and serves `player_stats` - over GraphQL with `@live`. The service has no commands: nothing on that - side can name a player or an amount. + modeled projector (`projection!` → `upsert_player_stats`, defined in + `crawler/src/stats.rs`), checkpointed and deduplicated in its own SQLite + database, and serves `player_stats` over GraphQL with `@live`. The service + has no commands: nothing on that side can name a player or an amount. + The zone server projects the same events into its own copy + (`mobpit/directory.rs`, bus consumer `mobpit-zone`, + `{data-dir}/read-models.db`): `/dev/mobpit/crawlers` and the leaderboard + rebuild list crawlers from it (eventually consistent; the rebuild then + reads each listed crawler's kill history from its cell). The read path: on join the zone asks a background task to load the player's cell and sends `profile` (`source: "cell"`) a few ticks later. diff --git a/examples/game/cells/Cargo.toml b/examples/game/cells/Cargo.toml index b9213dc34..47f109f76 100644 --- a/examples/game/cells/Cargo.toml +++ b/examples/game/cells/Cargo.toml @@ -1,5 +1,5 @@ # The game demos' durable cells on celld (workers-rs, wasm32): Mob Pit's -# per-player Crawler aggregate, the crawler roster and per-player presence +# per-player Crawler aggregate and per-player presence # (with the outbox Queue consumer that relays Crawler domain events to the # zone server), and the zone lease and hibernation-state cells every # multi-host demo uses. diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs index 95ddfe272..0239c1622 100644 --- a/examples/game/cells/src/lib.rs +++ b/examples/game/cells/src/lib.rs @@ -8,11 +8,6 @@ //! from the mob tier; the principal comes from the trusted fact session). //! Each commit writes the `crawler.mob_killed` domain event to the cell //! outbox, which drains to the `mobpit-outbox` Queue. -//! - `crawler-roster/all` (`CrawlerRosterCell`): the principals that have a -//! Crawler cell. Durable Objects cannot be enumerated, so the zone server -//! enlists a principal (`crawler.enlist`, idempotent by principal) before -//! its first kill commits; the board rebuild and `/dev/mobpit/crawlers` -//! read it. //! - `presence/{principal}` (`PresenceCell`): the runtime's player lifecycle //! facts (`zone.*`). A fact emitted before the one that set the current //! entry (`FactStamp::emission_order`) is ignored, so a late "left pit-1" @@ -40,8 +35,6 @@ //! publishes it onto the projection bus read by `mobpit-service`. Delivery //! is at-least-once with stable message ids; the projector deduplicates. -use std::collections::BTreeSet; - use distributed::bus::{CelldQueueEnvelope, CelldQueueHttpPublisher, CelldQueueRelay}; use distributed::cell_host::{ AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, @@ -61,8 +54,7 @@ use distributed_game::world::zone_lease::{mount_zone_lease, ZoneLease}; use distributed_game::world::zone_state::{celld_worker, mount_zone_state, ZoneState}; use distributed_game::FactStamp; use mobpit_crawler::{ - Crawler, MobKilledArgs, RecordKill, CRAWLER_KIND, ENLIST, MOB_KILLED_EVENT, PRESENCE_KIND, - ROSTER_KIND, ROSTER_SHARD, + Crawler, MobKilledArgs, RecordKill, CRAWLER_KIND, MOB_KILLED_EVENT, PRESENCE_KIND, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -72,68 +64,6 @@ fn rejected(error: impl std::fmt::Display) -> HandlerError { HandlerError::Rejected(error.to_string()) } -// ---------------------------------------------------------------- Roster - -#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] -pub struct CrawlerRoster { - entity: Entity, - principals: BTreeSet, -} - -#[sourced( - entity, - aggregate_type = "CrawlerRoster", - events = "CrawlerRosterEvent" -)] -impl CrawlerRoster { - #[event("crawler_roster.enlisted", version = 1)] - fn enlisted(&mut self, roster: String, principal: String) { - if self.entity.id().is_empty() { - self.entity.set_id(roster); - } - self.principals.insert(principal); - } -} - -#[derive(Debug, Deserialize, distributed::CommandInput)] -pub struct EnlistInput {} - -#[derive(Debug, Serialize, distributed::CommandOutput)] -pub struct Enlisted { - pub crawlers: u32, -} - -struct Enlist; -impl PortableCommand for Enlist -where - D: CausalRouteDependencies + Send + Sync + 'static, -{ - fn install(self, routes: Routes) -> Routes { - routes - .typed_command(typed_command::>(ENLIST)) - .handle(handle_enlist) - } -} - -async fn handle_enlist( - ctx: &CausalCommandContext<'_, CrawlerRoster>, - _input: EnlistInput, -) -> std::result::Result>, HandlerError> { - let principal = ctx.user_id()?.to_string(); - let repo = ctx.repo(); - let mut roster = match repo.get(ROSTER_SHARD).await? { - Some(roster) => roster, - None => repo.create(), - }; - if !roster.principals.contains(&principal) { - roster - .enlisted(ROSTER_SHARD.to_string(), principal) - .map_err(rejected)?; - } - let crawlers = roster.principals.len() as u32; - repo.commit(roster)?.succeeded(Enlisted { crawlers }) -} - // ---------------------------------------------------------------- Presence /// The ordering key of the fact that set a presence entry. @@ -640,13 +570,6 @@ async fn read_crawler(cell: &AggregateCell) -> std::result::Result) -> std::result::Result { - let roster = cell.load().await.map_err(|error| error.to_string())?; - Ok(json!({ - "principals": roster.map(|r| r.principals.into_iter().collect::>()).unwrap_or_default(), - })) -} - async fn read_presence(cell: &AggregateCell) -> std::result::Result { let presence = cell.load().await.map_err(|error| error.to_string())?; Ok(json!({ "current": presence.and_then(|p| p.current) })) @@ -681,31 +604,6 @@ impl DurableObject for CrawlerCell { } } -#[durable_object] -pub struct CrawlerRosterCell { - cell: AggregateCell, - env: Env, -} - -impl DurableObject for CrawlerRosterCell { - fn new(state: State, env: Env) -> Self { - console_error_panic_hook::set_once(); - let cell = AggregateCell::::from_state_with_snapshots(state, 50) - .expect("roster cell identity") - .mount(Enlist) - .with_celld_outbox(outbox(&env)); - Self { cell, env } - } - - async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_roster, fact_session).await - } - - async fn alarm(&self) -> Result { - alarm(&self.env, &self.cell).await - } -} - #[durable_object] pub struct PresenceCell { cell: AggregateCell, @@ -837,7 +735,6 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { let parts: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); let binding = match parts.first().copied() { Some(CRAWLER_KIND) => "CRAWLER", - Some(ROSTER_KIND) => "CRAWLER_ROSTER", Some(PRESENCE_KIND) => "PRESENCE", Some("gem-pouch") => "GEM_POUCH", Some("content-log") => "CONTENT_LOG", @@ -849,9 +746,6 @@ async fn main(req: Request, env: Env, _ctx: Context) -> Result { return json_status(json!({ "error": "missing shard" }), 400); }; let shard = percent_decode(shard).ok_or_else(|| Error::RustError("bad shard".into()))?; - if binding == "CRAWLER_ROSTER" && shard != ROSTER_SHARD { - return json_status(json!({ "error": "unknown roster" }), 404); - } let stub = env .durable_object(binding)? .id_from_name(&shard)? diff --git a/examples/game/cells/wrangler.jsonc b/examples/game/cells/wrangler.jsonc index 2d06fcd8f..b4b42e049 100644 --- a/examples/game/cells/wrangler.jsonc +++ b/examples/game/cells/wrangler.jsonc @@ -9,7 +9,6 @@ "durable_objects": { "bindings": [ { "name": "CRAWLER", "class_name": "CrawlerCell" }, - { "name": "CRAWLER_ROSTER", "class_name": "CrawlerRosterCell" }, { "name": "PRESENCE", "class_name": "PresenceCell" }, { "name": "ZONE_LEASE", "class_name": "ZoneLeaseCell" }, { "name": "ZONE_STATE", "class_name": "ZoneStateCell" }, @@ -18,7 +17,7 @@ ] }, "migrations": [ - { "tag": "v1", "new_sqlite_classes": ["CrawlerCell", "CrawlerRosterCell", "PresenceCell"] }, + { "tag": "v1", "new_sqlite_classes": ["CrawlerCell", "PresenceCell"] }, { "tag": "v2", "new_sqlite_classes": ["ZoneLeaseCell", "ZoneStateCell"] }, { "tag": "v3", "new_sqlite_classes": ["GemPouchCell", "ContentLogCell"] } ], diff --git a/examples/game/client/tests/helpers/demo-server.mjs b/examples/game/client/tests/helpers/demo-server.mjs index 5f95bc205..0e2c76278 100644 --- a/examples/game/client/tests/helpers/demo-server.mjs +++ b/examples/game/client/tests/helpers/demo-server.mjs @@ -49,6 +49,17 @@ export async function until(predicate, timeoutMs = 10_000, what = 'condition') { } } +/** Poll an async `probe` until it returns a truthy value (returned). */ +export async function eventually(probe, timeoutMs = 10_000, what = 'condition') { + const start = Date.now(); + for (;;) { + const value = await probe(); + if (value) return value; + if (Date.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); /** In-memory Storage for DevApi in Node. */ diff --git a/examples/game/client/tests/helpers/mobpit.mjs b/examples/game/client/tests/helpers/mobpit.mjs index cac3d558e..0dec97c60 100644 --- a/examples/game/client/tests/helpers/mobpit.mjs +++ b/examples/game/client/tests/helpers/mobpit.mjs @@ -1,6 +1,6 @@ /** - * Headless Mob Pit helpers: a server on its own `celld dev` (the Crawler, - * roster and presence cells), a throwaway data dir and a scripted hunter + * Headless Mob Pit helpers: a server on its own `celld dev` (the Crawler + * and presence cells), a throwaway data dir and a scripted hunter * that walks to the nearest mob, faces it and swings until `done()` is true. */ import { mkdtempSync } from 'node:fs'; diff --git a/examples/game/client/tests/leaderboard-headless.test.mjs b/examples/game/client/tests/leaderboard-headless.test.mjs index 68f248ae0..4a72e22e8 100644 --- a/examples/game/client/tests/leaderboard-headless.test.mjs +++ b/examples/game/client/tests/leaderboard-headless.test.mjs @@ -8,7 +8,7 @@ import test from 'node:test'; import { createBoardWatcher, createMobPitClient, MobPitApi } from '../src/mobpit/client.ts'; import { DevApi } from '../src/shared/dev-api.ts'; -import { memoryStorage, until } from './helpers/demo-server.mjs'; +import { eventually, memoryStorage, until } from './helpers/demo-server.mjs'; import { hunt, startPit } from './helpers/mobpit.mjs'; test('board pushes match the cells and rebuild is identical', { timeout: 600_000 }, async () => { @@ -41,8 +41,14 @@ test('board pushes match the cells and rebuild is identical', { timeout: 600_000 await until(() => row(watcher.board(), 'kills')?.score === profile.kills, 10_000, 'pushed kills'); assert.equal(row(watcher.board(), 'kills').name, 'Ranker'); assert.equal(row(watcher.board(), 'xp').score, profile.xp); - const { crawlers } = await pitApi.crawlers(); - assert.deepEqual(crawlers.map((c) => [c.principal, c.kills, c.xp]), [[me, profile.kills, profile.xp]]); + // The crawler list (and so the rebuild) is a projection of the cells' + // kill events: eventually equal to the cells. + const listed = async () => (await pitApi.crawlers()).crawlers.map((c) => [c.principal, c.kills, c.xp]); + await eventually( + async () => JSON.stringify(await listed()) === JSON.stringify([[me, profile.kills, profile.xp]]), + 15_000, + 'crawler directory' + ); const { me: mine } = await pitApi.leaderboard(me); assert.equal(mine.kills.rank, 1); diff --git a/examples/game/client/tests/mobpit-headless.test.mjs b/examples/game/client/tests/mobpit-headless.test.mjs index 231d3d189..2bb038757 100644 --- a/examples/game/client/tests/mobpit-headless.test.mjs +++ b/examples/game/client/tests/mobpit-headless.test.mjs @@ -11,7 +11,7 @@ import test from 'node:test'; import { isPit, pitFields } from '../src/generated/mobpit/index.ts'; import { createMobPitClient, MobPitApi } from '../src/mobpit/client.ts'; import { DevApi } from '../src/shared/dev-api.ts'; -import { memoryStorage, until } from './helpers/demo-server.mjs'; +import { eventually, memoryStorage, until } from './helpers/demo-server.mjs'; import { hunt, startPit } from './helpers/mobpit.mjs'; const XP = new Set([10, 25, 60]); @@ -84,8 +84,15 @@ test('kills become durable facts without stalling the tick, and survive a restar assert.equal(again.profile().source, 'cell'); assert.equal(again.profile().kills, before.kills); assert.equal(again.profile().xp, before.xp); - const crawlers = (await new MobPitApi(server.httpUrl).crawlers()).crawlers; - assert.equal(crawlers.length, 1); + // The crawler list is a projection of the cells' kill events. + const crawlers = await eventually( + async () => { + const { crawlers } = await new MobPitApi(server.httpUrl).crawlers(); + return crawlers.length === 1 && crawlers[0].kills === kills.length && crawlers; + }, + 15_000, + 'crawler directory' + ); assert.equal(crawlers[0].kills, kills.length); } finally { for (const c of clients) c.close(); diff --git a/examples/game/run.sh b/examples/game/run.sh index ba84fbabd..2ce4d481d 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -32,8 +32,8 @@ # uses GAME_DEMOS_PORT, default 2967 there, and its read-model service # MOBPIT_SERVICE_PORT, default GAME_DEMOS_PORT + 2). # -# celld: the demos' durable cells (examples/game/cells: Mob Pit's Crawler, -# roster and presence cells, The Freeze's gem and content cells, every +# celld: the demos' durable cells (examples/game/cells: Mob Pit's Crawler +# and presence cells, The Freeze's gem and content cells, every # multi-host demo's zone leases and hibernation snapshots) run on celld, # never natively; state persists in .data//celld (Mob Pit: # MOBPIT_DATA_DIR/celld). CELLD_BIN names the binary (default: `celld` on diff --git a/examples/game/server/Cargo.toml b/examples/game/server/Cargo.toml index b74143b12..61c192142 100644 --- a/examples/game/server/Cargo.toml +++ b/examples/game/server/Cargo.toml @@ -30,10 +30,11 @@ game_sim = { path = "sim" } # The Crawler aggregate + RecordKill (shared with the read-model service). mobpit_crawler = { path = "crawler" } distributed_game = { path = "../../../distributed_game", features = ["net", "cell", "celld"] } -# Mob Pit: sorted-set leaderboards (Redis optional at runtime) + the SQLite -# bus for the read-model service. Its durable cells run on celld +# Mob Pit: sorted-set leaderboards (Redis optional at runtime), the SQLite +# projection bus and the modeled projector of the crawler directory +# (`graphql` gates modeled projections). Its durable cells run on celld # (`examples/game/cells`), reached with `distributed_game`'s `celld` feature. -distributed = { path = "../../..", features = ["redis", "sqlite"] } +distributed = { path = "../../..", features = ["redis", "sqlite", "graphql"] } # Reads of the celld cells over the internal path. reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] } # The SQLite bus the cells' domain events are relayed onto (`--projection-bus`). diff --git a/examples/game/server/crawler/src/lib.rs b/examples/game/server/crawler/src/lib.rs index e3dab6627..f014053ec 100644 --- a/examples/game/server/crawler/src/lib.rs +++ b/examples/game/server/crawler/src/lib.rs @@ -21,8 +21,11 @@ //! post-kill [`CrawlerState`] into the cell's outbox in the same cell (SQLite) //! transaction ([`CrawlerMobKilledDomainEvent`]). celld drains that outbox //! to a Queue whose consumer relays it to the zone server, which publishes -//! it onto a bus; the read-model service (`mobpit_service`) projects it -//! into the `player_stats` table behind GraphQL `@live`. +//! it onto a bus; [`stats`] is its `player_stats` read model, projected by +//! the read-model service (`mobpit_service`, GraphQL `@live`) and by the +//! zone server (its crawler list). + +pub mod stats; use distributed::{sourced, DomainState, Entity}; use serde::{Deserialize, Serialize}; @@ -31,13 +34,6 @@ use serde::{Deserialize, Serialize}; pub const AGGREGATE_TYPE: &str = "Crawler"; /// celld cell kind (URL segment) of the Crawler cells: `crawler/{principal}`. pub const CRAWLER_KIND: &str = "crawler"; -/// celld cell kind of the crawler roster (celld cannot enumerate cells, so -/// the principals with a Crawler cell are indexed there). -pub const ROSTER_KIND: &str = "crawler-roster"; -/// The roster's one shard. -pub const ROSTER_SHARD: &str = "all"; -/// Roster command: add the session principal (idempotent). -pub const ENLIST: &str = "crawler.enlist"; /// celld cell kind of per-player presence (the runtime lifecycle facts). pub const PRESENCE_KIND: &str = "presence"; /// Command dispatched for every `MobKilled` fact. diff --git a/examples/game/server/service/src/mutations/save_player_stats.mutation.graphql b/examples/game/server/crawler/src/mutations/save_player_stats.mutation.graphql similarity index 100% rename from examples/game/server/service/src/mutations/save_player_stats.mutation.graphql rename to examples/game/server/crawler/src/mutations/save_player_stats.mutation.graphql diff --git a/examples/game/server/crawler/src/stats.rs b/examples/game/server/crawler/src/stats.rs new file mode 100644 index 000000000..92c08175d --- /dev/null +++ b/examples/game/server/crawler/src/stats.rs @@ -0,0 +1,81 @@ +//! The `player_stats` read model: one row per Crawler, the post-kill +//! [`CrawlerState`](crate::CrawlerState) carried by every +//! `crawler.mob_killed` domain event. +//! +//! Projected from the bus by every reader that needs it, each with its own +//! consumer identity and its own database: `mobpit_service` (GraphQL +//! `@live` leaderboard) and the zone server (the crawler list behind +//! `/dev/mobpit/crawlers` and the leaderboard rebuild). celld cannot list +//! cells, and a cell every player passes through would be a global hot +//! spot; the read model is the list. + +use distributed::graphql::SurfaceProjector; +use distributed::projection::lower::{EventualOnly, ProjectionDescriptor}; +use distributed::{ + mutation_file, LocalProjectionMountsBuilder, Mutation, ReadModel, ReadModelCatalog, + RelationalReadModel, +}; +use serde::{Deserialize, Serialize}; + +use crate::CrawlerMobKilledDomainEvent; + +/// Application of the read model (catalog and projector owner). +pub const APPLICATION: &str = "mobpit"; +/// Projection name. +pub const PROJECTOR: &str = "project_player_stats"; +/// Projection epoch: bump to rebuild the table from the bus log. +pub const EPOCH: &str = "mobpit-player-stats-v1"; + +/// One player's settled progress (a projection of their Crawler cell). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ReadModel)] +#[table("player_stats")] +pub struct PlayerStats { + /// Principal (the verified session's user id). + #[id("player_id")] + pub player_id: String, + /// Display name at the last kill. + pub name: String, + /// Total XP (decided by the Crawler cell, never by a caller). + pub xp: i64, + /// Mobs killed. + pub kills: i64, + /// Level for `xp`. + pub level: i64, +} + +/// Upsert one `player_stats` row from the domain event's state body. +#[allow(non_snake_case)] +pub fn SavePlayerStats() -> Mutation<()> { + mutation_file!("src/mutations/save_player_stats.mutation.graphql") +} + +distributed::projection! { + pub const PLAYER_STATS: ProjectionDescriptor = { + name: "project_player_stats", + version: 1, + epoch: "mobpit-player-stats-v1", + model: PlayerStats, + on { + events: [CrawlerMobKilledDomainEvent], + mutation: SavePlayerStats, + input: { player: body }, + }, + }; +} + +/// Physical read-model catalog (`player_stats`). +pub fn read_model_catalog() -> ReadModelCatalog { + ReadModelCatalog::new(APPLICATION).table_schema(PlayerStats::schema().clone()) +} + +/// The modeled projector mount for [`PLAYER_STATS`]. +pub fn projector() -> SurfaceProjector { + LocalProjectionMountsBuilder::new(APPLICATION, "ordered-domain-events") + .expect("projection source") + .eventual_model::(PROJECTOR, PLAYER_STATS, EPOCH) + .expect("player stats mount") + .build() + .expect("projection catalog") + .projector(PROJECTOR) + .expect("player stats projector") +} diff --git a/examples/game/server/service/src/lib.rs b/examples/game/server/service/src/lib.rs index 690232fb7..a17ff3566 100644 --- a/examples/game/server/service/src/lib.rs +++ b/examples/game/server/service/src/lib.rs @@ -32,89 +32,28 @@ use distributed::bus::SqliteBus; use distributed::graphql::{ build_surface, read, surface_for_application_contract, DistributedClientSurfaceExport, GraphqlEngine, IdentityConfig, ModelPermissions, OidcConfig, RoleGrant, Surface, - SurfaceOptions, SurfaceProjector, + SurfaceOptions, }; use distributed::microsvc::{ spawn_service_consumer_loop, CausalProjectorContext, HandlerError, ModeledProjection, Routes, Service, CONSUMER_IDLE_POLL, }; -use distributed::projection::lower::{EventualOnly, ProjectionDescriptor}; use distributed::{ - mutation_file, Application, ApplicationManifest, LocalProjectionMountsBuilder, Mutation, - ReadModel, ReadModelCatalog, RelationalReadModel, SqliteRepository, SurfaceSpec, + Application, ApplicationManifest, RelationalReadModel, SqliteRepository, SurfaceSpec, }; -use mobpit_crawler::CrawlerMobKilledDomainEvent; -use serde::{Deserialize, Serialize}; -/// Application, GraphQL service id and client surface name. -pub const APPLICATION: &str = "mobpit"; -/// Projection name. -pub const PROJECTOR: &str = "project_player_stats"; -/// Projection epoch: bump to rebuild the table from the bus log. -pub const EPOCH: &str = "mobpit-player-stats-v1"; +/// The read model and its projection (shared with the zone server). +pub use mobpit_crawler::stats::{ + projector, read_model_catalog, PlayerStats, SavePlayerStats, APPLICATION, EPOCH, PLAYER_STATS, + PROJECTOR, +}; /// The one engine role. Every signed-in player reads the whole board. pub const ROLE: &str = "user"; -/// One player's settled progress (a projection of their Crawler cell). -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ReadModel)] -#[table("player_stats")] -pub struct PlayerStats { - /// Principal (the verified session's user id). - #[id("player_id")] - pub player_id: String, - /// Display name at the last kill. - pub name: String, - /// Total XP (decided by the Crawler cell, never by a caller). - pub xp: i64, - /// Mobs killed. - pub kills: i64, - /// Level for `xp`. - pub level: i64, -} - -impl PlayerStats { - /// Read authorization: the leaderboard is public to signed-in players; - /// there is nothing to write (no commands, no mutations). - pub fn permissions() -> ModelPermissions { - ModelPermissions::new().grant(ROLE, read().all_columns()) - } -} - -/// Upsert one `player_stats` row from the domain event's state body. -#[allow(non_snake_case)] -pub fn SavePlayerStats() -> Mutation<()> { - mutation_file!("src/mutations/save_player_stats.mutation.graphql") -} - -distributed::projection! { - pub const PLAYER_STATS: ProjectionDescriptor = { - name: "project_player_stats", - version: 1, - epoch: "mobpit-player-stats-v1", - model: PlayerStats, - on { - events: [CrawlerMobKilledDomainEvent], - mutation: SavePlayerStats, - input: { player: body }, - }, - }; -} - -/// Physical read-model catalog (`player_stats`). -pub fn read_model_catalog() -> ReadModelCatalog { - ReadModelCatalog::new(APPLICATION).table_schema(PlayerStats::schema().clone()) -} - -/// The modeled projector mount for [`PLAYER_STATS`]. -pub fn projector() -> SurfaceProjector { - LocalProjectionMountsBuilder::new(APPLICATION, "ordered-domain-events") - .expect("projection source") - .eventual_model::(PROJECTOR, PLAYER_STATS, EPOCH) - .expect("player stats mount") - .build() - .expect("projection catalog") - .projector(PROJECTOR) - .expect("player stats projector") +/// Read authorization: the leaderboard is public to signed-in players; +/// there is nothing to write (no commands, no mutations). +pub fn permissions() -> ModelPermissions { + ModelPermissions::new().grant(ROLE, read().all_columns()) } async fn project( @@ -146,7 +85,7 @@ pub fn spawn_projector(read_models: SqliteRepository, bus: sqlx::SqlitePool) { fn grants() -> BTreeMap> { let mut grants: BTreeMap> = BTreeMap::new(); - for (role, grant) in PlayerStats::permissions().surface_grants() { + for (role, grant) in permissions().surface_grants() { grants .entry(role) .or_default() @@ -217,7 +156,7 @@ pub fn build_engine( .protocol_token_key(protocol_key) .roles(&[ROLE]) .client_application_surface(APPLICATION, [ROLE], [ROLE]) - .model::(PlayerStats::permissions()) + .model::(permissions()) .client_projection_owners([projector().into()]) .identity(identity.config()) .graphiql(false) diff --git a/examples/game/server/service/tests/projection.rs b/examples/game/server/service/tests/projection.rs index ee5bb6734..29659d4f7 100644 --- a/examples/game/server/service/tests/projection.rs +++ b/examples/game/server/service/tests/projection.rs @@ -100,14 +100,17 @@ async fn settled_kills_project_into_live_player_stats() { distributed::bus::CELLD_QUEUE_RELAY_PATH ); let celld = DevCelld::start(Some(relay)).unwrap_or_else(|error| panic!("celld dev: {error}")); - let world = MobPitWorld::open(WorldOptions::new(dir.join("local"), celld.options())).unwrap(); - let routes = game_demos::mobpit::routes::internal_routes(world.clone()); + let mut options = WorldOptions::new(dir.join("local"), celld.options()); + options.projection_bus = Some(bus_url.clone()); + let world = MobPitWorld::open(options).unwrap(); + let routes = + game_demos::common::relay::relay_routes(world.celld().secret.clone(), world.projection()); tokio::spawn(async move { axum::serve(listener, routes).await.unwrap() }); let sink = world.fact_sink(); // A kill settled before the bus is attached waits in the Queue (the // relay answers 503 and the consumer retries). sink.commit(&kill("k0", "dev-ana", 2, "ana")).await.unwrap(); - world.attach_projection_bus(&bus_url).await.unwrap(); + world.attach_projection().await.unwrap(); // Service side. let repo = SqliteRepository::connect_and_migrate(&db_url) diff --git a/examples/game/server/src/common/mod.rs b/examples/game/server/src/common/mod.rs index bdc57e3c8..095ba6d0a 100644 --- a/examples/game/server/src/common/mod.rs +++ b/examples/game/server/src/common/mod.rs @@ -11,6 +11,7 @@ pub mod celld; pub mod dev; mod guests; +pub mod relay; use std::future::Future; use std::net::SocketAddr; diff --git a/examples/game/server/src/common/relay.rs b/examples/game/server/src/common/relay.rs new file mode 100644 index 000000000..ab01c0252 --- /dev/null +++ b/examples/game/server/src/common/relay.rs @@ -0,0 +1,129 @@ +//! The demos' end of celld's outbox Queue: cells commit domain events into +//! their outbox, celld drains it to the cells worker's Queue, and the +//! worker's consumer relays every message to `CELLD_QUEUE_RELAY_URL` — a +//! demo server's authenticated `/internal/celld-queue/relay` (this module). +//! The route publishes the message onto a [`ProjectionBus`] (a SQLite bus), +//! where read-model projectors (each with its own consumer identity) +//! consume it. +//! +//! Delivery is at-least-once with stable message ids; the projectors +//! deduplicate. Until a bus is attached the route answers 503 and the Queue +//! keeps (retries) the message. + +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::post; +use axum::{Json, Router}; +use distributed::bus::{ + celld_queue_relay_handler, CelldQueueEnvelope, CelldQueueRelayHandler, SqliteBus, + CELLD_QUEUE_RELAY_PATH, +}; +use distributed::cell_host::{InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER}; +use distributed::BusPublisher; +use serde_json::{json, Value}; + +/// Open (creating) a SQLite database another process may share: WAL and a +/// busy timeout, so writers and readers in different processes wait for +/// each other instead of failing. +pub async fn open_shared_sqlite( + url: &str, + max_connections: u32, +) -> Result { + use std::str::FromStr; + let options = sqlx::sqlite::SqliteConnectOptions::from_str(url) + .map_err(|e| format!("{url}: {e}"))? + .create_if_missing(true) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) + .busy_timeout(Duration::from_secs(10)); + sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(max_connections) + .connect_with(options) + .await + .map_err(|e| format!("{url}: {e}")) +} + +/// The SQLite bus relayed Queue messages are published onto. +#[derive(Clone)] +pub struct ProjectionBus { + pool: sqlx::SqlitePool, + relay: CelldQueueRelayHandler, +} + +impl ProjectionBus { + /// Open (creating) the bus at a `sqlite:` URL (shareable between + /// processes). + pub async fn open(url: &str) -> Result { + let pool = open_shared_sqlite(url, 4) + .await + .map_err(|e| format!("projection bus {e}"))?; + let bus = SqliteBus::new(pool.clone()); + bus.ensure_tables() + .await + .map_err(|e| format!("projection bus tables: {e}"))?; + Ok(Self { + relay: celld_queue_relay_handler(BusPublisher::new(Arc::new(bus))), + pool, + }) + } + + /// The bus database (projectors consume it with their own identity). + pub fn pool(&self) -> &sqlx::SqlitePool { + &self.pool + } + + /// The Queue relay handler publishing onto this bus. + pub fn relay_handler(&self) -> &CelldQueueRelayHandler { + &self.relay + } +} + +#[derive(Clone)] +struct RelayState { + secret: InternalHttpSecret, + bus: Arc>, +} + +/// `POST /internal/celld-queue/relay` (`DISTRIBUTED_INTERNAL_SECRET`): +/// publish the relayed message onto `bus` (503 while it is unset). +pub fn relay_routes(secret: InternalHttpSecret, bus: Arc>) -> Router { + Router::new() + .route(CELLD_QUEUE_RELAY_PATH, post(relay)) + .with_state(RelayState { secret, bus }) +} + +async fn relay( + State(state): State, + headers: HeaderMap, + Json(envelope): Json, +) -> (StatusCode, Json) { + let authorized = headers + .get(CELL_INTERNAL_SECRET_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| state.secret.matches(candidate)); + if !authorized { + return ( + StatusCode::UNAUTHORIZED, + Json(json!({ "code": "UNAUTHORIZED", "error": "unauthorized" })), + ); + } + let Some(bus) = state.bus.get() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "code": "UNAVAILABLE", "error": "no projection bus attached" })), + ); + }; + match bus.relay_handler()(envelope).await { + Ok(()) => (StatusCode::ACCEPTED, Json(json!({ "ok": true }))), + Err(error) if error.is_retryable() => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "code": "UNAVAILABLE", "error": error.message() })), + ), + Err(error) => ( + StatusCode::BAD_REQUEST, + Json(json!({ "code": "BAD_REQUEST", "error": error.message() })), + ), + } +} diff --git a/examples/game/server/src/mobpit/board.rs b/examples/game/server/src/mobpit/board.rs index 22b02da4a..9e976aa2d 100644 --- a/examples/game/server/src/mobpit/board.rs +++ b/examples/game/server/src/mobpit/board.rs @@ -11,9 +11,10 @@ //! "mob_killed")` — the same id whether it comes from the live receipt, a //! ledger replay of the same fact, or the cell's event history. The //! projector's apply keys (`"{id}#{change}"`) make every repeat a no-op. -//! - Rebuild ([`Leaderboards::rebuild`]): read every cell's durable event -//! history, project it into the *staging* boards and atomically promote -//! them. Runs at startup too (the in-memory board starts empty). +//! - Rebuild ([`Leaderboards::rebuild`]): read the durable event history +//! of every crawler in the directory (`super::directory`) from its cell, +//! project it into the *staging* boards and atomically promote them. +//! Runs at startup too (the in-memory board starts empty). //! - Snapshot ([`BoardSnapshot`]): a `#[derive(GameRecord)]` struct, pushed //! to clients as the zone's typed `board` event and served as camelCase //! JSON by `/dev/mobpit/leaderboard`; the generated TypeScript type @@ -428,15 +429,18 @@ impl Leaderboards { )) } - /// Rebuild both boards from every cell's durable event history and check - /// the result against the cells' current state. - pub async fn rebuild(&self, cells: &CrawlerCells) -> Result { + /// Rebuild both boards from the durable event history of `principals`' + /// cells and check the result against the cells' current state. + pub async fn rebuild( + &self, + principals: Vec, + cells: &CrawlerCells, + ) -> Result { let _one_at_a_time = self.rebuilding.lock().await; let started = Instant::now(); let before = self.top_rows().await.map_err(|e| e.to_string())?; let mut occurrences = Vec::new(); let mut expected: BTreeMap = BTreeMap::new(); - let principals = cells.principals().await?; for principal in &principals { let view = cells.crawler(principal).await?; for kill in view.kills { diff --git a/examples/game/server/src/mobpit/directory.rs b/examples/game/server/src/mobpit/directory.rs new file mode 100644 index 000000000..44aebbebe --- /dev/null +++ b/examples/game/server/src/mobpit/directory.rs @@ -0,0 +1,93 @@ +//! The zone server's list of crawlers: the `player_stats` read model +//! (`mobpit_crawler::stats`) projected from the projection bus into a +//! database of its own, under a consumer identity of its own +//! ([`CONSUMER`]), next to `mobpit-service`'s copy. +//! +//! celld cannot enumerate cells, and an index cell every new player passes +//! through would be a global hot spot; the Crawler cells' own +//! `crawler.mob_killed` domain events (cell outbox → Queue → relay → bus) +//! already carry every crawler's post-kill state. `/dev/mobpit/crawlers` +//! and the leaderboard rebuild read this projection. It is eventually +//! consistent: a kill shows up once its event has been relayed and +//! projected. + +use distributed::bus::SqliteBus; +use distributed::microsvc::{ + spawn_service_consumer_loop, CausalProjectorContext, HandlerError, ModeledProjection, Routes, + Service, CONSUMER_IDLE_POLL, +}; +use distributed::SqliteRepository; +use mobpit_crawler::stats::{ + projector, read_model_catalog, PlayerStats, APPLICATION, PLAYER_STATS, +}; + +/// Bus consumer group of the zone server's projector (its offset in the +/// bus; `mobpit-service` consumes as `mobpit`). The service identity stays +/// the projector's owner application, `mobpit`. +pub const CONSUMER: &str = "mobpit-zone"; + +/// See the module docs. +#[derive(Clone)] +pub struct CrawlerDirectory { + repo: SqliteRepository, +} + +async fn project( + context: CausalProjectorContext, + projection: ModeledProjection, +) -> Result<(), HandlerError> { + projection.apply(PLAYER_STATS, &context).await +} + +impl CrawlerDirectory { + /// Open (creating) the read model at `url` and project `bus` into it + /// forever (the cursor lives in the read model, so a restart resumes). + pub async fn open(url: &str, bus: sqlx::SqlitePool) -> Result { + let repo = SqliteRepository::connect_and_migrate(url) + .await + .map_err(|e| format!("crawler directory {url}: {e}"))?; + let registry = read_model_catalog() + .table_registry() + .map_err(|e| format!("crawler directory tables: {e}"))?; + repo.bootstrap_table_schema_for_dev(®istry) + .await + .map_err(|e| format!("crawler directory tables: {e}"))?; + let read_models = repo.clone(); + spawn_service_consumer_loop(move || { + Service::new() + .named(APPLICATION) + .routes( + Routes::new() + .with_read_model_store(read_models.clone()) + .modeled_projector(projector()) + .handle(project), + ) + .with_bus( + SqliteBus::new_with_group(bus.clone(), CONSUMER) + .with_idle_poll(CONSUMER_IDLE_POLL), + ) + }); + Ok(Self { repo }) + } + + /// Every projected crawler, by principal. + pub async fn list(&self) -> Result, String> { + sqlx::query_as::<_, (String, String, i64, i64, i64)>( + "SELECT player_id, name, xp, kills, level FROM player_stats ORDER BY player_id", + ) + .fetch_all(self.repo.pool()) + .await + .map(|rows| { + rows.into_iter() + .map(|(player_id, name, xp, kills, level)| PlayerStats { + player_id, + name, + xp, + kills, + level, + }) + .collect() + }) + .map_err(|e| format!("crawler directory: {e}")) + } +} diff --git a/examples/game/server/src/mobpit/mod.rs b/examples/game/server/src/mobpit/mod.rs index d0a092eb9..83c4d2c82 100644 --- a/examples/game/server/src/mobpit/mod.rs +++ b/examples/game/server/src/mobpit/mod.rs @@ -9,8 +9,9 @@ //! - [`store`]: the zone server's side of the celld cells //! (`examples/game/cells`): the `FactSink` (with the "slow storage" //! latency knob) that routes facts to them via -//! `distributed_game::celld::CelldFactSink`, the roster and cell reader, -//! and the projection bus fed by the outbox Queue relay. +//! `distributed_game::celld::CelldFactSink`, and the cell reader. +//! - [`directory`]: the crawler list, a projection of the cells' domain +//! events (outbox → Queue → `crate::common::relay` → bus). //! - [`board`]: kills/XP leaderboards (in-memory or Redis), fire-and-forget //! live updates, idempotent apply keys, rebuild from the cells' events. //! - [`presence`]: who is online where, projected from the runtime's player @@ -27,6 +28,7 @@ pub mod board; pub mod crawler; +pub mod directory; pub mod presence; pub mod routes; pub mod store; @@ -49,13 +51,14 @@ pub struct MobPitArgs { /// Shared demo options. #[command(flatten)] pub common: DemoArgs, - /// celld serving `examples/game/cells` (the Crawler, roster and - /// presence cells). The internal secret comes from + /// celld serving `examples/game/cells` (the Crawler and presence + /// cells). The internal secret comes from /// `DISTRIBUTED_INTERNAL_SECRET`. #[arg(long, env = "CELLD_URL")] pub celld_url: String, - /// Directory of local dev files: the persisted dev ticket secret and - /// the default projection bus. Default: `$TMPDIR/game-demos-mobpit`. + /// Directory of local dev files: the persisted dev ticket secret, the + /// default projection bus and the crawler directory's read models. + /// Default: `$TMPDIR/game-demos-mobpit`. #[arg(long, env = "MOBPIT_DATA_DIR")] pub data_dir: Option, /// Redis for the leaderboard (`redis://127.0.0.1:6379`); default in memory. @@ -66,8 +69,8 @@ pub struct MobPitArgs { pub storage_latency_ms: u64, /// SQLite bus (`sqlite:/path/bus.db`) the Crawler cells' /// `crawler.mob_killed` domain events are published to (relayed from - /// celld's outbox Queue); `mobpit-service` projects them into its - /// `player_stats` read model. Default: `sqlite:{data-dir}/bus.db`. + /// celld's outbox Queue); `mobpit-service` and this server's crawler + /// directory project them. Default: `sqlite:{data-dir}/bus.db`. #[arg(long, env = "MOBPIT_PROJECTION_BUS")] pub projection_bus: Option, } @@ -144,15 +147,19 @@ pub async fn run(mut args: MobPitArgs) -> std::io::Result<()> { let celld = store::CelldOptions::new(&args.celld_url, secret).map_err(std::io::Error::other)?; let mut options = WorldOptions::new(&data_dir, celld); options.redis_url = args.redis_url.clone(); - let world = MobPitWorld::open(options).map_err(std::io::Error::other)?; + options.projection_bus = args.projection_bus.clone(); + let world = MobPitWorld::start(options) + .await + .map_err(std::io::Error::other)?; world .sink_state() .set_latency(std::time::Duration::from_millis(args.storage_latency_ms)); if args.common.secret.is_none() { args.common.secret = Some(persisted_secret(&data_dir)?); } - // Board = projection of the cells: rebuild it before accepting players - // (the in-memory board starts empty; Redis may have missed updates). + // Board = projection of the cells: rebuild it from the crawler + // directory before accepting players (the in-memory board starts + // empty; Redis may have missed updates). match world.rebuild_board().await { Ok(report) => eprintln!( "mobpit: leaderboard rebuilt from cells: {}", @@ -160,14 +167,7 @@ pub async fn run(mut args: MobPitArgs) -> std::io::Result<()> { ), Err(error) => eprintln!("mobpit: leaderboard rebuild failed (serving anyway): {error}"), } - let bus = args - .projection_bus - .clone() - .unwrap_or_else(|| format!("sqlite:{}", data_dir.join("bus.db").display())); - world - .attach_projection_bus(&bus) - .await - .map_err(std::io::Error::other)?; + let bus = world.bus_url().to_string(); eprintln!( "mobpit: celld {} · projection bus {bus} · data dir {} · boot {} · leaderboard {}", args.celld_url, diff --git a/examples/game/server/src/mobpit/routes.rs b/examples/game/server/src/mobpit/routes.rs index 87b1b9626..8beeab242 100644 --- a/examples/game/server/src/mobpit/routes.rs +++ b/examples/game/server/src/mobpit/routes.rs @@ -5,30 +5,27 @@ //! artificial latency in front of every cell commit ("slow storage"). //! - `GET /dev/mobpit/stats` — boot id, sink counters, pending facts, recent //! commits (key, exact input, receipt). -//! - `GET /dev/mobpit/crawlers` — every Crawler cell on the roster (read -//! over celld's internal path). +//! - `GET /dev/mobpit/crawlers` — every crawler in the directory (the +//! `player_stats` projection of the cells' domain events; eventually +//! consistent). //! - `GET /dev/mobpit/leaderboard[?member=principal]` — the pushed top-N //! snapshot, plus rank / neighbours of `member` on both boards. //! - `POST /dev/mobpit/leaderboard/rebuild` — rebuild both boards from the -//! cells' event history; reports whether the result matches the cells. +//! event history of every crawler in the directory (read from its cell); +//! reports whether the result matches the cells. //! - `GET /dev/mobpit/presence` — who is online / away / offline where, //! projected from the runtime's player lifecycle facts. //! -//! [`internal_routes`] (always mounted): `POST /internal/celld-queue/relay`, -//! the celld outbox Queue consumer's hop, authenticated with -//! `DISTRIBUTED_INTERNAL_SECRET`; it publishes the Crawler domain event onto -//! the projection bus (503 until one is attached, so the Queue retries). +//! The always-mounted internal relay route is `crate::common::relay`. use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::http::StatusCode; use axum::routing::{get, post}; use axum::{Json, Router}; -use distributed::bus::{CelldQueueEnvelope, CELLD_QUEUE_RELAY_PATH}; -use distributed::cell_host::CELL_INTERNAL_SECRET_HEADER; use serde::Deserialize; use serde_json::{json, Value}; @@ -55,47 +52,6 @@ pub fn routes(world: Arc) -> Router { .with_state(world) } -/// Internal routes (see the module docs). -pub fn internal_routes(world: Arc) -> Router { - Router::new() - .route(CELLD_QUEUE_RELAY_PATH, post(relay)) - .with_state(world) -} - -async fn relay( - State(world): State>, - headers: HeaderMap, - Json(envelope): Json, -) -> (StatusCode, Json) { - let authorized = headers - .get(CELL_INTERNAL_SECRET_HEADER) - .and_then(|value| value.to_str().ok()) - .is_some_and(|candidate| world.celld().secret.matches(candidate)); - if !authorized { - return ( - StatusCode::UNAUTHORIZED, - Json(json!({ "code": "UNAUTHORIZED", "error": "unauthorized" })), - ); - } - let Some(bus) = world.projection_bus() else { - return ( - StatusCode::SERVICE_UNAVAILABLE, - Json(json!({ "code": "UNAVAILABLE", "error": "no projection bus attached" })), - ); - }; - match bus.relay_handler()(envelope).await { - Ok(()) => (StatusCode::ACCEPTED, Json(json!({ "ok": true }))), - Err(error) if error.is_retryable() => ( - StatusCode::SERVICE_UNAVAILABLE, - Json(json!({ "code": "UNAVAILABLE", "error": error.message() })), - ), - Err(error) => ( - StatusCode::BAD_REQUEST, - Json(json!({ "code": "BAD_REQUEST", "error": error.message() })), - ), - } -} - /// `POST /dev/mobpit/storage` body. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] diff --git a/examples/game/server/src/mobpit/store.rs b/examples/game/server/src/mobpit/store.rs index f13a056be..82f6689b6 100644 --- a/examples/game/server/src/mobpit/store.rs +++ b/examples/game/server/src/mobpit/store.rs @@ -11,38 +11,29 @@ //! the fact key, so a retry replays through the cell's command ledger //! instead of crediting twice; celld slow or down keeps facts in the //! zone's queue (`RetryPolicy::until_settled`), never in a local store. -//! - celld cannot enumerate cells, so before a principal's first kill -//! commits the sink enlists it in the `crawler-roster/all` cell -//! (`crawler.enlist`, idempotent by principal). Every committed Crawler -//! cell is therefore on the roster; [`CrawlerCells`] reads both over the -//! internal path (profiles, `/dev/mobpit/crawlers`, the board rebuild). //! - Each kill commit also writes the `crawler.mob_killed` domain event to //! the cell's outbox in the same cell transaction. celld drains it to the //! `mobpit-outbox` Queue; the worker's consumer relays every message to -//! this server's authenticated `/internal/celld-queue/relay`, which -//! publishes it onto the [`ProjectionBus`] (a SQLite bus read by -//! `mobpit-service`). Delivery is at-least-once with stable message ids; -//! the projector deduplicates. A relay that fails is retried by the Queue. +//! this server's authenticated `/internal/celld-queue/relay` +//! (`crate::common::relay`), which publishes it onto the projection bus. +//! `mobpit-service` and this server's [`super::directory`] project it +//! (the crawler list). [`CrawlerCells`] reads one cell by principal +//! (profiles, the board rebuild's kill histories). -use std::collections::{HashSet, VecDeque}; +use std::collections::VecDeque; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use distributed::bus::{celld_queue_relay_handler, CelldQueueRelayHandler, SqliteBus}; use distributed::cell_host::{InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER}; -use distributed::BusPublisher; use distributed_game::celld::{CelldFactRoute, CelldFactSink}; use distributed_game::lifecycle::Lifecycle; use distributed_game::{BoxFuture, Fact, FactError, FactReceipt, FactSink}; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; +use serde_json::Value; use super::board::Leaderboards; -use super::crawler::{ - KillReceipt, RecordKillInput, CRAWLER_KIND, ENLIST, PRESENCE_KIND, RECORD_KILL, ROSTER_KIND, - ROSTER_SHARD, -}; +use super::crawler::{KillReceipt, RecordKillInput, CRAWLER_KIND, PRESENCE_KIND, RECORD_KILL}; use super::presence::Presence; /// Command-ledger service id for game facts. @@ -97,7 +88,7 @@ pub struct CrawlerView { pub kills: Vec, } -/// Read side of the Crawler cells and the roster, over the internal path. +/// Read side of one Crawler or presence cell, over the internal path. #[derive(Clone)] pub struct CrawlerCells { client: reqwest::Client, @@ -142,15 +133,6 @@ impl CrawlerCells { Ok(body) } - /// Principals with a Crawler cell (sorted), from the roster. - pub async fn principals(&self) -> Result, String> { - let body = self.get(ROSTER_KIND, ROSTER_SHARD).await?; - let mut out: Vec = serde_json::from_value(body["principals"].clone()) - .map_err(|error| format!("roster: {error}"))?; - out.sort(); - Ok(out) - } - /// The durable presence entry of `principal` (`null` when none). pub async fn presence(&self, principal: &str) -> Result { Ok(self.get(PRESENCE_KIND, principal).await?["current"].take()) @@ -176,42 +158,6 @@ fn encode_segment(segment: &str) -> String { encoded } -/// The SQLite bus Crawler domain events are published to (consumed by -/// `mobpit-service`'s projector). -#[derive(Clone)] -pub struct ProjectionBus { - relay: CelldQueueRelayHandler, -} - -impl ProjectionBus { - /// Open (creating) the bus at a `sqlite:` URL shared with the service. - pub async fn open(url: &str) -> Result { - use std::str::FromStr; - let options = sqlx::sqlite::SqliteConnectOptions::from_str(url) - .map_err(|e| format!("projection bus {url}: {e}"))? - .create_if_missing(true) - .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) - .busy_timeout(Duration::from_secs(10)); - let pool = sqlx::sqlite::SqlitePoolOptions::new() - .max_connections(2) - .connect_with(options) - .await - .map_err(|e| format!("projection bus {url}: {e}"))?; - let bus = SqliteBus::new(pool); - bus.ensure_tables() - .await - .map_err(|e| format!("projection bus tables: {e}"))?; - Ok(Self { - relay: celld_queue_relay_handler(BusPublisher::new(Arc::new(bus))), - }) - } - - /// The Queue relay handler publishing onto this bus. - pub fn relay_handler(&self) -> &CelldQueueRelayHandler { - &self.relay - } -} - /// A committed kill as the dev routes report it. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -292,8 +238,6 @@ pub struct MobPitFactSink { state: Arc, board: Arc, presence: Arc, - /// Principals enlisted on the roster by this process. - enlisted: Arc>>, } impl MobPitFactSink { @@ -314,49 +258,25 @@ impl MobPitFactSink { .route(CelldFactRoute::by_principal::( CRAWLER_KIND, )) - .route(CelldFactRoute::new(ENLIST, ROSTER_KIND, |_, _| { - Some(ROSTER_SHARD.to_string()) - })) .lifecycle_route(PRESENCE_KIND); Ok(Self { celld: sink, state, board, presence, - enlisted: Arc::default(), }) } - /// Enlist `principal` on the roster before its first kill commits (once - /// per process; the ledger makes a repeat a replay). - async fn enlist(&self, principal: &str, fact: &Fact) -> Result<(), FactError> { - if self.enlisted.lock().expect("enlisted").contains(principal) { - return Ok(()); - } - let enlist = Fact::trusted( - format!("{ENLIST}:{principal}"), - ENLIST, - json!({}), - Some(principal.to_string()), - fact.stamp().clone(), - ); - self.celld.commit(&enlist).await?; - self.enlisted - .lock() - .expect("enlisted") - .insert(principal.to_string()); - Ok(()) - } - async fn commit_kill(&self, fact: &Fact) -> Result { let latency = self.state.latency(); if !latency.is_zero() { tokio::time::sleep(latency).await; } - let principal = fact - .principal() - .ok_or_else(|| FactError::Permanent("kill facts need a player principal".into()))?; - self.enlist(principal, fact).await?; + if fact.principal().is_none() { + return Err(FactError::Permanent( + "kill facts need a player principal".into(), + )); + } let receipt = self.celld.commit(fact).await?; if fact.command() == RECORD_KILL { if let Ok(kill) = serde_json::from_value::(receipt.payload.clone()) { diff --git a/examples/game/server/src/mobpit/world.rs b/examples/game/server/src/mobpit/world.rs index 22fa5d717..78f561a6a 100644 --- a/examples/game/server/src/mobpit/world.rs +++ b/examples/game/server/src/mobpit/world.rs @@ -1,10 +1,11 @@ //! Process-wide Mob Pit state shared by the zones, the fact worker and the //! dev routes: the celld cells (fact sink and reader), the sink's -//! latency/counters, the leaderboards and the projection bus. +//! latency/counters, the leaderboards, the projection bus and the crawler +//! directory projected from it. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use distributed_game::net::{GameServer, ServerConfig, ZoneFactory}; @@ -12,10 +13,12 @@ use distributed_game::RetryPolicy; use serde_json::{json, Value}; use super::board::Leaderboards; +use super::directory::CrawlerDirectory; use super::presence::Presence; -use super::store::{CelldOptions, CrawlerCells, MobPitFactSink, ProjectionBus, SinkState}; +use super::store::{CelldOptions, CrawlerCells, MobPitFactSink, SinkState}; use super::zone::{MobPit, MobPitConfig}; use super::{routes, MobPitDemo}; +use crate::common::relay::{relay_routes, ProjectionBus}; use crate::common::{random_hex, Demo, DemoRoutes, Guests}; /// Unsettled facts one zone may hold before `emit_fact` reports backpressure. @@ -33,6 +36,12 @@ pub struct WorldOptions { pub redis_url: Option, /// Redis key prefix (tests isolate themselves with it). pub redis_prefix: String, + /// SQLite bus the relayed domain events are published onto (shared + /// with `mobpit-service`). Default: `sqlite:{data_dir}/bus.db`. + pub projection_bus: Option, + /// SQLite database of this server's read models (the crawler + /// directory). Default: `sqlite:{data_dir}/read-models.db`. + pub read_models: Option, } impl WorldOptions { @@ -43,9 +52,26 @@ impl WorldOptions { celld, redis_url: None, redis_prefix: "mobpit".into(), + projection_bus: None, + read_models: None, } } + fn bus_url(&self) -> String { + self.projection_bus + .clone() + .unwrap_or_else(|| format!("sqlite:{}", self.data_dir.join("bus.db").display())) + } + + fn read_models_url(&self) -> String { + self.read_models.clone().unwrap_or_else(|| { + format!( + "sqlite:{}?mode=rwc", + self.data_dir.join("read-models.db").display() + ) + }) + } + /// A fresh local directory under the system temp dir. pub fn temporary(celld: CelldOptions) -> Self { Self::new( @@ -69,14 +95,27 @@ pub struct MobPitWorld { board: Arc, zone_pending: AtomicUsize, profiles: Arc>>, - projection: Arc>, + projection: Arc>, + directory: OnceLock, presence: Arc, + bus_url: String, + read_models_url: String, } impl MobPitWorld { - /// Open the world over celld. Must be called inside a Tokio runtime. - /// Does not contact celld (cells are reached on first use) and does not - /// rebuild the board: call [`Self::rebuild_board`]. + /// Open the world over celld and attach its projection bus and crawler + /// directory ([`Self::attach_projection`]). Does not rebuild the board: + /// call [`Self::rebuild_board`]. + pub async fn start(options: WorldOptions) -> Result, String> { + let world = Self::open(options)?; + world.attach_projection().await?; + Ok(world) + } + + /// Open the world over celld without its projection (the relay answers + /// 503 and the crawler list is unavailable until + /// [`Self::attach_projection`]). Must be called inside a Tokio runtime. + /// Does not contact celld (cells are reached on first use). pub fn open(options: WorldOptions) -> Result, String> { std::fs::create_dir_all(&options.data_dir) .map_err(|e| format!("create {}: {e}", options.data_dir.display()))?; @@ -89,7 +128,12 @@ impl MobPitWorld { board.clone(), presence.clone(), )?; + let bus_url = options.bus_url(); + let read_models_url = options.read_models_url(); Ok(Arc::new(Self { + bus_url, + read_models_url, + directory: OnceLock::new(), boot: random_hex(8), data_dir: options.data_dir, cells: Arc::new(CrawlerCells::new(&options.celld)?), @@ -145,15 +189,19 @@ impl MobPitWorld { self.fact_sink.clone() } - /// Publish the Crawler domain events that celld's outbox Queue relays - /// to `/internal/celld-queue/relay` onto the SQLite bus at `url` (the - /// read-model service consumes it). Until a bus is attached the relay - /// answers 503 and the Queue keeps the messages. Call once. - pub async fn attach_projection_bus(&self, url: &str) -> Result<(), String> { - let bus = ProjectionBus::open(url).await?; + /// Open the projection bus the outbox Queue relay publishes onto + /// (`/internal/celld-queue/relay`; 503 until this runs, so the Queue + /// keeps the messages) and start projecting it into the crawler + /// directory. Call once. + pub async fn attach_projection(&self) -> Result<(), String> { + let bus = ProjectionBus::open(&self.bus_url).await?; + let directory = CrawlerDirectory::open(&self.read_models_url, bus.pool().clone()).await?; + self.directory + .set(directory) + .map_err(|_| "projection already attached".to_string())?; self.projection .set(bus) - .map_err(|_| "projection bus already attached".to_string()) + .map_err(|_| "projection already attached".to_string()) } /// The attached projection bus, if any. @@ -161,6 +209,23 @@ impl MobPitWorld { self.projection.get() } + /// The slot the relay route publishes through (empty until + /// [`Self::attach_projection`]). + pub fn projection(&self) -> Arc> { + self.projection.clone() + } + + /// The bus URL (`--projection-bus`, default in the data dir). + pub fn bus_url(&self) -> &str { + &self.bus_url + } + + fn directory(&self) -> Result<&CrawlerDirectory, String> { + self.directory + .get() + .ok_or_else(|| "no projection attached: the crawler list is unavailable".into()) + } + pub(crate) fn set_zone_pending(&self, n: usize) { self.zone_pending.store(n, Ordering::Relaxed); } @@ -198,23 +263,34 @@ impl MobPitWorld { std::mem::take(&mut *self.profiles.lock().expect("profiles")) } - /// Rebuild the leaderboards from the cells' event history. + /// Rebuild the leaderboards from the event history of every crawler in + /// the directory (read from each one's cell). pub async fn rebuild_board(&self) -> Result { - self.board.rebuild(&self.cells).await + let principals = self + .directory()? + .list() + .await? + .into_iter() + .map(|row| row.player_id) + .collect(); + self.board.rebuild(principals, &self.cells).await } - /// Every crawler on the roster: `{principal, name, xp, kills, level, version}`. + /// Every crawler in the directory: `{principal, name, xp, kills, level}` + /// (eventually consistent with the cells). pub async fn crawlers(&self) -> Result, String> { - let mut out = Vec::new(); - for principal in self.cells.principals().await? { - if let Some(c) = self.cells.crawler(&principal).await?.state { - out.push(json!({ - "principal": principal, "name": c.name, "xp": c.xp, - "kills": c.kills, "level": c.level, "version": c.version, - })); - } - } - Ok(out) + Ok(self + .directory()? + .list() + .await? + .into_iter() + .map(|c| { + json!({ + "principal": c.player_id, "name": c.name, "xp": c.xp, + "kills": c.kills, "level": c.level, + }) + }) + .collect()) } /// Game server for `config` plus the Mob Pit dev and internal routes. @@ -249,7 +325,7 @@ impl MobPitWorld { server, DemoRoutes { dev: routes::routes(self.clone()), - internal: routes::internal_routes(self.clone()), + internal: relay_routes(self.celld.secret.clone(), self.projection.clone()), }, ) } diff --git a/examples/game/server/tests/mobpit_smoke.rs b/examples/game/server/tests/mobpit_smoke.rs index a87af1594..e7bbdec55 100644 --- a/examples/game/server/tests/mobpit_smoke.rs +++ b/examples/game/server/tests/mobpit_smoke.rs @@ -1,6 +1,9 @@ //! Mob Pit + Leaderboard smoke tests, every one against its own `celld dev` //! serving `examples/game/cells` (`game_demos::devcelld`; `CELLD_BIN` must //! name celld and the worker must be built — `run.sh smoke` does both). +//! celld relays its outbox Queue to the test's server, whose crawler +//! directory projects the relayed events (the crawler list is eventually +//! consistent with the cells). //! //! 1. Facts → celld Crawler cells exactly once (ledger replay), server-side //! XP, trusted principal; leaderboard occurrences have the same ids live @@ -47,8 +50,45 @@ impl Behaviour for Spin { } } -fn celld() -> DevCelld { - DevCelld::start(None).unwrap_or_else(|error| panic!("celld dev: {error}")) +/// A listener for the game server and a celld whose outbox Queue relays +/// to it. +async fn relayed_celld() -> (tokio::net::TcpListener, DevCelld) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let relay = format!( + "http://{}{}", + listener.local_addr().unwrap(), + distributed::bus::CELLD_QUEUE_RELAY_PATH + ); + let celld = DevCelld::start(Some(relay)).unwrap_or_else(|error| panic!("celld dev: {error}")); + (listener, celld) +} + +/// The projected crawler list has caught up: `principals`, with their +/// cells' kills. +async fn crawlers_settled(world: &MobPitWorld, principals: &[&str]) { + eventually("crawler directory", Duration::from_secs(30), || async { + let Ok(crawlers) = world.crawlers().await else { + return false; + }; + let listed: Vec<&str> = crawlers + .iter() + .map(|c| c["principal"].as_str().unwrap_or_default()) + .collect(); + if listed != principals { + return false; + } + for c in &crawlers { + let principal = c["principal"].as_str().unwrap(); + let Ok(view) = world.cells().crawler(principal).await else { + return false; + }; + if view.state.map(|s| u64::from(s.kills)) != c["kills"].as_u64() { + return false; + } + } + true + }) + .await; } fn temp_dir(tag: &str) -> PathBuf { @@ -111,8 +151,11 @@ async fn assert_board_matches_cells(world: &MobPitWorld) -> u64 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn facts_commit_once_and_board_rebuilds_identically() { let dir = temp_dir("unit"); - let mut celld = celld(); - let world = MobPitWorld::open(WorldOptions::new(&dir, celld.options())).unwrap(); + let (listener, mut celld) = relayed_celld().await; + let world = MobPitWorld::start(WorldOptions::new(&dir, celld.options())) + .await + .unwrap(); + let (_demo, _stop) = start(listener, world.clone()).await; let sink = world.fact_sink(); let input = json!({ "mob": "pit-1/m3", "tier": 2, "name": "Ana" }); @@ -178,11 +221,8 @@ async fn facts_commit_once_and_board_rebuilds_identically() { ana.kills[0].sequence, first.payload["sequence"].as_u64().unwrap() ); - assert_eq!( - world.cells().principals().await.unwrap(), - vec!["guest-ana", "guest-bo"], - "enlisted on the roster before the first kill" - ); + // Listed by the directory once their kill events are projected. + crawlers_settled(&world, &["guest-ana", "guest-bo"]).await; // Live updates are fire-and-forget; the replay was deduplicated. eventually("live board", Duration::from_secs(5), || async { @@ -253,9 +293,9 @@ async fn facts_commit_once_and_board_rebuilds_identically() { } async fn start( + listener: tokio::net::TcpListener, world: Arc, ) -> (RunningDemo, tokio::sync::oneshot::Sender<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); let running = mobpit::spawn(listener, DemoOptions::dev(SECRET), world, async move { let _ = stopped.await; @@ -283,9 +323,11 @@ fn spin(i: usize) -> Box { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn bots_fight_slow_storage_keeps_the_tick_flat_and_boards_match_cells() { let dir = temp_dir("bots"); - let celld = celld(); - let world = MobPitWorld::open(WorldOptions::new(&dir, celld.options())).unwrap(); - let (demo, stop) = start(world.clone()).await; + let (listener, celld) = relayed_celld().await; + let world = MobPitWorld::start(WorldOptions::new(&dir, celld.options())) + .await + .unwrap(); + let (demo, stop) = start(listener, world.clone()).await; // Unknown fields (e.g. an identity) are rejected on the dev routes too. let (status, _) = http( @@ -362,7 +404,20 @@ async fn bots_fight_slow_storage_keeps_the_tick_flat_and_boards_match_cells() { assert!(stats.committed >= 3, "bots killed mobs: {stats:?}"); assert_eq!(stats.errors, 0, "{stats:?}"); - // Boards = cells (fire-and-forget updates have drained), rebuild = same. + // Boards = cells (fire-and-forget updates have drained, the directory + // has projected every kill), rebuild = same. + let w = world.clone(); + eventually("directory caught up", Duration::from_secs(30), || { + let w = w.clone(); + async move { + let mut total = 0; + for c in w.crawlers().await.unwrap() { + total += c["kills"].as_u64().unwrap(); + } + total == w.sink_state().stats().committed + } + }) + .await; let w = world.clone(); eventually("board caught up", Duration::from_secs(10), || { let w = w.clone(); @@ -419,17 +474,10 @@ struct Binary { addr: SocketAddr, } -async fn start_binary(dir: &Path, celld: &DevCelld) -> Binary { +async fn start_binary(dir: &Path, celld: &DevCelld, port: u16) -> Binary { let mut child = Command::new(env!("CARGO_BIN_EXE_game-demos")) - .args([ - "mobpit", - "--port", - "0", - "--dev", - "--secret", - SECRET, - "--data-dir", - ]) + .args(["mobpit", "--port", &port.to_string(), "--dev"]) + .args(["--secret", SECRET, "--data-dir"]) .arg(dir) .args(["--celld-url", &celld.url()]) .env("DISTRIBUTED_INTERNAL_SECRET", celld.secret()) @@ -459,8 +507,15 @@ async fn start_binary(dir: &Path, celld: &DevCelld) -> Binary { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { let dir = temp_dir("crash"); - let mut celld = celld(); - let mut first = start_binary(&dir, &celld).await; + // celld relays its outbox Queue to the server's fixed port (both runs). + let port = game_demos::devcelld::free_port(); + let relay = format!( + "http://127.0.0.1:{port}{}", + distributed::bus::CELLD_QUEUE_RELAY_PATH + ); + let mut celld = + DevCelld::start(Some(relay)).unwrap_or_else(|error| panic!("celld dev: {error}")); + let mut first = start_binary(&dir, &celld, port).await; let ws = format!("ws://{}", first.addr); // Settle some kills at full speed, then slow storage so facts are in // flight at the moment of the crash. @@ -507,14 +562,34 @@ async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { // facts are all there; of the in-flight ones each either committed // atomically or not at all. celld.restart().unwrap(); - let mut second = start_binary(&dir, &celld).await; - let (_, crawlers) = http(second.addr, "GET", "/dev/mobpit/crawlers", None).await; - let total: u64 = crawlers["crawlers"] - .as_array() - .unwrap() - .iter() - .map(|c| c["kills"].as_u64().unwrap()) - .sum(); + let mut second = start_binary(&dir, &celld, port).await; + // The directory (persisted in the data dir) catches up with the kill + // events the Queue relays; then it is stable. + let addr = second.addr; + let directory_total = || async move { + let (_, crawlers) = http(addr, "GET", "/dev/mobpit/crawlers", None).await; + crawlers["crawlers"] + .as_array() + .unwrap() + .iter() + .map(|c| c["kills"].as_u64().unwrap()) + .sum::() + }; + let mut total = 0; + let mut stable_since = Instant::now(); + let start = Instant::now(); + while stable_since.elapsed() < Duration::from_secs(3) || total < settled_before { + assert!( + start.elapsed() < Duration::from_secs(60), + "directory never settled: {total} kills, {settled_before} settled before the crash" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + let now = directory_total().await; + if now != total { + total = now; + stable_since = Instant::now(); + } + } eprintln!( "crash: settled {settled_before}, in flight {in_flight}, durable after restart {total}" ); @@ -522,7 +597,10 @@ async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { total >= settled_before && total <= settled_before + in_flight, "settled {settled_before} ≤ durable {total} ≤ settled + in flight ({in_flight})" ); - // The board was rebuilt from the cells at boot. + // The board is rebuilt from the cells of every crawler in the directory + // (at boot, and again now that the directory has caught up). + let (_, rebuild) = http(second.addr, "POST", "/dev/mobpit/leaderboard/rebuild", None).await; + assert_eq!(rebuild["matchesCells"], true, "{rebuild}"); let (_, board) = http(second.addr, "GET", "/dev/mobpit/leaderboard", None).await; let board_kills: f64 = board["snapshot"]["kills"] .as_array() @@ -571,11 +649,12 @@ async fn redis_board_projects_and_rebuilds_when_redis_url_is_set() { return; }; let dir = temp_dir("redis"); - let celld = celld(); + let (listener, celld) = relayed_celld().await; let mut options = WorldOptions::new(&dir, celld.options()); options.redis_url = Some(url); options.redis_prefix = format!("mobpit-test-{}", random_hex(8)); - let world = MobPitWorld::open(options).unwrap(); + let world = MobPitWorld::start(options).await.unwrap(); + let (_demo, _stop) = start(listener, world.clone()).await; assert_eq!(world.board().backend(), "redis"); let sink = world.fact_sink(); for (i, tier) in [1u8, 3].into_iter().enumerate() { @@ -588,6 +667,7 @@ async fn redis_board_projects_and_rebuilds_when_redis_url_is_set() { world.board().scores("guest-rae").await.unwrap() == (2.0, 70.0) }) .await; + crawlers_settled(&world, &["guest-rae"]).await; let report = world.rebuild_board().await.unwrap(); assert!(report.matches_cells, "{report:?}"); assert!(report.identical_top, "{report:?}"); @@ -607,9 +687,11 @@ async fn presence_is_settled_in_presence_cells_from_the_lifecycle_facts() { use tokio_tungstenite::tungstenite::Message; let dir = temp_dir("presence"); - let celld = celld(); - let world = MobPitWorld::open(WorldOptions::new(&dir, celld.options())).unwrap(); - let (demo, stop) = start(world.clone()).await; + let (listener, celld) = relayed_celld().await; + let world = MobPitWorld::start(WorldOptions::new(&dir, celld.options())) + .await + .unwrap(); + let (demo, stop) = start(listener, world.clone()).await; let join = |principal: &'static str| { let url = format!("{}/zones/pit-1", demo.ws_url()); async move { @@ -698,7 +780,7 @@ async fn presence_is_settled_in_presence_cells_from_the_lifecycle_facts() { let bo = world.cells().presence("guest-bo").await.unwrap(); assert_eq!(bo["status"], "away", "{bo}"); // Lifecycle facts never touch the Crawler cells. - assert!(world.cells().principals().await.unwrap().is_empty()); + assert!(world.crawlers().await.unwrap().is_empty()); let _ = stop.send(()); let _ = std::fs::remove_dir_all(dir); } From 20d1421af05e6780db4c69405579e83953b67490 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 14:37:30 -0500 Subject: [PATCH 19/40] feat(game)!: zone status read model for lease and snapshot events Zone lease events had no consumer (the Queue retried them into the DLQ) and the grid map opened a `ZoneState` cell per zone, creating cells that were never saved (celld materialises a Durable Object's storage on any request, so a cell read cannot answer "not found" without it). - `ZoneLease` and `ZoneState` publish compact post-transition state: `ZoneLeaseStatus` on `zone.activated|released|lease_expired`, `ZoneSnapshotStatus` (no bytes) on `zone_state.saved|deleted`. - New `world::zone_status`: `zone_leases` / `zone_snapshots` read models, their projections, `projectors()` and `ZoneStatus::of` (active / hibernated / cold). Tested end to end on the reference host (outbox -> SQLite bus -> consumer service -> rows). - Cluster hosts serve the celld Queue relay, share a projection bus in `--data-dir` and each project it into their own zone status database. - `/dev/grid/map` reads the read model; only live tier/player counts are asked from the hosts. `spawn_cluster` is now async. Refs [[tasks/gr-p37-celld-zone-state]] [[tasks/gr-p36-mobpit-3d-services]] --- distributed_game/Cargo.toml | 9 + distributed_game/README.md | 5 +- distributed_game/src/world/mod.rs | 7 + .../save_zone_lease.mutation.graphql | 5 + .../save_zone_snapshot.mutation.graphql | 5 + distributed_game/src/world/zone_lease.rs | 65 ++++- distributed_game/src/world/zone_state.rs | 81 +++++- distributed_game/src/world/zone_status.rs | 230 +++++++++++++++++ distributed_game/tests/zone_status/main.rs | 232 ++++++++++++++++++ examples/game/README.md | 16 +- examples/game/client/tests/helpers/celld.mjs | 2 +- .../game/client/tests/helpers/cluster.mjs | 16 +- examples/game/run.sh | 6 +- examples/game/server/src/cluster.rs | 57 ++++- examples/game/server/src/common/mod.rs | 1 + .../game/server/src/common/zone_status.rs | 151 ++++++++++++ examples/game/server/src/grid/dev.rs | 109 ++++---- examples/game/server/tests/grid_smoke.rs | 17 +- examples/game/server/tests/support/mod.rs | 16 +- 19 files changed, 927 insertions(+), 103 deletions(-) create mode 100644 distributed_game/src/world/mutations/save_zone_lease.mutation.graphql create mode 100644 distributed_game/src/world/mutations/save_zone_snapshot.mutation.graphql create mode 100644 distributed_game/src/world/zone_status.rs create mode 100644 distributed_game/tests/zone_status/main.rs create mode 100644 examples/game/server/src/common/zone_status.rs diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index c2f118f32..c23108bdf 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -49,6 +49,10 @@ worker = { version = "0.8", optional = true } [dev-dependencies] base64 = "0.23" +# `tests/zone_status`: the read model projected by a consumer service over +# a SQLite bus and read-model store. +distributed = { path = "..", default-features = false, features = ["sqlite", "graphql"] } +sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "sqlite"] } trybuild = "=1.0.120" tokio = { version = "1", features = ["full", "test-util"] } tokio-tungstenite = "0.30" @@ -85,6 +89,11 @@ name = "zone_state" path = "tests/zone_state/main.rs" required-features = ["cell"] +[[test]] +name = "zone_status" +path = "tests/zone_status/main.rs" +required-features = ["cell"] + [[test]] name = "celld_zone_lease" path = "tests/celld_zone_lease/main.rs" diff --git a/distributed_game/README.md b/distributed_game/README.md index 1bf09895e..b92b64f16 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -1105,8 +1105,9 @@ principals commit concurrently; settlements arrive in completion order. | Piece | Purpose | |---|---| | `ZoneDirectory` (`StaticDirectory`, `HashRing`) | zone → owning `HostAddr`. `HashRing` uses virtual nodes; adding/removing one of N hosts moves ~1/N of the zones. A Hello for a zone owned elsewhere gets `Redirect(zone, url)`; `url` is the owner's `HostAddr::ws_url` (advertise a proxied public URL with `HostAddr::with_ws_url`) or `ServerConfig::with_redirect_url(\|host, zone\| …)`. | -| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the cell's alarm publishes `zone.lease_expired { zone_id, holder, token }` when a holder stops renewing, so services see a crashed zone server. `world::ownership_conformance` is the behaviour every implementation passes. | -| `ZoneStateStore` (`celld::CellZoneStateStore`; `MemoryZoneStateStore` for unit tests) | hibernation snapshots (async `load` / `save` / `delete`), written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). On celld each zone's snapshot is its own `ZoneState:{zone}` cell (`world::zone_state`): inline up to 64 KiB, larger ones in the worker's R2 bucket with the digest in the cell. `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | +| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the cell's alarm publishes `zone.lease_expired` when a holder stops renewing, so services see a crashed zone server; each event carries the lease's `ZoneLeaseStatus { zone_id, holder, token, status }`. `world::ownership_conformance` is the behaviour every implementation passes. | +| `ZoneStateStore` (`celld::CellZoneStateStore`; `MemoryZoneStateStore` for unit tests) | hibernation snapshots (async `load` / `save` / `delete`), written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). On celld each zone's snapshot is its own `ZoneState:{zone}` cell (`world::zone_state`): inline up to 64 KiB, larger ones in the worker's R2 bucket with the digest in the cell; saves and deletes publish `zone_state.saved` / `zone_state.deleted` with a `ZoneSnapshotStatus { zone_id, token, stored, content_version, bytes }` (no bytes). `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | +| `world::zone_status` (feature `cell`) | the read model of those events: `zone_leases` and `zone_snapshots` rows (`projection!` upserts of the state bodies), `projectors()` to mount on any consumer service and read-model store, and `ZoneStatus::of` → `Active { holder }` / `Hibernated { bytes }` / `Cold`. For views (world maps, minimaps): it answers "which zones are active or asleep?" without opening a cell per zone (on celld a read of a never-used cell creates it). Eventually consistent; ownership and restore always go to the cells. | | `ZoneTopology` (`DottedTopology` default: `eldoria.port.docks` → `eldoria.port` → `eldoria`) | parent links for tier-up events. | **Per-zone host settings.** `GameServer::with_host_config(|zone_id, host| diff --git a/distributed_game/src/world/mod.rs b/distributed_game/src/world/mod.rs index 839e8d22c..2183010e6 100644 --- a/distributed_game/src/world/mod.rs +++ b/distributed_game/src/world/mod.rs @@ -14,6 +14,10 @@ //! - [`allocation`]: [`ZoneAllocation`] records written by the service that //! allocates a zone, read before the zone is created //! ([`ZoneActivation`], [`ZoneAllocationStore`]). +//! - `zone_status` (feature `cell`): the read model of every zone's lease +//! and hibernation slot, projected from the `zone_lease` / `zone_state` +//! cells' outward events (active / hibernated / cold without opening a +//! cell). //! - [`barrier`]: content-version policy types shared by the host and the //! world barrier ([`StalePolicy`], [`BarrierMode`]). @@ -30,6 +34,9 @@ pub mod zone_lease; #[cfg(feature = "cell")] #[allow(missing_docs)] pub mod zone_state; +#[cfg(feature = "cell")] +#[allow(missing_docs)] +pub mod zone_status; pub use crate::facts::BoxFuture; pub use allocation::{ diff --git a/distributed_game/src/world/mutations/save_zone_lease.mutation.graphql b/distributed_game/src/world/mutations/save_zone_lease.mutation.graphql new file mode 100644 index 000000000..a09fb8d2b --- /dev/null +++ b/distributed_game/src/world/mutations/save_zone_lease.mutation.graphql @@ -0,0 +1,5 @@ +# Syntax-only read-model mutation → MutationProgram IR (not a public field). +# The row is the lease's post-transition `ZoneLeaseStatus`. +mutation SaveZoneLease { + upsert_zone_leases(object: $input.lease) +} diff --git a/distributed_game/src/world/mutations/save_zone_snapshot.mutation.graphql b/distributed_game/src/world/mutations/save_zone_snapshot.mutation.graphql new file mode 100644 index 000000000..3683b586d --- /dev/null +++ b/distributed_game/src/world/mutations/save_zone_snapshot.mutation.graphql @@ -0,0 +1,5 @@ +# Syntax-only read-model mutation → MutationProgram IR (not a public field). +# The row is the hibernation slot's post-command `ZoneSnapshotStatus`. +mutation SaveZoneSnapshot { + upsert_zone_snapshots(object: $input.snapshot) +} diff --git a/distributed_game/src/world/zone_lease.rs b/distributed_game/src/world/zone_lease.rs index e7357e67c..1d7d767a6 100644 --- a/distributed_game/src/world/zone_lease.rs +++ b/distributed_game/src/world/zone_lease.rs @@ -11,6 +11,10 @@ //! | `zone_lease.release` | no holder, alarm cancelled | `zone.released` | //! | `zone_lease.expire` (alarm only) | no holder, if still expired | `zone.lease_expired` | //! +//! Outward events carry the lease's post-transition [`ZoneLeaseStatus`] +//! (zone, holder, token, `active` / `released` / `expired`); the +//! [`zone_status`](super::zone_status) read model projects them. +//! //! Fencing tokens increase on every acquisition and are never reset. Expiry //! is declared by the cell's alarm ([`distributed::cell_host::CellAlarm`]): //! the alarm handler re-reads the lease and expires it only when the @@ -50,6 +54,13 @@ pub const LOST: &str = "lost"; /// The expiry alarm's name. pub const EXPIRY_ALARM: &str = "expiry"; +/// [`ZoneLeaseStatus::status`] of a held lease. +pub const STATUS_ACTIVE: &str = "active"; +/// [`ZoneLeaseStatus::status`] after `zone.released`. +pub const STATUS_RELEASED: &str = "released"; +/// [`ZoneLeaseStatus::status`] after `zone.lease_expired`. +pub const STATUS_EXPIRED: &str = "expired"; + /// The lease of one zone. #[derive(Clone, Debug, Default, Serialize, Deserialize, distributed::Snapshot)] pub struct ZoneLease { @@ -57,18 +68,62 @@ pub struct ZoneLease { holder: Option, token: u64, expires_at_ms: u64, + /// Holder of the last claim (kept after release and expiry). + #[serde(default)] + last_holder: String, + /// How the last claim ended (`released` / `expired`; empty while held + /// or never claimed). + #[serde(default)] + ended: String, +} + +/// The public state of a lease carried by its outward events (versioned +/// independently of the replay events): one `zone_leases` read-model row. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, distributed::DomainState)] +#[domain_state(version = 1)] +pub struct ZoneLeaseStatus { + /// Zone (the cell's stream id). + pub zone_id: String, + /// Holder of the current or, once ended, the last claim. + pub holder: String, + /// Fencing token of that claim. + pub token: u64, + /// [`STATUS_ACTIVE`], [`STATUS_RELEASED`] or [`STATUS_EXPIRED`]. + pub status: String, +} + +impl From<&ZoneLease> for ZoneLeaseStatus { + fn from(lease: &ZoneLease) -> Self { + Self { + zone_id: lease.entity.id().to_string(), + holder: lease.last_holder.clone(), + token: lease.token, + status: if lease.holder.is_some() { + STATUS_ACTIVE.to_string() + } else { + lease.ended.clone() + }, + } + } } -#[sourced(entity, aggregate_type = "ZoneLease", events = "ZoneLeaseEvent")] +#[sourced( + entity, + aggregate_type = "ZoneLease", + events = "ZoneLeaseEvent", + domain_state = ZoneLeaseStatus +)] impl ZoneLease { - #[event("zone.activated", version = 1, domain = event)] + #[event("zone.activated", version = 1, domain)] fn activated(&mut self, zone_id: String, holder: String, token: u64, expires_at_ms: u64) { if self.entity.id().is_empty() { self.entity.set_id(zone_id); } + self.last_holder = holder.clone(); self.holder = Some(holder); self.token = token; self.expires_at_ms = expires_at_ms; + self.ended.clear(); } #[event("zone_lease.renewed", version = 1)] @@ -76,16 +131,18 @@ impl ZoneLease { self.expires_at_ms = expires_at_ms; } - #[event("zone.released", version = 1, domain = event)] + #[event("zone.released", version = 1, domain)] fn released(&mut self, zone_id: String, holder: String, token: u64) { let _ = (zone_id, holder, token); self.holder = None; + self.ended = STATUS_RELEASED.to_string(); } - #[event("zone.lease_expired", version = 1, domain = event)] + #[event("zone.lease_expired", version = 1, domain)] fn lease_expired(&mut self, zone_id: String, holder: String, token: u64) { let _ = (zone_id, holder, token); self.holder = None; + self.ended = STATUS_EXPIRED.to_string(); } } diff --git a/distributed_game/src/world/zone_state.rs b/distributed_game/src/world/zone_state.rs index aa6e93ed2..0c705a383 100644 --- a/distributed_game/src/world/zone_state.rs +++ b/distributed_game/src/world/zone_state.rs @@ -27,6 +27,12 @@ //! Every save is one small event (inline bytes or a blob reference); cells //! load from the aggregate snapshot, which is refreshed on every commit. //! +//! Both commands publish the post-command [`ZoneSnapshotStatus`] (zone, +//! token, whether a snapshot is stored, its content version and size, never +//! the bytes) as `zone_state.saved` / `zone_state.deleted` domain events; +//! the [`zone_status`](super::zone_status) read model projects them, so +//! "is this zone asleep with state?" is answered without opening its cell. +//! //! Callers must hold the `zone-host` role (an internal identity on the //! authenticated internal path). @@ -90,9 +96,52 @@ pub struct ZoneState { bytes: Option, } -#[sourced(entity, aggregate_type = "ZoneState", events = "ZoneStateEvent")] +/// The public state of a zone's hibernation slot carried by its outward +/// events: one `zone_snapshots` read-model row (no snapshot bytes). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, distributed::DomainState)] +#[domain_state(version = 1)] +pub struct ZoneSnapshotStatus { + /// Zone (the cell's stream id). + pub zone_id: String, + /// Highest token seen. + pub token: u64, + /// A snapshot is stored. + pub stored: bool, + /// Content version of the stored snapshot (empty when none). + pub content_version: String, + /// Size of the stored snapshot in bytes (0 when none). + pub bytes: u64, +} + +impl From<&ZoneState> for ZoneSnapshotStatus { + fn from(state: &ZoneState) -> Self { + let bytes = match &state.bytes { + Some(SnapshotBytes::Inline(bytes)) => bytes.len() as u64, + Some(SnapshotBytes::Blob(blob)) => blob.len, + None => 0, + }; + Self { + zone_id: state.entity.id().to_string(), + token: state.token, + stored: state.bytes.is_some(), + content_version: if state.bytes.is_some() { + state.content_version.clone() + } else { + String::new() + }, + bytes, + } + } +} + +#[sourced( + entity, + aggregate_type = "ZoneState", + events = "ZoneStateEvent", + domain_state = ZoneSnapshotStatus +)] impl ZoneState { - #[event("zone_state.saved", version = 1)] + #[event("zone_state.saved", version = 1, domain)] fn saved( &mut self, zone_id: String, @@ -108,7 +157,7 @@ impl ZoneState { self.bytes = Some(bytes); } - #[event("zone_state.deleted", version = 1)] + #[event("zone_state.deleted", version = 1, domain)] fn deleted(&mut self, zone_id: String, token: u64) { if self.entity.id().is_empty() { self.entity.set_id(zone_id); @@ -236,11 +285,14 @@ async fn handle_save( bytes, ) .map_err(rejected)?; - ctx.repo().commit(state)?.succeeded(StatePayload { - zone_id: input.zone_id, - token: input.token, - superseded_blob: superseded, - }) + ctx.repo() + .publish_events() + .commit(state)? + .succeeded(StatePayload { + zone_id: input.zone_id, + token: input.token, + superseded_blob: superseded, + }) } async fn handle_delete( @@ -256,11 +308,14 @@ async fn handle_delete( state .deleted(input.zone_id.clone(), input.token) .map_err(rejected)?; - ctx.repo().commit(state)?.succeeded(StatePayload { - zone_id: input.zone_id, - token: input.token, - superseded_blob: superseded, - }) + ctx.repo() + .publish_events() + .commit(state)? + .succeeded(StatePayload { + zone_id: input.zone_id, + token: input.token, + superseded_blob: superseded, + }) } struct Save; diff --git a/distributed_game/src/world/zone_status.rs b/distributed_game/src/world/zone_status.rs new file mode 100644 index 000000000..2d029ec22 --- /dev/null +++ b/distributed_game/src/world/zone_status.rs @@ -0,0 +1,230 @@ +//! Zone status read model: which zones are active (and on which holder), +//! which are asleep with a hibernation snapshot, and which never ran — +//! answered from SQL rows instead of opening one cell per zone. +//! +//! Projected from the outward events of the per-zone cells: +//! +//! | Source cell | Events | Row | +//! |---|---|---| +//! | [`ZoneLease`](super::zone_lease::ZoneLease) | `zone.activated`, `zone.released`, `zone.lease_expired` | [`ZoneLeases`] (`zone_leases`) | +//! | [`ZoneState`](super::zone_state::ZoneState) | `zone_state.saved`, `zone_state.deleted` | [`ZoneSnapshots`] (`zone_snapshots`) | +//! +//! Each event carries its cell's post-transition state, so a row is an +//! upsert of the body ([`ZONE_LEASES`], [`ZONE_SNAPSHOTS`]); the projector +//! runtime orders and deduplicates per stream. [`ZoneStatus::of`] combines +//! a zone's two rows. Mount [`projectors`] on a consumer service (any read +//! model store: SQLite, Postgres) fed by the bus the cells' outbox reaches. +//! +//! The read model is eventually consistent: it is for views (world maps, +//! minimaps, dashboards), never for decisions a zone server must fence — +//! ownership and restore always go to the cells. + +use distributed::graphql::SurfaceProjector; +use distributed::projection::lower::{EventualOnly, ProjectionDescriptor}; +use distributed::{ + mutation_file, LocalProjectionMountsBuilder, Mutation, ReadModel, ReadModelCatalog, + RelationalReadModel, +}; +use serde::{Deserialize, Serialize}; + +use super::zone_lease::{ + ZoneLeaseActivatedDomainEvent, ZoneLeaseLeaseExpiredDomainEvent, ZoneLeaseReleasedDomainEvent, + STATUS_ACTIVE, +}; +use super::zone_state::{ZoneStateDeletedDomainEvent, ZoneStateSavedDomainEvent}; + +/// Application of the read model (catalog and projector owner; the +/// consumer service is named after it). +pub const APPLICATION: &str = "zone-status"; +/// Projector of [`ZoneLeases`]. +pub const LEASES_PROJECTOR: &str = "project_zone_leases"; +/// Projector of [`ZoneSnapshots`]. +pub const SNAPSHOTS_PROJECTOR: &str = "project_zone_snapshots"; +/// Epoch of [`ZONE_LEASES`] (bump to rebuild from the bus log). +pub const LEASES_EPOCH: &str = "zone-leases-v1"; +/// Epoch of [`ZONE_SNAPSHOTS`]. +pub const SNAPSHOTS_EPOCH: &str = "zone-snapshots-v1"; + +/// One `zone_leases` row: the last lease transition of a zone +/// ([`ZoneLeaseStatus`](super::zone_lease::ZoneLeaseStatus); the type is +/// named after its table, as `mutation_file!` requires). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ReadModel)] +#[table("zone_leases")] +pub struct ZoneLeases { + /// Zone id. + #[id("zone_id")] + pub zone_id: String, + /// Holder of the current (or last) claim. + pub holder: String, + /// Its fencing token. + pub token: i64, + /// `active`, `released` or `expired`. + pub status: String, +} + +/// One `zone_snapshots` row: the hibernation slot of a zone +/// ([`ZoneSnapshotStatus`](super::zone_state::ZoneSnapshotStatus)). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ReadModel)] +#[table("zone_snapshots")] +pub struct ZoneSnapshots { + /// Zone id. + #[id("zone_id")] + pub zone_id: String, + /// Highest token seen. + pub token: i64, + /// A snapshot is stored. + pub stored: bool, + /// Content version of the stored snapshot. + pub content_version: String, + /// Snapshot size in bytes. + pub bytes: i64, +} + +/// Upsert a [`ZoneLeases`] from a lease event's state body. +#[allow(non_snake_case)] +pub fn SaveZoneLease() -> Mutation<()> { + mutation_file!("src/world/mutations/save_zone_lease.mutation.graphql") +} + +/// Upsert a [`ZoneSnapshots`] from a zone-state event's state body. +#[allow(non_snake_case)] +pub fn SaveZoneSnapshot() -> Mutation<()> { + mutation_file!("src/world/mutations/save_zone_snapshot.mutation.graphql") +} + +distributed::projection! { + pub const ZONE_LEASES: ProjectionDescriptor = { + name: "project_zone_leases", + version: 1, + epoch: "zone-leases-v1", + model: ZoneLeases, + on { + events: [ + ZoneLeaseActivatedDomainEvent, + ZoneLeaseReleasedDomainEvent, + ZoneLeaseLeaseExpiredDomainEvent, + ], + mutation: SaveZoneLease, + input: { lease: body }, + }, + }; +} + +distributed::projection! { + pub const ZONE_SNAPSHOTS: ProjectionDescriptor = { + name: "project_zone_snapshots", + version: 1, + epoch: "zone-snapshots-v1", + model: ZoneSnapshots, + on { + events: [ZoneStateSavedDomainEvent, ZoneStateDeletedDomainEvent], + mutation: SaveZoneSnapshot, + input: { snapshot: body }, + }, + }; +} + +/// Physical read-model catalog (`zone_leases`, `zone_snapshots`). +pub fn read_model_catalog() -> ReadModelCatalog { + ReadModelCatalog::new(APPLICATION) + .table_schema(ZoneLeases::schema().clone()) + .table_schema(ZoneSnapshots::schema().clone()) +} + +/// The modeled projector mounts `(leases, snapshots)`: mount both on one +/// consumer service named [`APPLICATION`], applying [`ZONE_LEASES`] and +/// [`ZONE_SNAPSHOTS`] respectively. +pub fn projectors() -> (SurfaceProjector, SurfaceProjector) { + let mounts = LocalProjectionMountsBuilder::new(APPLICATION, "ordered-domain-events") + .expect("projection source") + .eventual_model::(LEASES_PROJECTOR, ZONE_LEASES, LEASES_EPOCH) + .expect("zone leases mount") + .eventual_model::(SNAPSHOTS_PROJECTOR, ZONE_SNAPSHOTS, SNAPSHOTS_EPOCH) + .expect("zone snapshots mount") + .build() + .expect("projection catalog"); + ( + mounts + .projector(LEASES_PROJECTOR) + .expect("zone leases projector"), + mounts + .projector(SNAPSHOTS_PROJECTOR) + .expect("zone snapshots projector"), + ) +} + +/// What the read model knows about a zone. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "camelCase")] +pub enum ZoneStatus { + /// A holder has the lease. + Active { + /// The lease holder. + holder: String, + }, + /// No holder; a hibernation snapshot is stored. + Hibernated { + /// Snapshot size in bytes. + bytes: u64, + }, + /// No holder and no snapshot: never ran, or ran without hibernating. + Cold, +} + +impl ZoneStatus { + /// Combine a zone's rows (either may be missing). + pub fn of(lease: Option<&ZoneLeases>, snapshot: Option<&ZoneSnapshots>) -> Self { + if let Some(lease) = lease.filter(|lease| lease.status == STATUS_ACTIVE) { + return Self::Active { + holder: lease.holder.clone(), + }; + } + match snapshot.filter(|snapshot| snapshot.stored) { + Some(snapshot) => Self::Hibernated { + bytes: u64::try_from(snapshot.bytes).unwrap_or(0), + }, + None => Self::Cold, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_prefers_an_active_lease_then_a_stored_snapshot() { + let lease = |status: &str| ZoneLeases { + zone_id: "z".into(), + holder: "A#1".into(), + token: 3, + status: status.into(), + }; + let snapshot = |stored| ZoneSnapshots { + zone_id: "z".into(), + token: 3, + stored, + content_version: "v".into(), + bytes: if stored { 42 } else { 0 }, + }; + assert_eq!( + ZoneStatus::of(Some(&lease("active")), Some(&snapshot(true))), + ZoneStatus::Active { + holder: "A#1".into() + } + ); + assert_eq!( + ZoneStatus::of(Some(&lease("released")), Some(&snapshot(true))), + ZoneStatus::Hibernated { bytes: 42 } + ); + assert_eq!( + ZoneStatus::of(Some(&lease("expired")), Some(&snapshot(false))), + ZoneStatus::Cold + ); + assert_eq!(ZoneStatus::of(None, None), ZoneStatus::Cold); + // The projections compile against the cells' outward events. + let (leases, snapshots) = projectors(); + drop((leases, snapshots)); + assert_eq!(read_model_catalog().tables.len(), 2); + } +} diff --git a/distributed_game/tests/zone_status/main.rs b/distributed_game/tests/zone_status/main.rs new file mode 100644 index 000000000..18e6ffa15 --- /dev/null +++ b/distributed_game/tests/zone_status/main.rs @@ -0,0 +1,232 @@ +//! The zone status read model end to end on the reference host: lease and +//! zone-state cells commit their outward events into their outboxes; the +//! messages go onto a SQLite bus; the `zone_status` projectors (a consumer +//! service like any other) upsert `zone_leases` / `zone_snapshots`; and +//! `ZoneStatus::of` tells active, hibernated and cold zones apart without +//! opening a cell. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; + +use distributed::bus::{Message, MessagePublisher, SqliteBus}; +use distributed::cell_host::{AggregateCell, CellCommandIdentity, CellStreamStore}; +use distributed::microsvc::{ + spawn_service_consumer_loop, CausalProjectorContext, HandlerError, ModeledProjection, Routes, + Service, Session, CONSUMER_IDLE_POLL, ROLE_KEY, USER_ID_KEY, +}; +use distributed::{Aggregate, BusPublisher, ManualClock, SqliteRepository, StreamIdentity}; +use distributed_game::cell::command_id_for; +use distributed_game::world::zone_lease::{mount_zone_lease, ZoneLease, ACQUIRE, RELEASE}; +use distributed_game::world::zone_state::{mount_zone_state, ZoneState, SAVE, ZONE_HOST_ROLE}; +use distributed_game::world::zone_status::{ + projectors, read_model_catalog, ZoneLeases, ZoneSnapshots, ZoneStatus, APPLICATION, + ZONE_LEASES, ZONE_SNAPSHOTS, +}; +use serde_json::{json, Value}; + +const T0: u64 = 1_700_000_000_000; +static CALLS: AtomicU64 = AtomicU64::new(0); + +fn host_session() -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, "zone-server-a"); + session.set(ROLE_KEY, ZONE_HOST_ROLE); + session +} + +fn cell( + kind: &str, + zone: &str, + clock: &ManualClock, +) -> AggregateCell { + let store = CellStreamStore::for_identity(StreamIdentity::new(kind, zone).unwrap()) + .with_clock(clock.clone()); + AggregateCell::from_store(store).unwrap() +} + +async fn dispatch( + cell: &AggregateCell, + command: &str, + input: Value, +) -> Value { + let n = CALLS.fetch_add(1, Ordering::SeqCst); + let identity = CellCommandIdentity::new( + "zone-host", + "zone-server-a", + command_id_for("t", &n.to_string()), + ) + .unwrap(); + cell.dispatch_idempotent(command, &identity, input, host_session()) + .await + .unwrap_or_else(|error| panic!("{command}: {error:?}")) + .payload() + .clone() +} + +async fn project_leases( + context: CausalProjectorContext, + projection: ModeledProjection, +) -> Result<(), HandlerError> { + projection.apply(ZONE_LEASES, &context).await +} + +async fn project_snapshots( + context: CausalProjectorContext, + projection: ModeledProjection, +) -> Result<(), HandlerError> { + projection.apply(ZONE_SNAPSHOTS, &context).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cell_events_project_into_zone_status_rows() { + let clock = ManualClock::new(SystemTime::UNIX_EPOCH + Duration::from_millis(T0)); + // z-active: held. z-asleep: held, snapshot saved, released. + // z-cold: held and released without a snapshot. + let active = mount_zone_lease(cell::("ZoneLease", "z-active", &clock)); + let asleep = mount_zone_lease(cell::("ZoneLease", "z-asleep", &clock)); + let asleep_state = mount_zone_state(cell::("ZoneState", "z-asleep", &clock)); + let cold = mount_zone_lease(cell::("ZoneLease", "z-cold", &clock)); + let acquire = |zone: &str| json!({ "zone_id": zone, "holder": "A#1", "ttl_ms": 60_000 }); + dispatch(&active, ACQUIRE, acquire("z-active")).await; + let held = dispatch(&asleep, ACQUIRE, acquire("z-asleep")).await; + use base64::Engine as _; + dispatch( + &asleep_state, + SAVE, + json!({ "zone_id": "z-asleep", "token": held["token"], "content_version": "v1", + "inline_b64": base64::engine::general_purpose::STANDARD.encode([7u8; 42]) }), + ) + .await; + dispatch( + &asleep, + RELEASE, + json!({ "zone_id": "z-asleep", "holder": "A#1", "token": held["token"] }), + ) + .await; + let held = dispatch(&cold, ACQUIRE, acquire("z-cold")).await; + dispatch( + &cold, + RELEASE, + json!({ "zone_id": "z-cold", "holder": "A#1", "token": held["token"] }), + ) + .await; + + // Outbox → bus, in commit order per cell. + let dir = std::env::temp_dir().join(format!( + "zone-status-{}-{}", + std::process::id(), + CALLS.load(Ordering::SeqCst) + )); + std::fs::create_dir_all(&dir).unwrap(); + let bus_url = format!("sqlite:{}?mode=rwc", dir.join("bus.db").display()); + let pool = sqlx::SqlitePool::connect(&bus_url).await.unwrap(); + let bus = SqliteBus::new(pool.clone()); + bus.ensure_tables().await.unwrap(); + let publisher = BusPublisher::new(Arc::new(bus)); + let mut messages = Vec::new(); + messages.extend(active.durable_outbox().unwrap()); + messages.extend(asleep.durable_outbox().unwrap()); + messages.extend(asleep_state.durable_outbox().unwrap()); + messages.extend(cold.durable_outbox().unwrap()); + messages.sort_by_key(|m| m.created_at); + assert_eq!(messages.len(), 6, "activated ×3, released ×2, saved"); + for message in messages { + publisher.publish(Message::from(message)).await.unwrap(); + } + + // The projectors, as a consumer service. + let read_url = format!("sqlite:{}?mode=rwc", dir.join("read.db").display()); + let repo = SqliteRepository::connect_and_migrate(&read_url) + .await + .unwrap(); + repo.bootstrap_table_schema_for_dev(&read_model_catalog().table_registry().unwrap()) + .await + .unwrap(); + let read_models = repo.clone(); + spawn_service_consumer_loop(move || { + let (leases, snapshots) = projectors(); + Service::new() + .named(APPLICATION) + .routes( + Routes::new() + .with_read_model_store(read_models.clone()) + .modeled_projector(leases) + .handle(project_leases) + .modeled_projector(snapshots) + .handle(project_snapshots), + ) + .with_bus(SqliteBus::new(pool.clone()).with_idle_poll(CONSUMER_IDLE_POLL)) + }); + + let status = |zone: &'static str| { + let repo = repo.clone(); + async move { + let lease: Option = sqlx::query_as::<_, (String, String, i64, String)>( + "SELECT zone_id, holder, token, status FROM zone_leases WHERE zone_id = ?", + ) + .bind(zone) + .fetch_optional(repo.pool()) + .await + .unwrap() + .map(|(zone_id, holder, token, status)| ZoneLeases { + zone_id, + holder, + token, + status, + }); + let snapshot: Option = + sqlx::query_as::<_, (String, i64, bool, String, i64)>( + "SELECT zone_id, token, stored, content_version, bytes FROM zone_snapshots WHERE zone_id = ?", + ) + .bind(zone) + .fetch_optional(repo.pool()) + .await + .unwrap() + .map(|(zone_id, token, stored, content_version, bytes)| ZoneSnapshots { + zone_id, + token, + stored, + content_version, + bytes, + }); + (lease, snapshot) + } + }; + let start = Instant::now(); + loop { + let (lease, snapshot) = status("z-asleep").await; + let done = lease.as_ref().is_some_and(|l| l.status == "released") && snapshot.is_some(); + if done + && status("z-cold") + .await + .0 + .is_some_and(|l| l.status == "released") + { + break; + } + assert!( + start.elapsed() < Duration::from_secs(20), + "projection timed out" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let of = |(lease, snapshot): (Option, Option)| { + ZoneStatus::of(lease.as_ref(), snapshot.as_ref()) + }; + assert_eq!( + of(status("z-active").await), + ZoneStatus::Active { + holder: "A#1".into() + } + ); + assert_eq!( + of(status("z-asleep").await), + ZoneStatus::Hibernated { bytes: 42 } + ); + let (_, snapshot) = status("z-asleep").await; + assert_eq!(snapshot.unwrap().content_version, "v1"); + assert_eq!(of(status("z-cold").await), ZoneStatus::Cold); + assert_eq!(of(status("z-never").await), ZoneStatus::Cold); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/examples/game/README.md b/examples/game/README.md index 759262bc0..299bc45ba 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -161,6 +161,14 @@ required, there is no local fallback): (`CellZoneStateStore`), fenced by the lease token; a sleeping zone is a sleeping Durable Object. Snapshots above 64 KiB go to celld's object storage (R2 binding `ZONE_SNAPSHOTS`), the cell keeps their digest. +- **Zone status**: both cells publish their transitions (`zone.activated` + / `zone.released` / `zone.lease_expired`, `zone_state.saved` / + `zone_state.deleted`, each carrying the cell's small status, never the + snapshot bytes). celld's outbox Queue relays them to host A's + `/internal/celld-queue/relay`, which publishes onto the world's shared + projection bus (`--data-dir`, `run.sh` uses `.data/`); every host + projects the bus into its own zone status read model + (`distributed_game::world::zone_status`, `common/zone_status.rs`). The cluster smoke tests (`tests/grid_smoke.rs`, `tests/ripple_smoke.rs`) and the headless tests start their own `celld dev` the same way. @@ -191,9 +199,11 @@ and the headless tests start their own `celld dev` the same way. hibernated after 4 s more (`--hibernate-after-ms`): critters and markers (E) go to the zone's celld `ZoneState` cell and come back on the next activation (on either host); its hello says - `restored` and the page toasts. The minimap (`GET /dev/grid/map`, which - asks the peer for its cells) shows active / hibernated / cold cells per - host; borders are tinted by owning host. + `restored` and the page toasts. The minimap (`GET /dev/grid/map`) shows + active / hibernated / cold cells per host from the zone status read model + (eventually consistent; it never opens a cell, since reading a never-used + celld cell creates it), with live tier and player counts asked from the + host running each active cell; borders are tinted by owning host. Controls: WASD / stick, Space/Shift sprint, E drops a marker. diff --git a/examples/game/client/tests/helpers/celld.mjs b/examples/game/client/tests/helpers/celld.mjs index ac3442345..a138872c8 100644 --- a/examples/game/client/tests/helpers/celld.mjs +++ b/examples/game/client/tests/helpers/celld.mjs @@ -33,7 +33,7 @@ export function freePort() { } /** - * Start celld. `relayPort`: the Mob Pit server port its outbox Queue relays + * Start celld. `relayPort`: the demo server port its outbox Queue relays * domain events to (`/internal/celld-queue/relay`). */ export async function startCelld({ relayPort } = {}) { diff --git a/examples/game/client/tests/helpers/cluster.mjs b/examples/game/client/tests/helpers/cluster.mjs index 5f1487e1b..d12d0f6a7 100644 --- a/examples/game/client/tests/helpers/cluster.mjs +++ b/examples/game/client/tests/helpers/cluster.mjs @@ -2,11 +2,15 @@ * Start a multi-host demo (`game-demos --host-id … --peer …`) for a * Node test: one process per host on free ephemeral ports, a shared ticket * secret, dev routes on, and one private `celld dev` (zone leases and - * hibernation snapshots in `examples/game/cells`). Uses GAME_DEMOS_BIN when - * set, else `cargo run`. + * hibernation snapshots in `examples/game/cells`) whose outbox Queue relays + * to the first host; the hosts share a throwaway data dir (projection bus, + * zone status read models). Uses GAME_DEMOS_BIN when set, else `cargo run`. */ import { spawn } from 'node:child_process'; +import { mkdtempSync } from 'node:fs'; import { Socket } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { freePort, startCelld } from './celld.mjs'; @@ -20,7 +24,8 @@ const SERVER_DIR = fileURLToPath(new URL('../../../server/', import.meta.url)); export async function startCluster(demo, { hosts = ['A', 'B'], hibernateAfterMs = 4000, env = {} } = {}) { const ports = []; for (const _ of hosts) ports.push(await freePort()); - const celld = await startCelld(); + const celld = await startCelld({ relayPort: ports[0] }); + const dataDir = mkdtempSync(join(tmpdir(), `${demo}-js-`)); const secret = `test-secret-${process.pid}-${Date.now()}`; const bin = process.env.GAME_DEMOS_BIN; const children = []; @@ -33,7 +38,10 @@ export async function startCluster(demo, { hosts = ['A', 'B'], hibernateAfterMs hosts.forEach((peer, j) => { if (j !== i) args.push('--peer', `${peer}=127.0.0.1:${ports[j]}`); }); - const options = { env: { ...process.env, ...celld.env, ...env }, stdio: ['ignore', 'pipe', 'inherit'] }; + const options = { + env: { ...process.env, ...celld.env, GAME_DEMOS_DATA_DIR: dataDir, ...env }, + stdio: ['ignore', 'pipe', 'inherit'] + }; const child = bin ? spawn(bin, args, options) : spawn('cargo', ['run', '-q', '--', ...args], { ...options, cwd: SERVER_DIR }); diff --git a/examples/game/run.sh b/examples/game/run.sh index 2ce4d481d..7fdf79a15 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -307,11 +307,13 @@ run_cluster() { dir="$(mktemp -d "${TMPDIR:-/tmp}/game-demos-$demo.XXXXXX")" # Both hosts must share the ticket secret (and derive the admin secret). secret="${DISTRIBUTED_GAME_SECRET:-$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')}" + # The relay lands on host A; both hosts share the projection bus and keep + # their zone status read models in .data/. DISTRIBUTED_GAME_SECRET="$secret" "$BIN" "$demo" --dev --port "$port_a" --host-id A \ - --peer "B=127.0.0.1:$port_b" >"$dir/A.log" 2>&1 & + --peer "B=127.0.0.1:$port_b" --data-dir "$HERE/.data/$demo" >"$dir/A.log" 2>&1 & local pid_a=$! DISTRIBUTED_GAME_SECRET="$secret" "$BIN" "$demo" --dev --port "$port_b" --host-id B \ - --peer "A=127.0.0.1:$port_a" >"$dir/B.log" 2>&1 & + --peer "A=127.0.0.1:$port_a" --data-dir "$HERE/.data/$demo" >"$dir/B.log" 2>&1 & local pid_b=$! trap 'kill "$pid_a" "$pid_b" 2>/dev/null || true; wait "$pid_a" "$pid_b" 2>/dev/null || true; kill_tree "$CELLD_PID"; rm -rf "$dir"' EXIT INT TERM wait_listening "$dir/A.log" "$pid_a" "game-demos $demo host A" diff --git a/examples/game/server/src/cluster.rs b/examples/game/server/src/cluster.rs index 3bf3b7223..f9c6233fa 100644 --- a/examples/game/server/src/cluster.rs +++ b/examples/game/server/src/cluster.rs @@ -17,6 +17,12 @@ //! `CellZoneStateStore`, one `ZoneState` cell per zone holding its //! hibernation snapshot, fenced by the lease token. There is no local //! fallback: every host of a world shares the same celld; +//! - the zone status read model (`common::zone_status`): every host serves +//! the celld outbox Queue relay (`common::relay`; celld relays to one of +//! them) onto the world's shared projection bus (`{data-dir}/bus.db`), +//! and projects the lease and zone-state cells' outward events into its +//! own `{data-dir}/zone-status-{host}.db`. The grid's world map reads it +//! instead of opening one cell per zone; //! - guest display names shared with every peer: `/dev/ticket` on any host //! forwards the new `principal → name` to the others before it answers, //! so a zone on another host greets the player by name. @@ -25,7 +31,8 @@ use std::future::Future; use std::net::SocketAddr; -use std::sync::Arc; +use std::path::PathBuf; +use std::sync::{Arc, OnceLock}; use std::time::Duration; use axum::body::Body; @@ -43,6 +50,8 @@ use distributed_game::world::{ }; use serde_json::{json, Value}; +use crate::common::relay::{relay_routes, ProjectionBus}; +use crate::common::zone_status::ZoneStatusView; use crate::common::{ self, dev, random_hex, server_config, CelldOptions, Demo, DemoArgs, DemoOptions, Guests, RunningDemo, @@ -72,6 +81,12 @@ pub struct ClusterArgs { /// Internal secret for the celld cells. #[arg(long, env = "DISTRIBUTED_INTERNAL_SECRET", hide_env_values = true)] pub internal_secret: String, + /// Directory shared by every host of the world: the projection bus the + /// celld outbox Queue relay publishes onto (`bus.db`) and each host's + /// zone status read model (`zone-status-{host}.db`). Default: + /// `$TMPDIR/game-demos-{demo}`. + #[arg(long, env = "GAME_DEMOS_DATA_DIR")] + pub data_dir: Option, } /// In-process cluster options (tests and [`run_cluster`]). @@ -86,6 +101,9 @@ pub struct ClusterOptions { /// celld holding zone ownership (`ZoneLease`) and hibernation /// snapshots (`ZoneState`). pub celld: CelldOptions, + /// The world's shared directory (projection bus, zone status read + /// models). + pub data_dir: PathBuf, } /// Parse `ID=HOST:PORT`. @@ -115,6 +133,8 @@ pub struct World { pub guests: Guests, /// Dormant time before hibernation. pub hibernate_after: Duration, + /// Zone status read model (None for a [`Self::standalone`] world). + pub zone_status: Option, } impl World { @@ -135,6 +155,7 @@ impl World { admin_secret: String::new(), guests, hibernate_after, + zone_status: None, } } @@ -211,7 +232,7 @@ pub fn admin_secret(key: &TicketKey) -> String { } /// Start host `cluster.host_id` of demo `D` on `listener`. -pub fn spawn_cluster( +pub async fn spawn_cluster( listener: tokio::net::TcpListener, options: DemoOptions, cluster: ClusterOptions, @@ -236,6 +257,28 @@ pub fn spawn_cluster( host.id.clone(), ) .map_err(std::io::Error::other)?; + // The relay and the zone status read model (see the module docs). + std::fs::create_dir_all(&cluster.data_dir)?; + let bus = ProjectionBus::open(&format!( + "sqlite:{}", + cluster.data_dir.join("bus.db").display() + )) + .await + .map_err(std::io::Error::other)?; + let zone_status = ZoneStatusView::open( + &format!( + "sqlite:{}?mode=rwc", + cluster + .data_dir + .join(format!("zone-status-{}.db", host.id)) + .display() + ), + bus.pool().clone(), + &format!("zone-status-{}", host.id), + ) + .await + .map_err(std::io::Error::other)?; + let relay = relay_routes(cluster.celld.secret.clone(), Arc::new(OnceLock::from(bus))); let guests = Guests::default(); let world = Arc::new(World { host, @@ -245,6 +288,7 @@ pub fn spawn_cluster( admin_secret: admin_secret(&options.key), guests: guests.clone(), hibernate_after: cluster.hibernate_after, + zone_status: Some(zone_status), }); let mut config = server_config(options.key.clone(), options.net_sim); @@ -266,7 +310,7 @@ pub fn spawn_cluster( .with_ownership(ownership) .with_state_store(world.state.clone()); - let extra = if options.dev { + let extra = relay.merge(if options.dev { let tickets = dev::routes::(server.clone(), options.key, guests.clone()).layer( middleware::from_fn_with_state(world.clone(), share_guest_names), ); @@ -279,7 +323,7 @@ pub fn spawn_cluster( .merge(D::cluster_routes(server.clone(), world.clone())) } else { Router::new() - }; + }); let task = tokio::spawn(server.clone().serve_with(listener, extra, signal)); Ok(RunningHost { demo: RunningDemo { @@ -314,8 +358,11 @@ pub async fn run_cluster(args: ClusterArgs) -> std::io::Result<( hibernate_after: Duration::from_millis(args.hibernate_after_ms.max(100)), celld: CelldOptions::new(args.celld_url, args.internal_secret) .map_err(std::io::Error::other)?, + data_dir: args + .data_dir + .unwrap_or_else(|| std::env::temp_dir().join(format!("game-demos-{}", D::NAME))), }; - let running = spawn_cluster::(listener, options, cluster, common::shutdown_signal())?; + let running = spawn_cluster::(listener, options, cluster, common::shutdown_signal()).await?; // Parsed by run.sh and the client tests: keep the format. println!("LISTENING http://{}", running.demo.addr); let hosts: Vec = running diff --git a/examples/game/server/src/common/mod.rs b/examples/game/server/src/common/mod.rs index 095ba6d0a..4ef46cb8b 100644 --- a/examples/game/server/src/common/mod.rs +++ b/examples/game/server/src/common/mod.rs @@ -12,6 +12,7 @@ pub mod celld; pub mod dev; mod guests; pub mod relay; +pub mod zone_status; use std::future::Future; use std::net::SocketAddr; diff --git a/examples/game/server/src/common/zone_status.rs b/examples/game/server/src/common/zone_status.rs new file mode 100644 index 000000000..a5f466f5a --- /dev/null +++ b/examples/game/server/src/common/zone_status.rs @@ -0,0 +1,151 @@ +//! A host's view of `distributed_game::world::zone_status`: the zone lease +//! and zone-state cells' outward events (cell outbox → celld Queue → +//! [`super::relay`] → projection bus) projected into a SQLite read model of +//! this host, under a bus consumer group of its own. +//! +//! This is the consumer of `zone.activated` / `zone.released` / +//! `zone.lease_expired` and `zone_state.saved` / `zone_state.deleted`; the +//! grid's world map reads it to tell active, hibernated and cold zones apart +//! without opening one cell per zone (a read of a never-used cell would +//! create it). Eventually consistent: for views only. + +use std::collections::HashMap; + +use distributed::bus::SqliteBus; +use distributed::microsvc::{ + spawn_service_consumer_loop, CausalProjectorContext, HandlerError, ModeledProjection, Routes, + Service, CONSUMER_IDLE_POLL, +}; +use distributed::SqliteRepository; +use distributed_game::world::zone_status::{ + projectors, read_model_catalog, ZoneLeases, ZoneSnapshots, ZoneStatus, APPLICATION, + ZONE_LEASES, ZONE_SNAPSHOTS, +}; + +/// See the module docs. +#[derive(Clone)] +pub struct ZoneStatusView { + repo: SqliteRepository, +} + +async fn project_leases( + context: CausalProjectorContext, + projection: ModeledProjection, +) -> Result<(), HandlerError> { + projection.apply(ZONE_LEASES, &context).await +} + +async fn project_snapshots( + context: CausalProjectorContext, + projection: ModeledProjection, +) -> Result<(), HandlerError> { + projection.apply(ZONE_SNAPSHOTS, &context).await +} + +/// `{select} WHERE zone_id IN (?, ?, …)` over `zones`. +fn by_zone(select: &str, zones: &[String]) -> sqlx::QueryBuilder { + let mut query = sqlx::QueryBuilder::new(select); + query.push(" WHERE zone_id IN ("); + let mut separated = query.separated(", "); + for zone in zones { + separated.push_bind(zone); + } + query.push(")"); + query +} + +impl ZoneStatusView { + /// Open (creating) the read model at `url` and project `bus` into it + /// forever as consumer group `group` (the cursor lives in the read + /// model, so a restart resumes). + pub async fn open(url: &str, bus: sqlx::SqlitePool, group: &str) -> Result { + let repo = SqliteRepository::connect_and_migrate(url) + .await + .map_err(|e| format!("zone status {url}: {e}"))?; + let registry = read_model_catalog() + .table_registry() + .map_err(|e| format!("zone status tables: {e}"))?; + repo.bootstrap_table_schema_for_dev(®istry) + .await + .map_err(|e| format!("zone status tables: {e}"))?; + let read_models = repo.clone(); + let group = group.to_string(); + spawn_service_consumer_loop(move || { + let (leases, snapshots) = projectors(); + Service::new() + .named(APPLICATION) + .routes( + Routes::new() + .with_read_model_store(read_models.clone()) + .modeled_projector(leases) + .handle(project_leases) + .modeled_projector(snapshots) + .handle(project_snapshots), + ) + .with_bus( + SqliteBus::new_with_group(bus.clone(), group.clone()) + .with_idle_poll(CONSUMER_IDLE_POLL), + ) + }); + Ok(Self { repo }) + } + + /// The status of each of `zones` (`Cold` when the read model has no + /// row). + pub async fn statuses(&self, zones: &[String]) -> Result, String> { + let mut leases: HashMap = HashMap::new(); + let mut snapshots: HashMap = HashMap::new(); + for chunk in zones.chunks(256) { + let mut query = by_zone( + "SELECT zone_id, holder, token, status FROM zone_leases", + chunk, + ); + for (zone_id, holder, token, status) in query + .build_query_as::<(String, String, i64, String)>() + .fetch_all(self.repo.pool()) + .await + .map_err(|e| format!("zone status: {e}"))? + { + leases.insert( + zone_id.clone(), + ZoneLeases { + zone_id, + holder, + token, + status, + }, + ); + } + let mut query = by_zone( + "SELECT zone_id, token, stored, content_version, bytes FROM zone_snapshots", + chunk, + ); + for (zone_id, token, stored, content_version, bytes) in query + .build_query_as::<(String, i64, bool, String, i64)>() + .fetch_all(self.repo.pool()) + .await + .map_err(|e| format!("zone status: {e}"))? + { + snapshots.insert( + zone_id.clone(), + ZoneSnapshots { + zone_id, + token, + stored, + content_version, + bytes, + }, + ); + } + } + Ok(zones + .iter() + .map(|zone| { + ( + zone.clone(), + ZoneStatus::of(leases.get(zone), snapshots.get(zone)), + ) + }) + .collect()) + } +} diff --git a/examples/game/server/src/grid/dev.rs b/examples/game/server/src/grid/dev.rs index d3d066100..a9202bca9 100644 --- a/examples/game/server/src/grid/dev.rs +++ b/examples/game/server/src/grid/dev.rs @@ -2,11 +2,14 @@ //! //! - `GET /dev/grid/map?x=0&y=0&r=4` (any host): every cell within `r` //! (≤ 8) of `(x, y)` with its owning host and state — `active` (with -//! tier, players, observers), `hibernated` (a snapshot is stored) or -//! `cold` (never ran, or ran without being hibernated). Cells owned by -//! a peer are answered by that peer (`/dev/grid/local`). +//! tier, players, observers from the host running it), `hibernated` (a +//! snapshot is stored) or `cold` (never ran, or ran without being +//! hibernated). Live details of cells owned by a peer are answered by +//! that peer (`/dev/grid/local`); everything else comes from the zone +//! status read model (`crate::common::zone_status`), never from the +//! cells (reading a never-used cell would create it). //! - `POST /dev/grid/local {zones: [...]}` (admin secret, host to host): -//! the same facts for this host's cells. +//! live details of this host's active cells among `zones`. use std::sync::Arc; use std::time::Duration; @@ -16,6 +19,7 @@ use axum::http::{HeaderMap, StatusCode, Uri}; use axum::routing::{get, post}; use axum::{Json, Router}; use distributed_game::net::GameServer; +use distributed_game::world::zone_status::ZoneStatus; use distributed_game::Tier; use serde_json::{json, Value}; @@ -57,68 +61,54 @@ fn tier_name(tier: Tier) -> &'static str { } } -/// Facts about this host's cells among `zones`, keyed by zone id. A cell -/// that is not active is "hibernated" when celld holds a snapshot of it -/// (the lookups run concurrently, bounded to 400 ms; "unknown" after that). +/// Live details of this host's active cells among `zones`, keyed by zone +/// id. async fn local_cells(state: &MapState, zones: &[String]) -> serde_json::Map { let mut out = serde_json::Map::new(); - let mut lookups = tokio::task::JoinSet::new(); for zone in zones { - if state.world.owner(zone) != state.world.host.id || world::parse_zone(zone).is_none() { + if state.world.owner(zone) != state.world.host.id + || world::parse_zone(zone).is_none() + || !state.server.is_active(zone) + { continue; } - let live = if state.server.is_active(zone) { - let inspect = state.server.inspect(zone, |h| { - (tier_name(h.tier()), h.player_count(), h.observer_count()) - }); - tokio::time::timeout(Duration::from_millis(300), inspect) - .await - .ok() - .flatten() - } else { - None - }; - match live { - Some((tier, players, observers)) => { - out.insert( - zone.clone(), - json!({ - "state": "active", "tier": tier, "players": players, "observers": observers - }), - ); - } - None => { - let store = state.world.state.clone(); - let zone = zone.clone(); - lookups.spawn(async move { - let stored = store.load(&zone).await.ok().flatten(); - (zone, stored.is_some()) - }); - } - } - } - let mut pending: std::collections::BTreeSet = zones - .iter() - .filter(|zone| !out.contains_key(zone.as_str())) - .filter(|zone| state.world.owner(zone) == state.world.host.id) - .filter(|zone| world::parse_zone(zone).is_some()) - .cloned() - .collect(); - // Bounded, so a peer asking for our cells gets an answer in time (a - // cold celld cell can take a while to activate); late ones are unknown. - let _ = tokio::time::timeout(Duration::from_millis(400), async { - while let Some(Ok((zone, stored))) = lookups.join_next().await { - pending.remove(&zone); + let inspect = state.server.inspect(zone, |h| { + (tier_name(h.tier()), h.player_count(), h.observer_count()) + }); + if let Ok(Some((tier, players, observers))) = + tokio::time::timeout(Duration::from_millis(300), inspect).await + { out.insert( - zone, - json!({ "state": if stored { "hibernated" } else { "cold" } }), + zone.clone(), + json!({ + "state": "active", "tier": tier, "players": players, "observers": observers + }), ); } - }) - .await; - lookups.abort_all(); - for zone in pending { - out.insert(zone, json!({ "state": "unknown" })); + } + out +} + +/// The read model's state of every zone not live on a host. +async fn stored_cells(state: &MapState, zones: &[String]) -> serde_json::Map { + let mut out = serde_json::Map::new(); + let Some(view) = &state.world.zone_status else { + return out; + }; + match view.statuses(zones).await { + Ok(statuses) => { + for (zone, status) in statuses { + let fact = match status { + ZoneStatus::Active { holder } => json!({ "state": "active", "holder": holder }), + ZoneStatus::Hibernated { bytes } => { + json!({ "state": "hibernated", "bytes": bytes }) + } + ZoneStatus::Cold => json!({ "state": "cold" }), + }; + out.insert(zone, fact); + } + } + Err(error) => eprintln!("grid map: zone status: {error}"), } out } @@ -136,7 +126,8 @@ async fn map(State(state): State, uri: Uri) -> Json { } } let zones: Vec = cells.iter().map(|(_, _, z)| z.clone()).collect(); - let mut facts = local_cells(&state, &zones).await; + let mut facts = stored_cells(&state, &zones).await; + facts.extend(local_cells(&state, &zones).await); let mut totals = serde_json::Map::new(); totals.insert( state.world.host.id.clone(), diff --git a/examples/game/server/tests/grid_smoke.rs b/examples/game/server/tests/grid_smoke.rs index 05168ec8a..48c2d6379 100644 --- a/examples/game/server/tests/grid_smoke.rs +++ b/examples/game/server/tests/grid_smoke.rs @@ -195,17 +195,16 @@ async fn empty_cell_hibernates_and_restores_critters_and_markers() { let critters = snapshot["critters"].as_array().unwrap().clone(); assert!(critters.len() >= 2); - // The world map shows it hibernated (asked on the other host too). + // The world map shows it hibernated (asked on the other host too), from + // the zone status read model once the cells' events are projected. let other = if owner.world.host.id == "A" { "B" } else { "A" }; - let (status, map) = http( - cluster.host(other).demo.addr, - "GET", - "/dev/grid/map?x=5&y=5&r=0", - None, - ) + let other_addr = cluster.host(other).demo.addr; + eventually("map shows hibernated", || async move { + let (status, map) = http(other_addr, "GET", "/dev/grid/map?x=5&y=5&r=0", None).await; + status == 200 && map["cells"][0]["state"] == "hibernated" + }) .await; - assert_eq!(status, 200, "{map}"); - assert_eq!(map["cells"][0]["state"], "hibernated", "{map}"); + let (_, map) = http(other_addr, "GET", "/dev/grid/map?x=5&y=5&r=0", None).await; assert_eq!(map["cells"][0]["owner"], owner.world.host.id.as_str()); let mut again = join(&base, zone, &ticket("guest-marker", zone, Role::Player)) diff --git a/examples/game/server/tests/support/mod.rs b/examples/game/server/tests/support/mod.rs index 838faeb76..6070a9d98 100644 --- a/examples/game/server/tests/support/mod.rs +++ b/examples/game/server/tests/support/mod.rs @@ -70,7 +70,19 @@ pub async fn start_cluster(ids: &[&str], hibernate_after: Durati .iter() .map(|(id, addr, _)| HostAddr::new(id.clone(), addr.to_string())) .collect(); - let celld = DevCelld::start(None).unwrap_or_else(|error| panic!("celld dev: {error}")); + // celld relays its outbox Queue (zone lease and state events) to the + // first host; every host projects the shared bus into its own zone + // status read model. + let relay = format!( + "http://{}{}", + listeners[0].1, + distributed::bus::CELLD_QUEUE_RELAY_PATH + ); + let celld = DevCelld::start(Some(relay)).unwrap_or_else(|error| panic!("celld dev: {error}")); + let data_dir = std::env::temp_dir().join(format!( + "cluster-smoke-{}", + game_demos::common::random_hex(10) + )); let mut hosts = Vec::new(); let mut stops = Vec::new(); for (id, _, listener) in listeners { @@ -80,10 +92,12 @@ pub async fn start_cluster(ids: &[&str], hibernate_after: Durati peers: all.clone(), hibernate_after, celld: celld.options(), + data_dir: data_dir.clone(), }; let host = spawn_cluster::(listener, DemoOptions::dev(SECRET), options, async move { let _ = stopped.await; }) + .await .unwrap(); hosts.push(host); stops.push(stop); From 7f63c74443fa244f131a96a5eed03d5256ee8bf9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:14:52 -0500 Subject: [PATCH 20/40] fix(game): retry a locked shared projection bus open Both hosts of a cluster demo open the shared `bus.db` at startup; switching a new file to WAL and creating the bus tables take the write lock, so one host could exit with `database is locked` (seen in the grid and ripple headless tests). `ProjectionBus::open` now retries that error for up to 10 s. The Mob Pit crash test keeps the server's stderr in its data dir and names it on failure; the presence test waits up to 20 s for a cold presence cell under load. Refs [[tasks/gr-p37-celld-zone-state]] [[tasks/gr-p32-mobpit-celld]] --- examples/game/server/src/common/relay.rs | 34 +++++++++++++++++----- examples/game/server/tests/mobpit_smoke.rs | 19 +++++++++--- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/examples/game/server/src/common/relay.rs b/examples/game/server/src/common/relay.rs index ab01c0252..35dd6def2 100644 --- a/examples/game/server/src/common/relay.rs +++ b/examples/game/server/src/common/relay.rs @@ -54,15 +54,33 @@ pub struct ProjectionBus { impl ProjectionBus { /// Open (creating) the bus at a `sqlite:` URL (shareable between - /// processes). + /// processes). Hosts of one world open the same file at the same time: + /// switching a new database to WAL and creating its tables take the + /// write lock, so a `database is locked` at startup is retried (10 s). pub async fn open(url: &str) -> Result { - let pool = open_shared_sqlite(url, 4) - .await - .map_err(|e| format!("projection bus {e}"))?; - let bus = SqliteBus::new(pool.clone()); - bus.ensure_tables() - .await - .map_err(|e| format!("projection bus tables: {e}"))?; + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let (pool, bus) = loop { + let attempt = async { + let pool = open_shared_sqlite(url, 4) + .await + .map_err(|e| format!("projection bus {e}"))?; + let bus = SqliteBus::new(pool.clone()); + bus.ensure_tables() + .await + .map_err(|e| format!("projection bus tables: {e}"))?; + Ok::<_, String>((pool, bus)) + }; + match attempt.await { + Ok(opened) => break opened, + Err(error) + if error.contains("database is locked") + && std::time::Instant::now() < deadline => + { + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(error) => return Err(error), + } + }; Ok(Self { relay: celld_queue_relay_handler(BusPublisher::new(Arc::new(bus))), pool, diff --git a/examples/game/server/tests/mobpit_smoke.rs b/examples/game/server/tests/mobpit_smoke.rs index e7bbdec55..b356b202f 100644 --- a/examples/game/server/tests/mobpit_smoke.rs +++ b/examples/game/server/tests/mobpit_smoke.rs @@ -475,6 +475,7 @@ struct Binary { } async fn start_binary(dir: &Path, celld: &DevCelld, port: u16) -> Binary { + std::fs::create_dir_all(dir).expect("data dir"); let mut child = Command::new(env!("CARGO_BIN_EXE_game-demos")) .args(["mobpit", "--port", &port.to_string(), "--dev"]) .args(["--secret", SECRET, "--data-dir"]) @@ -483,7 +484,14 @@ async fn start_binary(dir: &Path, celld: &DevCelld, port: u16) -> Binary { .env("DISTRIBUTED_INTERNAL_SECRET", celld.secret()) .env_remove("REDIS_URL") .stdout(Stdio::piped()) - .stderr(Stdio::null()) + // Kept in the data dir: the failure messages below point at it. + .stderr( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(dir.join("server.log")) + .expect("server log"), + ) .spawn() .expect("spawn game-demos"); let stdout = child.stdout.take().unwrap(); @@ -581,7 +589,10 @@ async fn kill_dash_nine_mid_fight_keeps_settled_facts_exactly_once() { while stable_since.elapsed() < Duration::from_secs(3) || total < settled_before { assert!( start.elapsed() < Duration::from_secs(60), - "directory never settled: {total} kills, {settled_before} settled before the crash" + "directory never settled: {total} kills, {settled_before} settled before the crash; \ + server log {}, celld logs in {}", + dir.join("server.log").display(), + celld.project().display() ); tokio::time::sleep(Duration::from_millis(200)).await; let now = directory_total().await; @@ -742,7 +753,7 @@ async fn presence_is_settled_in_presence_cells_from_the_lifecycle_facts() { let mut ana = join("guest-ana").await; let bo = join("guest-bo").await; let online = Some(("pit-1".to_string(), PresenceStatus::Online)); - eventually("ana online", Duration::from_secs(5), { + eventually("ana online", Duration::from_secs(20), { let s = status("guest-ana"); let online = online.clone(); move || { @@ -756,7 +767,7 @@ async fn presence_is_settled_in_presence_cells_from_the_lifecycle_facts() { .await .unwrap(); drop(bo); - eventually("ana offline, bo away", Duration::from_secs(5), { + eventually("ana offline, bo away", Duration::from_secs(20), { let (a, b) = (status("guest-ana"), status("guest-bo")); move || { let (a, b) = (a.clone(), b.clone()); From cd89746b8476994ce2194a55a51a913b5b097630 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:24:22 -0500 Subject: [PATCH 21/40] feat!: portable aggregate schedules and clock, command fact sink Aggregates schedule work and read the host clock through the repository pattern instead of cell-only APIs: - `distributed::schedule`: `Schedule`, `ScheduleWrite`, `ArmedSchedule`, `schedule_session`/`schedule_guard`, `Capability`/`Capabilities` (were `cell_host::CellAlarm*`). `ctx.schedule`, `ctx.cancel_schedule`, `ctx.scheduled_due` and `ctx.now()` replace `arm_alarm`, `cancel_alarm`, `alarm_due` and `cell_now`. - `CommitBatch.schedules` (was `alarms`), owned by the batch's one stream, stored in the commit on every backend: cell `cell_alarms`, in-memory per-stream table, SQL `aggregate_schedules` (migration 12; relative due times by the database clock). - Backends declare capabilities: cells dispatch schedules and stamp the DO/store clock; in-memory (`with_clock`) and SQL (database clock) store schedules but do not dispatch yet. Routes declare `.requires(Capability::..)`; registration panics when the backend lacks one, `ctx.schedule` refuses on undeclared routes. - `AggregateCell::{armed,next,fire_due}_schedule*`; zone lease, alarm worker, tests and docs ported. - `distributed_game::command_sink::CommandFactSink` with `FactRoute`; `CelldFactSink` is that sink over a remote-only `CelldCommandHost` (session-aware cell routes, forwarded `x-game-*` stamp). Cells answer `CellDispatchResult::wait_path_wire`; `CausalDispatchResult::replayed()`. A local-host test runs the celld test's fact routes against an in-memory repository. Refs [[tasks/gr-p38-portable-schedule-and-fact-sink]] --- distributed_cli/src/contracts/tests.rs | 2 +- distributed_game/Cargo.toml | 15 +- distributed_game/README.md | 49 +- distributed_game/src/celld.rs | 403 +------- distributed_game/src/command_sink.rs | 482 +++++++++ distributed_game/src/lib.rs | 2 + distributed_game/src/world/zone_lease.rs | 53 +- distributed_game/src/world/zone_state.rs | 8 +- distributed_game/tests/celld_facts/main.rs | 8 +- distributed_game/tests/command_facts/main.rs | 286 ++++++ distributed_game/tests/zone_lease/main.rs | 21 +- docs/coordination-primitives.md | 96 +- examples/game/cells/src/lib.rs | 10 +- examples/game/server/src/freeze/world.rs | 12 +- examples/game/server/src/mobpit/store.rs | 15 +- migrations/inventory.json | 12 + .../postgres/0012_aggregate_schedules.sql | 24 + .../sqlite/0012_aggregate_schedules.sql | 24 + src/command_dispatch/host.rs | 21 +- src/commit_builder/mod.rs | 2 +- src/graphql/projection_delta/tests.rs | 3 + src/in_memory_repo/repository.rs | 79 +- src/lib.rs | 2 + src/microsvc/causal.rs | 47 +- src/microsvc/cell_host/alarm.rs | 786 +-------------- src/microsvc/cell_host/causal.rs | 23 +- src/microsvc/cell_host/cell.rs | 57 +- src/microsvc/cell_host/celld_outbox.rs | 2 +- src/microsvc/cell_host/command.rs | 280 +++++- src/microsvc/cell_host/mod.rs | 15 +- src/microsvc/cell_host/sql_store.rs | 38 +- src/microsvc/cell_host/store.rs | 51 +- src/microsvc/cell_host/tests/alarms.rs | 178 ++-- src/microsvc/dependencies.rs | 18 + src/microsvc/mod.rs | 6 +- src/microsvc/service/causal.rs | 28 +- src/microsvc/service/handlers.rs | 53 +- src/microsvc/service/routes.rs | 109 ++- src/microsvc/service/tests.rs | 31 +- src/microsvc/session.rs | 7 +- src/outbox/commit.rs | 2 +- src/queued_repo/repository.rs | 24 +- src/repository/sql.rs | 2 + src/repository/sql/schedules.rs | 109 +++ src/repository/traits.rs | 17 +- src/schedule.rs | 922 ++++++++++++++++++ src/sqlx_repo/repo/backend.rs | 18 +- src/sqlx_repo/repo/commit.rs | 92 +- tests/aggregate_schedules.rs | 83 ++ tests/celld/alarm-worker/src/lib.rs | 25 +- tests/celld/fact-worker/src/lib.rs | 12 +- tests/celld/main.rs | 5 +- tests/celld/worker/src/lib.rs | 12 +- .../read_models.rs | 2 +- .../scenario.rs | 2 +- tests/postgres_repository/main.rs | 4 +- tests/sqlite_repository/main.rs | 8 +- 57 files changed, 3095 insertions(+), 1602 deletions(-) create mode 100644 distributed_game/src/command_sink.rs create mode 100644 distributed_game/tests/command_facts/main.rs create mode 100644 migrations/postgres/0012_aggregate_schedules.sql create mode 100644 migrations/sqlite/0012_aggregate_schedules.sql create mode 100644 src/repository/sql/schedules.rs create mode 100644 src/schedule.rs create mode 100644 tests/aggregate_schedules.rs diff --git a/distributed_cli/src/contracts/tests.rs b/distributed_cli/src/contracts/tests.rs index 26deeacac..65105aa22 100644 --- a/distributed_cli/src/contracts/tests.rs +++ b/distributed_cli/src/contracts/tests.rs @@ -1259,7 +1259,7 @@ fn migration_inventory_is_deterministic_and_preserves_runtime_order() { .iter() .map(|migration| migration.version) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); assert_eq!( inventory.canonical_bytes().expect("canonical inventory"), inventory diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index c23108bdf..d71f92a8b 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -16,9 +16,13 @@ default = [] runtime = ["dep:tokio"] # Adapter that commits facts into a `distributed` AggregateCell. cell = ["dep:distributed", "dep:sha2", "dep:base64"] -# `celld::CelldFactSink`: a zone server's facts sent to celld cells over the -# internal HTTP path (routes by command; stable command ids; no local store). -celld = ["cell", "dep:reqwest", "dep:uuid"] +# `command_sink::CommandFactSink`: facts dispatched as +# commands through any command host (a local service over any repository, +# or celld cells). +command-host = ["cell", "distributed/graphql"] +# celld clients: `CelldFactSink` (the command sink over a remote-only +# `CelldCommandHost`), `CellZoneOwnership`, `CellZoneStateStore`. +celld = ["cell", "command-host", "dep:reqwest", "dep:uuid"] # celld worker glue (wasm32 Durable Objects): `world::zone_state::worker`, # the zone-state HTTP surface over a cell and an R2 bucket binding. workers-rs = ["cell", "dep:worker", "distributed/workers-rs"] @@ -89,6 +93,11 @@ name = "zone_state" path = "tests/zone_state/main.rs" required-features = ["cell"] +[[test]] +name = "command_facts" +path = "tests/command_facts/main.rs" +required-features = ["command-host"] + [[test]] name = "zone_status" path = "tests/zone_status/main.rs" diff --git a/distributed_game/README.md b/distributed_game/README.md index b92b64f16..764528e93 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -747,9 +747,11 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), fact with no principal; `ctx.fact(&input)` builds the `Fact` for a retry backlog. `Fact::new(key, command, json)` still works. - A background worker commits facts with retries. - `celld::CelldFactSink` sends each one to its celld cell (that player's - cell, a match cell, ...) as a command over the internal path, with a - command id derived from the fact key. A retry or a duplicate is + `command_sink::CommandFactSink` dispatches each one as a + command through a command host — `celld::CelldFactSink` to its celld cell + (that player's cell, a match cell, ...) over the internal path, or a + `LocalCommandHost` to a service over any repository — with a command id + derived from the fact key. A retry or a duplicate is therefore a harmless replay, never a double reward. (`cell::CellFactSink` does the same into an in-process reference cell, for unit tests.) - `on_fact_settled` reports the outcome on a later tick. @@ -768,7 +770,7 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), with `max_concurrent_commits > 1`) and wait in a host backlog rather than being dropped when the queue is full. `lifecycle::Lifecycle::of(&fact)` decodes them in a sink. Mob Pit routes them to per-player celld presence - cells (`CelldFactSink::lifecycle_route`), ordered by stamp. + cells (`CommandFactSink::lifecycle_route`), ordered by stamp. - Every fact carries a `FactStamp`: zone, activation (boot epoch), tick, host wall time `emitted_at_ms` (`Clock::unix_ms`; `ManualClock` in tests), content version, config version (`HostConfig::config_version`, @@ -940,7 +942,8 @@ trait and its ABI (`run_abi`, `call_bytes`). | *(default)* | Zone/host runtime, facts queue, protocol, schema, replication, authoring derives, game surface, predicted steps (compiles to `wasm32-unknown-unknown`, where `#[predict]` exports its steps) | `serde`, `serde_json`, `distributed_game_macros` | | `runtime` | `spawn_fact_worker` on Tokio | `tokio` (rt, time, sync) | | `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native, unit tests); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record`; the `world::zone_lease` and `world::zone_state` cell aggregates | `distributed`, `sha2`, `base64` | -| `celld` | `celld::CelldFactSink`: facts → celld cells over the internal HTTP path, routed by command; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells; `celld::CellZoneStateStore`: hibernation snapshots in celld `ZoneState` cells | `cell` + `reqwest`, `uuid` | +| `command-host` | `command_sink::CommandFactSink`: facts → commands through any command host, routed by command (`FactRoute`) | `cell` + `distributed/graphql` | +| `celld` | `celld::CelldFactSink` (the command sink over a remote-only `CelldCommandHost`): facts → celld cells over the internal HTTP path; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells; `celld::CellZoneStateStore`: hibernation snapshots in celld `ZoneState` cells | `command-host` + `reqwest`, `uuid` | | `workers-rs` | `world::zone_state::celld_worker`: the `zone-state/{zone}` route of a celld worker (cell + R2 bucket binding for large snapshots) | `cell` + `worker` | | `metrics` | `metrics::GameMetrics` per-zone Prometheus text | – | | `net` | `net::GameServer` (axum WebSocket), HMAC tickets, `GET /metrics` (implies `metrics`), handoff, admin peer routes, `WorldCoordinator` | `axum`, `hmac`, `sha2`, `base64`, `tokio` | @@ -984,25 +987,35 @@ double-crediting. A custom sink needs only crate-root imports: (`BoxFuture` is also re-exported from `world` and `net` for `ZoneOwnership` / `BarrierParticipant` impls). -**Facts to celld cells.** Durable aggregates live in celld cells; a zone -server sends its facts there with one `celld::CelldFactSink` (feature -`celld`), routing each fact by its command to a cell kind and shard: +**Facts as commands.** A zone server sends its facts to the durable +aggregates as commands through a `CommandHost`, with one +`command_sink::CommandFactSink` (feature `command-host`), routing each fact +by its command to an aggregate kind and shard (`FactRoute`: by principal, +by input field, by zone, or a function of the input and trusted session). +The same routes serve every host: `celld::CelldFactSink` (feature `celld`) +addresses celld cells by kind and shard; over a `LocalCommandHost` a +service routes by command and its handler picks the stream +(`tests/command_facts` runs the celld test's routes against an in-memory +repository): ```rust,ignore -let sink = CelldFactSink::new(celld_url, InternalHttpSecret::new(secret)?, "zone-server", "zone:pit-1")? - .route(CelldFactRoute::by_principal::("crawler")) // Crawler:{principal} - .route(CelldFactRoute::by_input_field::("match", "match_id")) +let sink = CelldFactSink::celld(celld_url, InternalHttpSecret::new(secret)?, "zone-server", "zone:pit-1")? + .route(FactRoute::by_principal::("crawler")) // Crawler:{principal} + .route(FactRoute::by_input_field::("match", "match_id")) .lifecycle_route("presence"); // zone.* → Presence:{account} +// or: CommandFactSink::new(LocalCommandHost::new(service), "zone-server", "zone:pit-1")?.route(..) let (queue, feed) = fact_channel(capacity, RetryPolicy::until_settled(base, max)); spawn_fact_worker(feed.with_max_concurrent_commits(n), sink); ``` -Each fact is `POST {CELLD_URL}/{kind}/{shard}/{command}` with -`{commandId, input}`; the command id is `cell::command_id_for(service, -key)`, so a retry after a timeout replays the cell's ledger receipt -(`replayed`) instead of committing twice. The principal travels as -`x-user-id` and the command-ledger principal partition, the stamp as -`x-game-*` headers, and every request carries +Over celld each fact is `POST {CELLD_URL}/{kind}/{shard}/{command}` with +`{commandId, input}` (the remote-only `CelldCommandHost`); the command id is +`cell::command_id_for(service, key)`, so a retry after a timeout replays the +cell's ledger receipt (`replayed`) instead of committing twice. Cells answer +the standard wait-path body (`CellDispatchResult::wait_path_wire`). The +principal travels as `x-user-id` and the command-ledger principal partition, +the stamp as `x-game-*` headers (`CelldCommandHost::forward_session_prefix`), +and every request carries `DISTRIBUTED_INTERNAL_SECRET`; the cell rebuilds the trusted session with `cell::fact_session_from_headers` after checking the secret. Player lifecycle facts route to presence, never to gameplay cells. celld slow, @@ -1105,7 +1118,7 @@ principals commit concurrently; settlements arrive in completion order. | Piece | Purpose | |---|---| | `ZoneDirectory` (`StaticDirectory`, `HashRing`) | zone → owning `HostAddr`. `HashRing` uses virtual nodes; adding/removing one of N hosts moves ~1/N of the zones. A Hello for a zone owned elsewhere gets `Redirect(zone, url)`; `url` is the owner's `HostAddr::ws_url` (advertise a proxied public URL with `HostAddr::with_ws_url`) or `ServerConfig::with_redirect_url(\|host, zone\| …)`. | -| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the cell's alarm publishes `zone.lease_expired` when a holder stops renewing, so services see a crashed zone server; each event carries the lease's `ZoneLeaseStatus { zone_id, holder, token, status }`. `world::ownership_conformance` is the behaviour every implementation passes. | +| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the lease's expiry schedule (`ctx.schedule`; the routes require `Capability::Scheduling` and `Clock`) publishes `zone.lease_expired` when a holder stops renewing, so services see a crashed zone server; each event carries the lease's `ZoneLeaseStatus { zone_id, holder, token, status }`. `world::ownership_conformance` is the behaviour every implementation passes. | | `ZoneStateStore` (`celld::CellZoneStateStore`; `MemoryZoneStateStore` for unit tests) | hibernation snapshots (async `load` / `save` / `delete`), written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). On celld each zone's snapshot is its own `ZoneState:{zone}` cell (`world::zone_state`): inline up to 64 KiB, larger ones in the worker's R2 bucket with the digest in the cell; saves and deletes publish `zone_state.saved` / `zone_state.deleted` with a `ZoneSnapshotStatus { zone_id, token, stored, content_version, bytes }` (no bytes). `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | | `world::zone_status` (feature `cell`) | the read model of those events: `zone_leases` and `zone_snapshots` rows (`projection!` upserts of the state bodies), `projectors()` to mount on any consumer service and read-model store, and `ZoneStatus::of` → `Active { holder }` / `Hibernated { bytes }` / `Cold`. For views (world maps, minimaps): it answers "which zones are active or asleep?" without opening a cell per zone (on celld a read of a never-used cell creates it). Eventually consistent; ownership and restore always go to the cells. | | `ZoneTopology` (`DottedTopology` default: `eldoria.port.docks` → `eldoria.port` → `eldoria`) | parent links for tier-up events. | diff --git a/distributed_game/src/celld.rs b/distributed_game/src/celld.rs index 54a81dc4a..7d51654b6 100644 --- a/distributed_game/src/celld.rs +++ b/distributed_game/src/celld.rs @@ -1,36 +1,11 @@ -//! [`CelldFactSink`]: a zone server's facts, sent over the network to the -//! durable aggregates in celld cells (game-services decision D3). Also the -//! zone server's other celld clients: [`CellZoneOwnership`] (zone leases) -//! and [`CellZoneStateStore`] (hibernation snapshots). -//! -//! One sink per zone server routes every fact by its command -//! ([`FactCommand::COMMAND`](crate::FactCommand::COMMAND)) to a registered -//! [`CelldFactRoute`]: the cell kind and a shard function of the fact, giving -//! `POST {CELLD_URL}/{kind}/{shard}/{command}` (for example -//! `Crawler:{character}` or `Match:{match_id}`). Player lifecycle facts -//! (`zone.*`, [`crate::lifecycle`]) route to the presence cell of the -//! player's account, never to a gameplay cell -//! ([`CelldFactSink::lifecycle_route`]). -//! -//! - **Exactly one commit per key.** The command id is -//! [`command_id_for`]`(service_id, fact key)`, so a retry after a timeout -//! replays through the cell's command ledger (`replayed` receipt) instead -//! of committing twice. -//! - **Trusted identity over the internal path only.** Every request carries -//! `DISTRIBUTED_INTERNAL_SECRET`; the principal (bound by the host from the -//! verified ticket) travels as `x-user-id` and the command-ledger principal -//! partition, and the [`FactStamp`](crate::FactStamp) as `x-game-*` -//! headers. The cell rebuilds the session with -//! [`fact_session_from_headers`](crate::cell::fact_session_from_headers). -//! - **No local durable fallback.** celld slow or unreachable, a 5xx, or a -//! command still in progress is [`FactError::Retryable`]: the fact stays in -//! the zone's [`FactQueue`](crate::FactQueue) backlog and the worker backs -//! off (use [`RetryPolicy::until_settled`](crate::RetryPolicy::until_settled) -//! so an outage never drops a fact); a full queue surfaces to the zone as -//! [`EmitError::Backpressure`](crate::EmitError::Backpressure). An unrouted -//! command and a rejected command are [`FactError::Permanent`]. +//! A zone server's celld clients: [`CelldFactSink`] (facts as commands to +//! the durable aggregates in celld cells, game-services decision D3: the +//! [`CommandFactSink`](crate::command_sink::CommandFactSink) over a +//! remote-only `CelldCommandHost`), [`CellZoneOwnership`] (zone leases) and +//! [`CellZoneStateStore`] (hibernation snapshots). + +pub use crate::command_sink::{CelldFactSink, FactRoute, DEFAULT_CELLD_FACT_TIMEOUT}; -use std::sync::Arc; use std::time::Duration; use distributed::cell_host::{ @@ -40,187 +15,7 @@ use distributed::cell_host::{ use distributed::USER_ID_KEY; use serde_json::{json, Value}; -use crate::cell::command_id_for; -use crate::facts::{BoxFuture, Fact, FactCommand, FactError, FactReceipt, FactSink}; -use crate::lifecycle::LIFECYCLE_COMMANDS; - -/// Default per-request timeout. -pub const DEFAULT_CELLD_FACT_TIMEOUT: Duration = Duration::from_secs(10); - -type ShardFn = Arc Option + Send + Sync>; - -/// Where facts of one command go: a celld cell kind and a shard function. -#[derive(Clone)] -pub struct CelldFactRoute { - command: String, - kind: String, - shard: ShardFn, -} - -impl CelldFactRoute { - /// Route `command` to `{kind}/{shard(fact, principal)}`. `principal` is the - /// fact's verified principal, or the sink's system principal for zone - /// facts. Return `None` when the fact cannot be routed (settles - /// `Permanent`). - pub fn new( - command: impl Into, - kind: impl Into, - shard: impl Fn(&Fact, &str) -> Option + Send + Sync + 'static, - ) -> Self { - Self { - command: command.into(), - kind: kind.into(), - shard: Arc::new(shard), - } - } - - /// Route the typed fact `C` to the cell of its principal (one cell per - /// player: `PlayerProgress:{character}`, `Crawler:{principal}`). - pub fn by_principal(kind: impl Into) -> Self { - Self::new(C::COMMAND, kind, |_, principal| Some(principal.to_string())) - } - - /// Route the typed fact `C` by a field of its input (one cell per match: - /// `Match:{match_id}` with `field = "match_id"`). - pub fn by_input_field(kind: impl Into, field: &'static str) -> Self { - Self::new(C::COMMAND, kind, move |fact, _| { - match fact.input().get(field)? { - Value::String(value) => Some(value.clone()), - Value::Number(value) => Some(value.to_string()), - _ => None, - } - }) - } - - /// The routed command. - pub fn command(&self) -> &str { - &self.command - } - - /// The cell kind (URL segment). - pub fn kind(&self) -> &str { - &self.kind - } -} - -/// Fact sink posting each fact to its celld cell over the internal path. -#[derive(Clone)] -pub struct CelldFactSink { - client: reqwest::Client, - base: String, - secret: InternalHttpSecret, - service_id: String, - system_principal: String, - routes: Vec, -} - -impl CelldFactSink { - /// A sink for `celld_url` under command-ledger `service_id`; zone facts - /// without a player use `system_principal`. - pub fn new( - celld_url: impl Into, - secret: InternalHttpSecret, - service_id: impl Into, - system_principal: impl Into, - ) -> Result { - Self::with_timeout( - celld_url, - secret, - service_id, - system_principal, - DEFAULT_CELLD_FACT_TIMEOUT, - ) - } - - /// As [`new`](Self::new) with an explicit per-request timeout. - pub fn with_timeout( - celld_url: impl Into, - secret: InternalHttpSecret, - service_id: impl Into, - system_principal: impl Into, - timeout: Duration, - ) -> Result { - let base = celld_url.into().trim_end_matches('/').to_string(); - if !(base.starts_with("http://") || base.starts_with("https://")) { - return Err(format!("celld URL must be http(s): {base}")); - } - let service_id = service_id.into(); - let system_principal = system_principal.into(); - if service_id.trim().is_empty() || system_principal.trim().is_empty() { - return Err("celld fact sink needs a service id and a system principal".into()); - } - let client = reqwest::Client::builder() - .timeout(timeout) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|error| format!("celld fact client: {error}"))?; - Ok(Self { - client, - base, - secret, - service_id, - system_principal, - routes: Vec::new(), - }) - } - - /// Add a route. A later route for the same command replaces the earlier. - pub fn route(mut self, route: CelldFactRoute) -> Self { - self.routes - .retain(|existing| existing.command != route.command); - self.routes.push(route); - self - } - - /// Route every player lifecycle fact (`zone.*`) to the presence cell - /// `{kind}/{principal}` (the player's account), not to gameplay cells. - pub fn lifecycle_route(mut self, kind: impl Into) -> Self { - let kind = kind.into(); - for command in LIFECYCLE_COMMANDS { - self = self.route(CelldFactRoute::new( - command, - kind.clone(), - |_, principal| Some(principal.to_string()), - )); - } - self - } - - /// The route a fact would take, if any. - pub fn route_for(&self, command: &str) -> Option<&CelldFactRoute> { - self.routes.iter().find(|route| route.command == command) - } - - /// The request a fact becomes: `(url, command id)`. - pub fn target(&self, fact: &Fact) -> Result<(String, String), FactError> { - let route = self.route_for(fact.command()).ok_or_else(|| { - FactError::Permanent(format!( - "no celld route for fact command `{}`", - fact.command() - )) - })?; - let principal = fact.principal().unwrap_or(&self.system_principal); - let shard = (route.shard)(fact, principal) - .filter(|shard| valid_segment(shard)) - .ok_or_else(|| { - FactError::Permanent(format!( - "fact `{}` has no valid `{}` shard", - fact.key(), - route.kind - )) - })?; - Ok(( - format!( - "{}/{}/{}/{}", - self.base, - encode_segment(&route.kind), - encode_segment(&shard), - encode_segment(fact.command()) - ), - command_id_for(&self.service_id, fact.key()), - )) - } -} +use crate::facts::BoxFuture; /// Path segments are non-empty, short, and printable. fn valid_segment(segment: &str) -> bool { @@ -241,57 +36,6 @@ fn encode_segment(segment: &str) -> String { encoded } -fn classify(status: u16, body: &Value) -> FactError { - let code = body.get("code").and_then(Value::as_str).unwrap_or(""); - let message = body - .get("error") - .and_then(Value::as_str) - .unwrap_or("celld rejected the fact"); - let detail = format!("HTTP {status} {code}: {message}"); - match (status, code) { - (_, "COMMAND_IN_PROGRESS") | (_, "INTERNAL") => FactError::Retryable(detail), - (408 | 429, _) => FactError::Retryable(detail), - (500..=599, _) => FactError::Retryable(detail), - _ => FactError::Permanent(detail), - } -} - -impl FactSink for CelldFactSink { - fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { - Box::pin(async move { - let (url, command_id) = self.target(fact)?; - let principal = fact.principal().unwrap_or(&self.system_principal); - let mut request = self - .client - .post(&url) - .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) - .header(CELL_SERVICE_ID_HEADER, &self.service_id) - .header(CELL_PRINCIPAL_PARTITION_HEADER, principal) - .header(USER_ID_KEY, principal) - .json(&json!({ "commandId": command_id, "input": fact.input() })); - for (name, value) in fact.stamp().session_vars() { - request = request.header(name, value); - } - let response = request.send().await.map_err(|error| { - FactError::Retryable(format!("celld unreachable at {url}: {error}")) - })?; - let status = response.status().as_u16(); - let body: Value = response.json().await.unwrap_or(Value::Null); - if !(200..300).contains(&status) { - return Err(classify(status, &body)); - } - let replayed = body - .pointer("/receipt/replayed") - .and_then(Value::as_bool) - .unwrap_or(false); - Ok(FactReceipt { - payload: body.get("payload").cloned().unwrap_or(Value::Null), - replayed, - }) - }) - } -} - /// [`ZoneOwnership`](crate::world::ZoneOwnership) over celld /// `ZoneLease:{zone_id}` cells ([`crate::world::zone_lease`]): zone servers /// acquire, renew and release over the internal path, and the cell's alarm @@ -663,128 +407,10 @@ impl crate::world::ZoneStateStore for CellZoneStateStore { #[cfg(test)] mod tests { - use super::*; + use crate::facts::Fact; use crate::stamp::FactStamp; - - const SECRET: &str = "test-only-internal-secret-change-me-2026"; - - #[derive(serde::Serialize)] - struct Kill { - mob: String, - } - impl FactCommand for Kill { - const COMMAND: &'static str = "crawler.record_kill"; - } - #[derive(serde::Serialize)] - struct Result_ { - match_id: String, - } - impl FactCommand for Result_ { - const COMMAND: &'static str = "match.record_result"; - } - - fn sink() -> CelldFactSink { - CelldFactSink::new( - "http://127.0.0.1:9/", - InternalHttpSecret::new(SECRET).unwrap(), - "zone-server", - "zone:pit-1", - ) - .unwrap() - .route(CelldFactRoute::by_principal::("crawler")) - .route(CelldFactRoute::by_input_field::( - "match", "match_id", - )) - .lifecycle_route("presence") - } - - fn fact(command: &str, input: Value, principal: Option<&str>) -> Fact { - Fact::trusted( - format!("{command}:pit-1:1:0"), - command, - input, - principal.map(str::to_string), - FactStamp::default(), - ) - } - - #[test] - fn routes_by_command_to_kind_and_shard_with_a_stable_command_id() { - let sink = sink(); - let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); - let (url, id) = sink.target(&kill).unwrap(); - assert_eq!(url, "http://127.0.0.1:9/crawler/alice/crawler.record_kill"); - assert_eq!(id, command_id_for("zone-server", kill.key())); - assert_eq!(sink.target(&kill).unwrap().1, id, "stable across retries"); - - let result = fact("match.record_result", json!({ "match_id": "m/7" }), None); - let (url, _) = sink.target(&result).unwrap(); - assert_eq!( - url, "http://127.0.0.1:9/match/m%2F7/match.record_result", - "a shard cannot add path segments" - ); - let spaced = fact("match.record_result", json!({ "match_id": "m 7" }), None); - assert!(matches!(sink.target(&spaced), Err(FactError::Permanent(_)))); - - for command in LIFECYCLE_COMMANDS { - let lifecycle = fact(command, json!({ "zone": "pit-1" }), Some("alice")); - let (url, _) = sink.target(&lifecycle).unwrap(); - assert_eq!(url, format!("http://127.0.0.1:9/presence/alice/{command}")); - } - } - - #[test] - fn unknown_commands_and_unroutable_facts_are_permanent() { - let sink = sink(); - let unknown = fact("guild.whatever", json!({}), Some("alice")); - assert!(matches!( - sink.target(&unknown), - Err(FactError::Permanent(_)) - )); - let missing = fact("match.record_result", json!({}), None); - assert!(matches!( - sink.target(&missing), - Err(FactError::Permanent(_)) - )); - } - - #[test] - fn classifies_cell_responses() { - assert!(matches!( - classify(409, &json!({ "code": "COMMAND_IN_PROGRESS" })), - FactError::Retryable(_) - )); - assert!(matches!( - classify(503, &Value::Null), - FactError::Retryable(_) - )); - assert!(matches!( - classify(500, &json!({ "code": "INTERNAL" })), - FactError::Retryable(_) - )); - assert!(matches!( - classify(400, &json!({ "code": "BAD_REQUEST" })), - FactError::Permanent(_) - )); - assert!(matches!( - classify(401, &json!({ "code": "UNAUTHORIZED" })), - FactError::Permanent(_) - )); - assert!(matches!( - classify(422, &json!({ "code": "REJECTED" })), - FactError::Permanent(_) - )); - } - - #[tokio::test] - async fn an_unreachable_celld_is_retryable() { - let sink = sink(); - let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); - assert!(matches!( - sink.commit(&kill).await, - Err(FactError::Retryable(_)) - )); - } + use distributed::USER_ID_KEY; + use serde_json::json; #[test] fn the_cell_rebuilds_the_trusted_fact_session_from_the_headers() { @@ -824,11 +450,4 @@ mod tests { crate::cell::fact_session_from_headers(|name| headers.get(name).cloned()).is_none() ); } - - #[test] - fn rejects_bad_configuration() { - let secret = || InternalHttpSecret::new(SECRET).unwrap(); - assert!(CelldFactSink::new("ftp://x", secret(), "s", "p").is_err()); - assert!(CelldFactSink::new("http://x", secret(), " ", "p").is_err()); - } } diff --git a/distributed_game/src/command_sink.rs b/distributed_game/src/command_sink.rs new file mode 100644 index 000000000..da08e751e --- /dev/null +++ b/distributed_game/src/command_sink.rs @@ -0,0 +1,482 @@ +//! [`CommandFactSink`]: a zone server's facts dispatched as commands through +//! any [`CommandHost`] — the same durable aggregates whether they run in +//! celld cells ([`CelldCommandHost`], `POST {celld}/{kind}/{shard}/{command}` +//! over the internal path) or in an ordinary service over any repository +//! ([`LocalCommandHost`]: in-memory, SQLite, Postgres). +//! +//! One sink per zone server routes every fact by its command +//! ([`FactCommand::COMMAND`](crate::FactCommand::COMMAND)) to a registered +//! [`FactRoute`]: a cell kind and a shard function of the fact's input and +//! trusted session (principal + [`FactStamp`](crate::FactStamp)). A host +//! that addresses cells uses both ([`FactRouting`]); a service routes by +//! command and its handler picks the stream from the same session or input. +//! Player lifecycle facts (`zone.*`, [`crate::lifecycle`]) route to the +//! presence aggregate of the player's account ([`CommandFactSink::lifecycle_route`]). +//! +//! - **Exactly one commit per key.** The command id is +//! [`command_id_for`]`(service_id, fact key)`, so a retry after a timeout +//! replays through the aggregate's command ledger (`replayed` receipt). +//! - **Trusted identity.** The principal (bound by the zone host from the +//! verified ticket) is the session's `x-user-id` and the verified +//! principal ([`VerifiedPrincipal::from_trusted_transport`]); the stamp +//! travels as `x-game-*` session variables (forwarded to cells as trusted +//! metadata, [`CelldCommandHost::forward_session_prefix`]). +//! - **No local durable fallback.** A host that is slow or unreachable, a +//! 5xx, or a command still in progress is [`FactError::Retryable`]: the fact +//! stays in the zone's queue (use +//! [`RetryPolicy::until_settled`](crate::RetryPolicy::until_settled)). An +//! unrouted or rejected command is [`FactError::Permanent`]. + +use std::sync::Arc; +use std::time::Duration; + +use distributed::cell_host::{CelldCommandHost, InternalHttpSecret}; +use distributed::command_dispatch::{CommandHost, LocalCommandHost}; +use distributed::graphql::identity::VerifiedPrincipal; +use distributed::microsvc::{CausalDispatchError, Session}; +use serde_json::Value; + +use crate::cell::{command_id_for, session_stamp, trusted_session}; +use crate::facts::{BoxFuture, Fact, FactCommand, FactError, FactReceipt, FactSink}; +use crate::lifecycle::LIFECYCLE_COMMANDS; +use crate::stamp::SESSION_PREFIX; + +type ShardFn = Arc Option + Send + Sync>; + +/// Where facts of one command go: an aggregate kind and a shard function of +/// the fact's input and trusted session. +#[derive(Clone)] +pub struct FactRoute { + command: String, + kind: String, + shard: ShardFn, +} + +impl FactRoute { + /// Route `command` to `{kind}/{shard(input, session)}`. The session holds + /// the fact's principal (`user_id()`) and stamp + /// ([`session_stamp`]). Return `None` when the fact cannot be routed. + pub fn new( + command: impl Into, + kind: impl Into, + shard: impl Fn(&Value, &Session) -> Option + Send + Sync + 'static, + ) -> Self { + Self { + command: command.into(), + kind: kind.into(), + shard: Arc::new(shard), + } + } + + /// Route the typed fact `C` to the aggregate of its principal. + pub fn by_principal(kind: impl Into) -> Self { + Self::new(C::COMMAND, kind, |_, session| { + session.user_id().map(str::to_string) + }) + } + + /// Route the typed fact `C` by a field of its input. + pub fn by_input_field(kind: impl Into, field: &'static str) -> Self { + Self::new(C::COMMAND, kind, move |input, _| { + match input.get(field)? { + Value::String(value) => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + _ => None, + } + }) + } + + /// Route the typed fact `C` to the aggregate of the zone that emitted it. + pub fn by_zone(kind: impl Into) -> Self { + Self::new(C::COMMAND, kind, |_, session| { + session_stamp(session).map(|stamp| stamp.zone_id) + }) + } + + /// The routed command. + pub fn command(&self) -> &str { + &self.command + } + + /// The aggregate kind (cell URL segment). + pub fn kind(&self) -> &str { + &self.kind + } + + /// The shard of a fact with `input` and `session`. + pub fn shard(&self, input: &Value, session: &Session) -> Option { + (self.shard)(input, session) + } +} + +/// A [`CommandHost`] that learns the fact routes it needs. +pub trait FactRouting { + /// Register `route` (a host that addresses aggregates by kind and shard + /// records it; one that routes by command ignores it). + fn register_fact_route(&mut self, route: &FactRoute); +} + +/// A service routes by command: its handler selects the stream. +impl FactRouting for LocalCommandHost { + fn register_fact_route(&mut self, _route: &FactRoute) {} +} + +/// celld addresses cells: `{kind}/{shard}/{command}`. +impl FactRouting for CelldCommandHost { + fn register_fact_route(&mut self, route: &FactRoute) { + let shard = route.shard.clone(); + self.add_cell_route(route.command(), route.kind(), move |input, session| { + shard(input, session) + }); + } +} + +/// See the module docs. +pub struct CommandFactSink { + host: Arc, + service_id: String, + system_principal: String, + routes: Vec, +} + +impl Clone for CommandFactSink { + fn clone(&self) -> Self { + Self { + host: self.host.clone(), + service_id: self.service_id.clone(), + system_principal: self.system_principal.clone(), + routes: self.routes.clone(), + } + } +} + +impl CommandFactSink { + /// A sink dispatching through `host` under command-ledger `service_id`; + /// zone facts without a player use `system_principal`. + pub fn new( + host: H, + service_id: impl Into, + system_principal: impl Into, + ) -> Result { + let service_id = service_id.into(); + let system_principal = system_principal.into(); + if service_id.trim().is_empty() || system_principal.trim().is_empty() { + return Err("a fact sink needs a service id and a system principal".into()); + } + Ok(Self { + host: Arc::new(host), + service_id, + system_principal, + routes: Vec::new(), + }) + } + + /// Add a route (before the sink is cloned). A later route for the same + /// command replaces the earlier. + pub fn route(mut self, route: FactRoute) -> Self { + Arc::get_mut(&mut self.host) + .expect("add fact routes before cloning the sink") + .register_fact_route(&route); + self.routes + .retain(|existing| existing.command != route.command); + self.routes.push(route); + self + } + + /// Route every player lifecycle fact (`zone.*`) to the presence + /// aggregate `{kind}/{principal}` (the player's account). + pub fn lifecycle_route(mut self, kind: impl Into) -> Self { + let kind = kind.into(); + for command in LIFECYCLE_COMMANDS { + self = self.route(FactRoute::new(command, kind.clone(), |_, session| { + session.user_id().map(str::to_string) + })); + } + self + } +} + +impl CommandFactSink { + /// The host. + pub fn host(&self) -> &H { + &self.host + } + + /// The route a fact would take, if any. + pub fn route_for(&self, command: &str) -> Option<&FactRoute> { + self.routes.iter().find(|route| route.command == command) + } + + /// `(principal, trusted session, command id)` of a routed fact. + pub fn prepare(&self, fact: &Fact) -> Result<(String, Session, String), FactError> { + let route = self.route_for(fact.command()).ok_or_else(|| { + FactError::Permanent(format!("no route for fact command `{}`", fact.command())) + })?; + let principal = fact + .principal() + .unwrap_or(&self.system_principal) + .to_string(); + let session = trusted_session(&principal, fact); + route + .shard(fact.input(), &session) + .filter(|shard| { + !shard.is_empty() + && shard.len() <= 256 + && shard.bytes().all(|byte| (0x21..=0x7e).contains(&byte)) + }) + .ok_or_else(|| { + FactError::Permanent(format!( + "fact `{}` has no valid `{}` shard", + fact.key(), + route.kind + )) + })?; + Ok(( + principal, + session, + command_id_for(&self.service_id, fact.key()), + )) + } +} + +/// celld slow, down or still running the command: keep the fact queued. +fn classify(error: CausalDispatchError) -> FactError { + let detail = format!("{}: {error:?}", error.code()); + match &error { + CausalDispatchError::InProgress | CausalDispatchError::Internal(_) => { + FactError::Retryable(detail) + } + CausalDispatchError::Rejected { status, .. } + if *status >= 500 || matches!(*status, 408 | 429) => + { + FactError::Retryable(detail) + } + CausalDispatchError::Handler(handler) if handler.status_code() >= 500 => { + FactError::Retryable(detail) + } + _ => FactError::Permanent(detail), + } +} + +impl FactSink for CommandFactSink { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (principal, session, command_id) = self.prepare(fact)?; + let result = self + .host + .invoke( + fact.command(), + &command_id, + fact.input().clone(), + session, + VerifiedPrincipal::from_trusted_transport(&principal), + None, + ) + .await + .map_err(classify)?; + Ok(FactReceipt { + payload: result.payload().clone(), + replayed: result.replayed(), + }) + }) + } +} + +/// Default per-request timeout of [`CelldFactSink`]. +pub const DEFAULT_CELLD_FACT_TIMEOUT: Duration = Duration::from_secs(10); + +/// Facts into celld cells over the internal path. +pub type CelldFactSink = CommandFactSink; + +impl CommandFactSink { + /// A sink for the cells at `celld_url` under command-ledger + /// `service_id`: a remote-only [`CelldCommandHost`] forwarding the fact + /// stamp (`x-game-*`). + pub fn celld( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + system_principal: impl Into, + ) -> Result { + Self::celld_with_timeout( + celld_url, + secret, + service_id, + system_principal, + DEFAULT_CELLD_FACT_TIMEOUT, + ) + } + + /// As [`celld`](Self::celld) with an explicit per-request timeout. + pub fn celld_with_timeout( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + system_principal: impl Into, + timeout: Duration, + ) -> Result { + let service_id = service_id.into(); + let host = CelldCommandHost::remote(celld_url, &service_id, secret, timeout) + .map_err(|error| format!("celld command host: {error:?}"))? + .forward_session_prefix(SESSION_PREFIX) + .map_err(|error| format!("celld command host: {error:?}"))?; + Self::new(host, service_id, system_principal) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::stamp::FactStamp; + use serde_json::json; + + const SECRET: &str = "test-only-internal-secret-change-me-2026"; + + #[derive(serde::Serialize)] + struct Kill { + mob: String, + } + impl FactCommand for Kill { + const COMMAND: &'static str = "crawler.record_kill"; + } + #[derive(serde::Serialize)] + struct Result_ { + match_id: String, + } + impl FactCommand for Result_ { + const COMMAND: &'static str = "match.record_result"; + } + + fn sink() -> CelldFactSink { + CelldFactSink::celld( + "http://127.0.0.1:9/", + InternalHttpSecret::new(SECRET).unwrap(), + "zone-server", + "zone:pit-1", + ) + .unwrap() + .route(FactRoute::by_principal::("crawler")) + .route(FactRoute::by_input_field::("match", "match_id")) + .lifecycle_route("presence") + } + + fn fact(command: &str, input: Value, principal: Option<&str>) -> Fact { + Fact::trusted( + format!("{command}:pit-1:1:0"), + command, + input, + principal.map(str::to_string), + FactStamp { + zone_id: "pit-1".into(), + ..FactStamp::default() + }, + ) + } + + fn shard(sink: &CelldFactSink, fact: &Fact) -> Option { + let (_, session, _) = sink.prepare(fact).ok()?; + sink.route_for(fact.command())? + .shard(fact.input(), &session) + } + + #[test] + fn routes_by_command_to_kind_and_shard_with_a_stable_command_id() { + let sink = sink(); + let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); + let (principal, session, id) = sink.prepare(&kill).unwrap(); + assert_eq!(principal, "alice"); + assert_eq!(session.user_id(), Some("alice")); + assert_eq!(session_stamp(&session).unwrap().zone_id, "pit-1"); + assert_eq!(id, command_id_for("zone-server", kill.key())); + assert_eq!(sink.prepare(&kill).unwrap().2, id, "stable across retries"); + assert_eq!(sink.route_for(kill.command()).unwrap().kind(), "crawler"); + assert_eq!(shard(&sink, &kill).as_deref(), Some("alice")); + + let result = fact("match.record_result", json!({ "match_id": "m/7" }), None); + assert_eq!(shard(&sink, &result).as_deref(), Some("m/7")); + assert_eq!( + sink.prepare(&result).unwrap().0, + "zone:pit-1", + "zone facts use the system principal" + ); + let spaced = fact("match.record_result", json!({ "match_id": "m 7" }), None); + assert!(matches!( + sink.prepare(&spaced), + Err(FactError::Permanent(_)) + )); + + for command in LIFECYCLE_COMMANDS { + let lifecycle = fact(command, json!({ "zone": "pit-1" }), Some("alice")); + assert_eq!(sink.route_for(command).unwrap().kind(), "presence"); + assert_eq!(shard(&sink, &lifecycle).as_deref(), Some("alice")); + } + let by_zone = FactRoute::by_zone::("zone-log"); + let (_, session, _) = sink.prepare(&kill).unwrap(); + assert_eq!( + by_zone.shard(&json!({}), &session).as_deref(), + Some("pit-1") + ); + } + + #[test] + fn unknown_commands_and_unroutable_facts_are_permanent() { + let sink = sink(); + let unknown = fact("guild.whatever", json!({}), Some("alice")); + assert!(matches!( + sink.prepare(&unknown), + Err(FactError::Permanent(_)) + )); + let missing = fact("match.record_result", json!({}), None); + assert!(matches!( + sink.prepare(&missing), + Err(FactError::Permanent(_)) + )); + } + + #[test] + fn classifies_dispatch_errors() { + let retryable = |error| matches!(classify(error), FactError::Retryable(_)); + assert!(retryable(CausalDispatchError::InProgress)); + assert!(retryable(CausalDispatchError::Internal("down".into()))); + assert!(retryable(CausalDispatchError::Rejected { + code: "REJECTED", + status: 503, + message: String::new() + })); + assert!(retryable(CausalDispatchError::Rejected { + code: "REJECTED", + status: 429, + message: String::new() + })); + assert!(!retryable(CausalDispatchError::BadRequest("bad".into()))); + assert!(!retryable(CausalDispatchError::Forbidden)); + assert!(!retryable(CausalDispatchError::Rejected { + code: "UNAUTHORIZED", + status: 401, + message: String::new() + })); + assert!(!retryable(CausalDispatchError::Rejected { + code: "REJECTED", + status: 422, + message: String::new() + })); + } + + #[tokio::test] + async fn an_unreachable_host_is_retryable() { + let sink = sink(); + let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); + assert!(matches!( + sink.commit(&kill).await, + Err(FactError::Retryable(_)) + )); + } + + #[test] + fn rejects_bad_configuration() { + let secret = || InternalHttpSecret::new(SECRET).unwrap(); + assert!(CelldFactSink::celld("ftp://x", secret(), "s", "p").is_err()); + assert!(CelldFactSink::celld("http://x", secret(), " ", "p").is_err()); + assert!(CelldFactSink::celld("http://x", secret(), "s", " ").is_err()); + } +} diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index 03f0e958a..721d48d87 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -59,6 +59,8 @@ pub mod celld; pub mod claims; pub mod clock; pub mod codec; +#[cfg(feature = "command-host")] +pub mod command_sink; pub mod entity; pub mod facts; pub mod handoff; diff --git a/distributed_game/src/world/zone_lease.rs b/distributed_game/src/world/zone_lease.rs index 1d7d767a6..626b4a807 100644 --- a/distributed_game/src/world/zone_lease.rs +++ b/distributed_game/src/world/zone_lease.rs @@ -6,32 +6,35 @@ //! //! | Command | Effect | Outward event | //! |---|---|---| -//! | `zone_lease.acquire` | new holder, token + 1, alarm at expiry | `zone.activated` | -//! | `zone_lease.renew` | later expiry, alarm re-armed | none (no heartbeat events) | -//! | `zone_lease.release` | no holder, alarm cancelled | `zone.released` | -//! | `zone_lease.expire` (alarm only) | no holder, if still expired | `zone.lease_expired` | +//! | `zone_lease.acquire` | new holder, token + 1, expiry scheduled | `zone.activated` | +//! | `zone_lease.renew` | later expiry, expiry rescheduled | none (no heartbeat events) | +//! | `zone_lease.release` | no holder, expiry cancelled | `zone.released` | +//! | `zone_lease.expire` (schedule only) | no holder, if still expired | `zone.lease_expired` | //! //! Outward events carry the lease's post-transition [`ZoneLeaseStatus`] //! (zone, holder, token, `active` / `released` / `expired`); the //! [`zone_status`](super::zone_status) read model projects them. //! //! Fencing tokens increase on every acquisition and are never reset. Expiry -//! is declared by the cell's alarm ([`distributed::cell_host::CellAlarm`]): -//! the alarm handler re-reads the lease and expires it only when the -//! holder's expiry has passed by the cell clock, so a renewal that races the -//! alarm never produces a false expiry. All times are the cell's clock -//! (`ctx.cell_now()`), never the caller's. +//! is declared by a schedule ([`distributed::schedule::Schedule`]): the +//! expiry command re-reads the lease and expires it only when the holder's +//! expiry has passed by the host clock, so a renewal that races it never +//! produces a false expiry. All times are the host's clock (`ctx.now()`), +//! never the caller's. The commands declare `Capability::Scheduling` and +//! `Capability::Clock`: they mount on aggregate cells (celld; the reference +//! host in tests) and on any backend that dispatches schedules. //! //! Callers must hold the `zone-host` role (an internal identity; the //! commands are not routed through the public GraphQL edge). use std::time::{Duration, SystemTime}; -use distributed::cell_host::{cell_alarm_guard, AggregateCell, CellAlarm}; +use distributed::cell_host::AggregateCell; use distributed::command::{typed_command, PreparedCommand, Succeeded}; use distributed::microsvc::{ CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, }; +use distributed::schedule::{schedule_guard, Capability, Schedule}; use distributed::{sourced, Entity}; use serde::{Deserialize, Serialize}; @@ -51,7 +54,7 @@ pub const EXPIRE: &str = "zone_lease.expire"; pub const HELD_PREFIX: &str = "held:"; /// Rejection message when a claim is no longer the live one. pub const LOST: &str = "lost"; -/// The expiry alarm's name. +/// The expiry schedule's name. pub const EXPIRY_ALARM: &str = "expiry"; /// [`ZoneLeaseStatus::status`] of a held lease. @@ -188,7 +191,7 @@ pub struct ReleaseInput { pub token: u64, } -/// `zone_lease.expire` input (armed by the cell itself). +/// `zone_lease.expire` input (scheduled by the lease itself). #[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] pub struct ExpireInput { pub zone_id: String, @@ -203,7 +206,7 @@ pub struct ClaimPayload { pub expires_at_ms: u64, } -/// What an expiry alarm did. +/// What an expiry schedule did. #[derive(Debug, Clone, PartialEq, Eq, Serialize, distributed::CommandOutput)] pub struct ExpiryPayload { pub expired: bool, @@ -233,10 +236,10 @@ fn is_zone_host(ctx: &CausalCommandContext<'_, ZoneLease>) -> bool { } fn now_ms(ctx: &CausalCommandContext<'_, ZoneLease>) -> Result { - ctx.cell_now() - .and_then(|now| now.duration_since(SystemTime::UNIX_EPOCH).ok()) + ctx.now()? + .duration_since(SystemTime::UNIX_EPOCH) .map(|now| now.as_millis() as u64) - .ok_or_else(|| rejected("zone leases run only in an aggregate cell")) + .map_err(|_| rejected("host clock is before the Unix epoch")) } fn ttl(ttl_ms: u64) -> Result { @@ -253,8 +256,8 @@ fn validate(zone_id: &str, holder: &str) -> Result<(), HandlerError> { Ok(()) } -fn expiry_alarm(zone_id: &str, expires_at_ms: u64) -> Result { - CellAlarm::at( +fn expiry_schedule(zone_id: &str, expires_at_ms: u64) -> Result { + Schedule::at( EXPIRY_ALARM, SystemTime::UNIX_EPOCH + Duration::from_millis(expires_at_ms), EXPIRE, @@ -288,7 +291,7 @@ async fn handle_acquire( }; return repo.commit(lease)?.succeeded(claim); } - // Held until the expiry alarm declares otherwise. + // Held until the expiry schedule declares otherwise. Some(holder) => return Err(rejected(format!("{HELD_PREFIX}{holder}"))), None => {} } @@ -302,7 +305,7 @@ async fn handle_acquire( expires_at_ms, ) .map_err(rejected)?; - ctx.arm_alarm(expiry_alarm(&input.zone_id, expires_at_ms)?)?; + ctx.schedule(expiry_schedule(&input.zone_id, expires_at_ms)?)?; repo.publish_events() .commit(lease)? .succeeded(ClaimPayload { @@ -330,7 +333,7 @@ async fn handle_renew( } let expires_at_ms = now + ttl; lease.renewed(expires_at_ms).map_err(rejected)?; - ctx.arm_alarm(expiry_alarm(&input.zone_id, expires_at_ms)?)?; + ctx.schedule(expiry_schedule(&input.zone_id, expires_at_ms)?)?; // Internal state only: no outward event, so no outbox row. repo.commit(lease)?.succeeded(ClaimPayload { zone_id: input.zone_id, @@ -363,7 +366,7 @@ async fn handle_release( lease .released(input.zone_id, input.holder, input.token) .map_err(rejected)?; - ctx.cancel_alarm(EXPIRY_ALARM)?; + ctx.cancel_schedule(EXPIRY_ALARM)?; repo.publish_events().commit(lease)?.succeeded(claim) } @@ -384,7 +387,7 @@ async fn handle_expire( .map_err(rejected)?; true } - // Renewed (or released) after this alarm was read: not expired. + // Renewed (or released) after this schedule was read: not expired. _ => false, }; if expired { @@ -406,6 +409,8 @@ macro_rules! lease_command { fn install(self, routes: Routes) -> Routes { routes .typed_command(typed_command::<$input, Succeeded<$output>>($command)) + .requires(Capability::Scheduling) + .requires(Capability::Clock) .guarded($guard, $handler) } } @@ -441,7 +446,7 @@ lease_command!( EXPIRE, ExpireInput, ExpiryPayload, - cell_alarm_guard::, + schedule_guard::, handle_expire ); diff --git a/distributed_game/src/world/zone_state.rs b/distributed_game/src/world/zone_state.rs index 0c705a383..f6a72dbdb 100644 --- a/distributed_game/src/world/zone_state.rs +++ b/distributed_game/src/world/zone_state.rs @@ -589,13 +589,7 @@ pub mod celld_worker { ::worker::console_error!("superseded zone-state blob {key}: {error}"); } } - json_status( - json!({ - "payload": dispatch.payload(), - "receipt": { "commandId": dispatch.command_id(), "replayed": dispatch.replayed() }, - }), - 200, - ) + json_status(dispatch.wait_path_wire(), 200) } Err(error) => { if let CellDispatchError::Internal(message) = &error { diff --git a/distributed_game/tests/celld_facts/main.rs b/distributed_game/tests/celld_facts/main.rs index 570f96acf..2e61cedc8 100644 --- a/distributed_game/tests/celld_facts/main.rs +++ b/distributed_game/tests/celld_facts/main.rs @@ -14,7 +14,7 @@ use std::time::{Duration, Instant}; use distributed::cell_host::InternalHttpSecret; -use distributed_game::celld::{CelldFactRoute, CelldFactSink}; +use distributed_game::celld::{CelldFactSink, FactRoute}; use distributed_game::lifecycle::{JoinVia, LeftReason, PlayerJoined, PlayerLeft}; use distributed_game::{ fact_channel, spawn_fact_worker, EmitError, Fact, FactCommand, FactError, FactOutcome, @@ -41,7 +41,7 @@ struct MatchResult { } fn sink(base: &str, timeout: Duration) -> CelldFactSink { - CelldFactSink::with_timeout( + CelldFactSink::celld_with_timeout( base, InternalHttpSecret::new(SECRET).unwrap(), SERVICE, @@ -49,8 +49,8 @@ fn sink(base: &str, timeout: Duration) -> CelldFactSink { timeout, ) .unwrap() - .route(CelldFactRoute::by_principal::("kills")) - .route(CelldFactRoute::by_input_field::( + .route(FactRoute::by_principal::("kills")) + .route(FactRoute::by_input_field::( "match", "match_id", )) .lifecycle_route("presence") diff --git a/distributed_game/tests/command_facts/main.rs b/distributed_game/tests/command_facts/main.rs new file mode 100644 index 000000000..da31577ac --- /dev/null +++ b/distributed_game/tests/command_facts/main.rs @@ -0,0 +1,286 @@ +//! The fact routes a zone server uses with celld (`tests/celld_facts`), on +//! an ordinary service instead: `CommandFactSink` over an +//! in-memory repository. Kills land in the killer's stream, match results in +//! the match's stream, lifecycle facts in the player's — no cell involved — +//! with the trusted principal and stamp, exactly once per fact key. + +use std::sync::Arc; + +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, Service, +}; +use distributed::{sourced, AggregateRepository, Entity, InMemoryRepository, LocalCommandHost}; +use distributed_game::cell::session_stamp; +use distributed_game::command_sink::{CommandFactSink, FactRoute}; +use distributed_game::lifecycle::{JoinVia, PlayerJoined, LIFECYCLE_COMMANDS}; +use distributed_game::{Fact, FactCommand, FactError, FactSink, FactStamp}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Serialize, FactCommand)] +#[fact(command = "kills.record")] +struct Kill { + mob: String, +} + +#[derive(Serialize, FactCommand)] +#[fact(command = "match.record_result")] +struct MatchResult { + match_id: String, + winner_slot: u32, +} + +/// One stream per routed shard: a player's kills and lifecycle, a match. +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +struct Log { + entity: Entity, + entries: Vec, +} + +#[sourced(entity, aggregate_type = "Log", events = "LogEvent")] +impl Log { + #[event("log.recorded", version = 1)] + fn recorded(&mut self, id: String, entry: String) { + if self.entity.id().is_empty() { + self.entity.set_id(id); + } + self.entries.push(entry); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +struct KillInput { + mob: String, +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +struct ResultInput { + match_id: String, + winner_slot: u32, +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +#[allow(dead_code)] +struct LifecycleInput { + zone: String, + via: Option, + from_zone: Option, + reason: Option, + grace_ms: Option, + cause: Option, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +struct Recorded { + stream: String, + entries: usize, +} + +fn rejected(error: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(error.to_string()) +} + +/// Append `entry` to stream `id` (the shard the route would address). +async fn record( + ctx: &CausalCommandContext<'_, Log>, + id: String, + entry: String, +) -> Result>, HandlerError> { + let repo = ctx.repo(); + let mut log = match repo.get(&id).await? { + Some(log) => log, + None => repo.create(), + }; + log.recorded(id.clone(), entry).map_err(rejected)?; + let entries = log.entries.len(); + repo.commit(log)?.succeeded(Recorded { + stream: id, + entries, + }) +} + +async fn handle_kill( + ctx: &CausalCommandContext<'_, Log>, + input: KillInput, +) -> Result>, HandlerError> { + // Same shard as `FactRoute::by_principal`: the trusted session's user. + let principal = ctx.user_id()?.to_string(); + let zone = session_stamp(ctx.session()) + .ok_or_else(|| rejected("no stamp"))? + .zone_id; + record( + ctx, + format!("kills/{principal}"), + format!("{}@{zone}", input.mob), + ) + .await +} + +async fn handle_result( + ctx: &CausalCommandContext<'_, Log>, + input: ResultInput, +) -> Result>, HandlerError> { + // Same shard as `FactRoute::by_input_field("match_id")`. + record( + ctx, + format!("match/{}", input.match_id), + format!("winner {}", input.winner_slot), + ) + .await +} + +async fn handle_joined( + ctx: &CausalCommandContext<'_, Log>, + input: LifecycleInput, +) -> Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + record( + ctx, + format!("presence/{principal}"), + format!("joined {}", input.zone), + ) + .await +} + +struct Commands; +impl PortableCommand for Commands +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + Kill::COMMAND, + )) + .handle(handle_kill) + .typed_command(typed_command::>( + MatchResult::COMMAND, + )) + .handle(handle_result) + .typed_command(typed_command::>( + LIFECYCLE_COMMANDS[0], + )) + .handle(handle_joined) + } +} + +fn stamp(tick: u64) -> FactStamp { + FactStamp { + zone_id: "pit-1".into(), + tick, + content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_000 + tick, + activation: 1, + config_version: "cfg-1".into(), + seq: tick, + } +} + +fn fact( + key: &str, + input: &C, + principal: Option<&str>, + tick: u64, +) -> Fact { + Fact::trusted( + key, + C::COMMAND, + serde_json::to_value(input).unwrap(), + principal.map(str::to_string), + stamp(tick), + ) +} + +#[tokio::test] +async fn the_celld_fact_routes_land_on_a_service_over_any_repository() { + let repo = InMemoryRepository::new(); + let routes = Commands.install(Routes::new().with_repo(AggregateRepository::new(repo.clone()))); + let service = Arc::new(Service::new().named("zone-server-test").routes(routes)); + // The same routes as the celld fact-routing test. + let sink = CommandFactSink::new( + LocalCommandHost::new(service), + "zone-server-test", + "zone:pit-1", + ) + .unwrap() + .route(FactRoute::by_principal::("kills")) + .route(FactRoute::by_input_field::( + "match", "match_id", + )) + .lifecycle_route("presence"); + + let kill = fact("kill:1", &Kill { mob: "m1".into() }, Some("alice"), 1); + let first = sink.commit(&kill).await.unwrap(); + assert!(!first.replayed); + assert_eq!( + first.payload, + json!({ "stream": "kills/alice", "entries": 1 }) + ); + let again = sink.commit(&kill).await.unwrap(); + assert!(again.replayed, "same fact key replays through the ledger"); + assert_eq!(again.payload, first.payload); + let second = sink + .commit(&fact( + "kill:2", + &Kill { mob: "m2".into() }, + Some("alice"), + 2, + )) + .await + .unwrap(); + assert_eq!(second.payload["entries"], 2, "credited once per key"); + + let result = sink + .commit(&fact( + "result:1", + &MatchResult { + match_id: "m-7".into(), + winner_slot: 1, + }, + None, + 3, + )) + .await + .unwrap(); + assert_eq!(result.payload["stream"], "match/m-7"); + + let joined = PlayerJoined { + zone: "pit-1".into(), + via: JoinVia::Ticket, + from_zone: None, + }; + let lifecycle = Fact::trusted( + "joined:1", + LIFECYCLE_COMMANDS[0], + serde_json::to_value(&joined).unwrap(), + Some("alice".into()), + stamp(4), + ); + let presence = sink.commit(&lifecycle).await.unwrap(); + assert_eq!(presence.payload["stream"], "presence/alice"); + + // Unrouted commands are permanent; so is a command the service rejects. + let unrouted = Fact::trusted( + "x", + "guild.whatever", + json!({}), + Some("alice".into()), + stamp(5), + ); + assert!(matches!( + sink.commit(&unrouted).await, + Err(FactError::Permanent(_)) + )); + let forged = Fact::trusted( + "kill:3", + Kill::COMMAND, + json!({ "mob": "m3", "xp": 9999 }), + Some("alice".into()), + stamp(6), + ); + assert!(matches!( + sink.commit(&forged).await, + Err(FactError::Permanent(_)) + )); +} diff --git a/distributed_game/tests/zone_lease/main.rs b/distributed_game/tests/zone_lease/main.rs index b45598afc..3c2ceaf50 100644 --- a/distributed_game/tests/zone_lease/main.rs +++ b/distributed_game/tests/zone_lease/main.rs @@ -7,9 +7,10 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use distributed::cell_host::{ - cell_alarm_session, AggregateCell, CellCommandIdentity, CellDispatchError, CellStreamStore, + AggregateCell, CellCommandIdentity, CellDispatchError, CellStreamStore, }; use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; +use distributed::schedule::schedule_session; use distributed::{Clock, ManualClock, StreamIdentity}; use distributed_game::cell::command_id_for; use distributed_game::world::zone_lease::{ @@ -168,18 +169,18 @@ async fn renewals_produce_no_outbox_rows_and_move_the_expiry_alarm() { let mut current = own.acquire("z1", "a#1", ttl).await.unwrap(); assert_eq!(outward(&own.cell), ["zone.activated"]); assert_eq!( - own.cell.next_alarm_due().unwrap(), + own.cell.next_schedule_due().unwrap(), Some(SystemTime::UNIX_EPOCH + Duration::from_millis(T0 + 10_000)) ); for step in 1..=20u64 { clock.advance(Duration::from_secs(3)); current = own.renew(¤t, ttl).await.unwrap(); assert_eq!( - own.cell.next_alarm_due().unwrap(), + own.cell.next_schedule_due().unwrap(), Some(clock.now() + ttl), "step {step}" ); - assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); } assert_eq!( outward(&own.cell), @@ -199,14 +200,14 @@ async fn a_holder_that_stops_renewing_expires_exactly_once() { let ttl = Duration::from_secs(10); let held = own.acquire("z2", "a#1", ttl).await.unwrap(); clock.advance(Duration::from_millis(9_999)); - assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); clock.advance(Duration::from_millis(1)); - let report = own.cell.fire_due_alarms().await.unwrap(); + let report = own.cell.fire_due_schedules().await.unwrap(); assert_eq!(report.fired.len(), 1); assert_eq!(report.fired[0].payload["expired"], true); assert_eq!(outward(&own.cell), ["zone.activated", "zone.lease_expired"]); clock.advance(Duration::from_secs(60)); - assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); assert_eq!(outward(&own.cell).len(), 2, "exactly one expiry"); assert_eq!(own.renew(&held, ttl).await, Err(OwnershipError::Lost)); let next = own.acquire("z2", "b#1", ttl).await.unwrap(); @@ -223,12 +224,12 @@ async fn a_renewal_racing_the_alarm_never_produces_a_false_expiry() { // The alarm comes due, but the renewal lands before the wake. clock.advance(ttl); let held = own.renew(&held, ttl).await.unwrap(); - assert!(own.cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); // The wake read the due alarm, then the renewal interleaved, then the // stale expiry command runs: the handler re-reads the lease and keeps it. clock.advance(ttl); - let stale = own.cell.armed_alarms().unwrap().remove(0); + let stale = own.cell.armed_schedules().unwrap().remove(0); let _held = own.renew(&held, ttl).await.unwrap(); let result = own .cell @@ -236,7 +237,7 @@ async fn a_renewal_racing_the_alarm_never_produces_a_false_expiry() { EXPIRE, &stale.command_identity().unwrap(), stale.input.clone(), - cell_alarm_session(&stale), + schedule_session(&stale), ) .await .unwrap(); diff --git a/docs/coordination-primitives.md b/docs/coordination-primitives.md index 2c88d2fcf..d5d168282 100644 --- a/docs/coordination-primitives.md +++ b/docs/coordination-primitives.md @@ -46,55 +46,77 @@ retry) resets availability to now, so a timer that failed to publish is retried promptly instead of being re-delayed. `OutboxMessage.deliver_at` is `#[serde(default)]`. -## Cell alarms: a cell schedules its own work +## Schedules: an aggregate schedules its own work -`deliver_at` leaves the cell and needs a consumer to send a command back. -When a cell must act on itself later (a matchmaking tick, an invite expiry, a -daily reset, a lease expiry), a causal command handler arms a **cell alarm** -with its events: +`deliver_at` leaves the aggregate and needs a consumer to send a command +back. When an aggregate must act on itself later (a matchmaking tick, an +invite expiry, a daily reset, a lease expiry), a causal command handler +**schedules** it with its events (`distributed::schedule`): ```rust -// in a causal command handler -ctx.arm_alarm(CellAlarm::after("tick", Duration::from_secs(5), "pool.tick", &TickInput { pool })?)?; -ctx.cancel_alarm("invite-expiry")?; +// on the route: what the command needs from its host +routes.typed_command(start).requires(Capability::Scheduling).requires(Capability::Clock) -// in the alarm's own handler: re-arm from the fired due time (no drift) -let due = ctx.alarm_due().expect("fired by an alarm") + period; -ctx.arm_alarm(CellAlarm::at("tick", due, "pool.tick", &input)?)?; +// in the handler +let now = ctx.now()?; // the host clock +ctx.schedule(Schedule::after("tick", Duration::from_secs(5), "pool.tick", &TickInput { pool })?)?; +ctx.cancel_schedule("invite-expiry")?; + +// in the schedule's own handler: re-schedule from the fired due time (no drift) +let due = ctx.scheduled_due().expect("fired by a schedule") + period; +ctx.schedule(Schedule::at("tick", due, "pool.tick", &input)?)?; ``` -- The alarm row (`cell_alarms`, migration 11) commits in the **same - transaction** as the command's events, ledger receipt and outbox rows. A - failed command arms nothing. `CellAlarm::after` resolves against the - cell's clock at commit; handlers have no wall clock of their own. -- One row per alarm name. Arming a name again replaces it and bumps its - `generation`; `cancel_alarm` disarms it. -- When due, the cell dispatches the alarm's command **into itself** through - its command ledger, with a command id derived from `(name, generation)` and - the trusted session from `cell_alarm_session` (role `cell-alarm`, plus - claims only the host sets). Mount alarm-only commands with - `guard: cell_alarm_guard` so callers cannot invoke them. A crash after the - handler committed but before the alarm settled replays the receipt; the - handler never runs twice for one arm. -- Rejections disarm the alarm (reported as `failed`). Retryable failures back - off from 1 s, doubling, up to 5 min (`retried`). At most 32 alarms fire per - wake, 16 alarm writes per command and 1,024 armed alarms per cell. -- Only aggregate cells have an alarm slot. Every other repository rejects a - batch carrying alarm writes. +- Schedule rows are part of `CommitBatch` (`batch.schedules`) and commit in + the **same transaction** as the command's events, ledger receipt and + outbox rows, on every backend: a cell's own `cell_alarms` table (migration + 11), the in-memory repository's per-stream table, SQL's + `aggregate_schedules` (migration 12, keyed by the owning stream). A failed + command schedules nothing. The batch must commit exactly one aggregate, + which owns the schedule. `Schedule::after` resolves against the backend's + clock at commit (cell clock, injected `Clock`, database clock). +- One row per (owner, name). Scheduling a name again replaces it and bumps + its `generation`; `cancel_schedule` disarms it. +- When due, the host dispatches the command **into the owner** through its + command ledger, with a command id derived from `(name, generation)` and + the trusted session from `schedule_session` (role `schedule`, plus claims + only the host sets). Guard schedule-only commands with `schedule_guard` + so callers cannot invoke them. A crash after the handler committed but + before the schedule settled replays the receipt; the handler never runs + twice for one arm. +- Rejections disarm the schedule (reported as `failed`). Retryable failures + back off from 1 s, doubling, up to 5 min (`retried`). At most 32 schedules + fire per wake, 16 schedule writes per command and 1,024 armed schedules + per aggregate. + +**Capabilities.** Backends declare what they provide (`Capabilities`): + +| Backend | Stores schedules | Dispatches them (`Scheduling`) | Clock (`Clock`) | +|---|---|---|---| +| Aggregate cell (celld; reference host) | `cell_alarms` | yes: the Durable Object alarm; `fire_due_schedules` on the reference host | DO clock / store clock | +| In-memory repository | per-stream table | not yet | injected `Clock` (`with_clock`) | +| SQL (Postgres, SQLite) | `aggregate_schedules` | not yet (schedule worker planned) | database clock | + +A route declares what it needs with `.requires(..)`; registering it on a +backend without it panics with the missing capabilities while the service is +built, never at the first commit. `ctx.now()` answers only on routes that +declared `Clock` (the dispatcher stamps the host clock into the session and +removes any caller value); `ctx.schedule` refuses unless the route declared +`Scheduling`. **celld.** A Durable Object has one alarm. The outbox watchdog -(`CelldOutbox`) and domain alarms share it: every `setAlarm` targets the -earliest of the next domain alarm and the next drain, so a watchdog rearm -never postpones a domain alarm and a far domain alarm never postpones a -drain. The Worker's `alarm()` handler calls `AggregateCell::alarm(&env)`, -which fires due alarms, then drains the outbox and re-plans the wake. -Commands that arm an alarm move the wake earlier right after they commit. +(`CelldOutbox`) and due schedules share it: every `setAlarm` targets the +earliest of the next schedule and the next drain, so a watchdog rearm never +postpones a schedule and a far schedule never postpones a drain. The +Worker's `alarm()` handler calls `AggregateCell::alarm(&env)`, which fires +due schedules, then drains the outbox and re-plans the wake. Commands that +schedule move the wake earlier right after they commit. **Reference host.** The in-process host (unit tests; durable cells run on celld) has no timer. Give the store a `ManualClock` (`CellStreamStore::with_clock`), advance it and call -`AggregateCell::fire_due_alarms()`: the same ledger and settlement path runs -deterministically. +`AggregateCell::fire_due_schedules()`: the same ledger and settlement path +runs deterministically. ## Leases with fencing tokens diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs index 0239c1622..d2a4cf4af 100644 --- a/examples/game/cells/src/lib.rs +++ b/examples/game/cells/src/lib.rs @@ -497,13 +497,7 @@ where worker::console_error!("post-commit drain deferred: {error}"); } match result { - Ok(dispatch) => json_status( - json!({ - "payload": dispatch.payload(), - "receipt": { "commandId": dispatch.command_id(), "replayed": dispatch.replayed() }, - }), - 200, - ), + Ok(dispatch) => json_status(dispatch.wait_path_wire(), 200), Err(error) => error_response(error), } } @@ -675,7 +669,7 @@ impl DurableObject for ZoneLeaseCell { .alarm(&self.env) .await .map_err(|error| Error::RustError(error.to_string()))?; - for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + for failed in wake.schedules.failed.iter().chain(&wake.schedules.retried) { worker::console_error!("lease alarm failed: {} {}", failed.code, failed.message); } Response::ok("ok") diff --git a/examples/game/server/src/freeze/world.rs b/examples/game/server/src/freeze/world.rs index 4335168ff..e00898995 100644 --- a/examples/game/server/src/freeze/world.rs +++ b/examples/game/server/src/freeze/world.rs @@ -36,7 +36,7 @@ use axum::extract::{Path, State}; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{get, post}; use axum::{Json, Router}; -use distributed_game::celld::{CelldFactRoute, CelldFactSink}; +use distributed_game::celld::{CelldFactSink, FactRoute}; use distributed_game::net::peer::{post_json, ADMIN_SECRET_HEADER}; use distributed_game::net::{ GameServer, RemoteHost, RolloutError, ServerConfig, TicketKey, WorldCoordinator, ZoneFactory, @@ -70,18 +70,14 @@ pub struct FreezeFactSink { impl FreezeFactSink { /// Sink into the cells at `celld`. pub fn new(celld: &CelldOptions) -> Result { - let sink = CelldFactSink::new( + let sink = CelldFactSink::celld( celld.url.clone(), celld.secret.clone(), SERVICE_ID, SYSTEM_PRINCIPAL, )? - .route(CelldFactRoute::by_principal::("gem-pouch")) - .route(CelldFactRoute::new( - ::COMMAND, - "content-log", - |fact, _| Some(fact.stamp().zone_id.clone()), - )); + .route(FactRoute::by_principal::("gem-pouch")) + .route(FactRoute::by_zone::("content-log")); Ok(Self { celld: sink, settled: Arc::default(), diff --git a/examples/game/server/src/mobpit/store.rs b/examples/game/server/src/mobpit/store.rs index 82f6689b6..bc8b50195 100644 --- a/examples/game/server/src/mobpit/store.rs +++ b/examples/game/server/src/mobpit/store.rs @@ -3,9 +3,10 @@ //! //! - [`MobPitFactSink`] is the game's `FactSink`: optional artificial //! latency (the "slow storage" toggle), then -//! `distributed_game::celld::CelldFactSink` sends the fact over the -//! internal path to its cell — `crawler.record_kill` to -//! `crawler/{principal}` (`CelldFactRoute::by_principal`), the runtime's +//! `distributed_game::celld::CelldFactSink` (the command fact sink over a +//! remote-only celld command host) sends the fact over the internal path +//! to its cell — `crawler.record_kill` to `crawler/{principal}` +//! (`FactRoute::by_principal`), the runtime's //! player lifecycle facts to `presence/{principal}` (`lifecycle_route`) — //! then a fire-and-forget leaderboard update. The command id comes from //! the fact key, so a retry replays through the cell's command ledger @@ -26,7 +27,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use distributed::cell_host::{InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER}; -use distributed_game::celld::{CelldFactRoute, CelldFactSink}; +use distributed_game::celld::{CelldFactSink, FactRoute}; use distributed_game::lifecycle::Lifecycle; use distributed_game::{BoxFuture, Fact, FactError, FactReceipt, FactSink}; use serde::{Deserialize, Serialize}; @@ -249,15 +250,13 @@ impl MobPitFactSink { board: Arc, presence: Arc, ) -> Result { - let sink = CelldFactSink::new( + let sink = CelldFactSink::celld( celld.url.clone(), celld.secret.clone(), SERVICE_ID, SYSTEM_PRINCIPAL, )? - .route(CelldFactRoute::by_principal::( - CRAWLER_KIND, - )) + .route(FactRoute::by_principal::(CRAWLER_KIND)) .lifecycle_route(PRESENCE_KIND); Ok(Self { celld: sink, diff --git a/migrations/inventory.json b/migrations/inventory.json index 249bdf2e3..ef1998c3b 100644 --- a/migrations/inventory.json +++ b/migrations/inventory.json @@ -132,6 +132,18 @@ "path": "migrations/postgres/0011_cell_alarms.sql", "sha256": "572da24fac96e56263b5dfdf17be26d99f7a330de5585325ba091a4fe3e6caf8" } + }, + { + "version": 12, + "description": "aggregate schedules", + "sqlite": { + "path": "migrations/sqlite/0012_aggregate_schedules.sql", + "sha256": "7b95a6a81e9595916b65170c22404f78da3639bce5f51bdf4f2693677629848d" + }, + "postgres": { + "path": "migrations/postgres/0012_aggregate_schedules.sql", + "sha256": "e56147e54924aadac88619ab722f005bbf127f3c009d0cdf80f7ee69b1c2823f" + } } ] } diff --git a/migrations/postgres/0012_aggregate_schedules.sql b/migrations/postgres/0012_aggregate_schedules.sql new file mode 100644 index 000000000..c2794da2a --- /dev/null +++ b/migrations/postgres/0012_aggregate_schedules.sql @@ -0,0 +1,24 @@ +-- Aggregate schedules of a shared repository (see `distributed::schedule`). +-- One row per (owning stream, schedule name), written in the command's +-- transaction; `generation` increments on every arm and is never reset +-- (cancelling keeps the row), so the command id derived from +-- (name, generation) is unique per arm. `due_at_ms` is epoch milliseconds; +-- `armed` is 1 while the schedule is pending. Aggregate cells keep their +-- own single-owner `cell_alarms` table instead. +CREATE TABLE IF NOT EXISTS aggregate_schedules ( + aggregate_type TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + name TEXT NOT NULL, + command TEXT NOT NULL, + input TEXT NOT NULL, + due_at_ms BIGINT NOT NULL, + generation BIGINT NOT NULL, + attempts BIGINT NOT NULL DEFAULT 0, + armed SMALLINT NOT NULL, + PRIMARY KEY (aggregate_type, aggregate_id, name), + CHECK (name <> ''), + CHECK (command <> ''), + CHECK (generation > 0), + CHECK (armed IN (0, 1)) +); +CREATE INDEX IF NOT EXISTS aggregate_schedules_armed_due ON aggregate_schedules (armed, due_at_ms); diff --git a/migrations/sqlite/0012_aggregate_schedules.sql b/migrations/sqlite/0012_aggregate_schedules.sql new file mode 100644 index 000000000..bea20c7b8 --- /dev/null +++ b/migrations/sqlite/0012_aggregate_schedules.sql @@ -0,0 +1,24 @@ +-- Aggregate schedules of a shared repository (see `distributed::schedule`). +-- One row per (owning stream, schedule name), written in the command's +-- transaction; `generation` increments on every arm and is never reset +-- (cancelling keeps the row), so the command id derived from +-- (name, generation) is unique per arm. `due_at_ms` is epoch milliseconds; +-- `armed` is 1 while the schedule is pending. Aggregate cells keep their +-- own single-owner `cell_alarms` table instead. +CREATE TABLE IF NOT EXISTS aggregate_schedules ( + aggregate_type TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + name TEXT NOT NULL, + command TEXT NOT NULL, + input TEXT NOT NULL, + due_at_ms INTEGER NOT NULL, + generation INTEGER NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + armed INTEGER NOT NULL, + PRIMARY KEY (aggregate_type, aggregate_id, name), + CHECK (name <> ''), + CHECK (command <> ''), + CHECK (generation > 0), + CHECK (armed IN (0, 1)) +); +CREATE INDEX IF NOT EXISTS aggregate_schedules_armed_due ON aggregate_schedules (armed, due_at_ms); diff --git a/src/command_dispatch/host.rs b/src/command_dispatch/host.rs index a4c30da73..d46f1c78d 100644 --- a/src/command_dispatch/host.rs +++ b/src/command_dispatch/host.rs @@ -329,20 +329,33 @@ pub struct HttpCommandHost { impl HttpCommandHost { const MAX_JSON_BYTES: usize = 2 * 1024 * 1024; + /// Default per-request timeout. + pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); + pub fn new(base: impl AsRef) -> Result { - Self::build(base.as_ref(), None) + Self::build(base.as_ref(), None, Self::DEFAULT_TIMEOUT) } pub fn new_internal( base: impl AsRef, secret: InternalHttpSecret, ) -> Result { - Self::build(base.as_ref(), Some(secret)) + Self::build(base.as_ref(), Some(secret), Self::DEFAULT_TIMEOUT) + } + + /// As [`new_internal`](Self::new_internal) with a per-request timeout. + pub fn new_internal_with_timeout( + base: impl AsRef, + secret: InternalHttpSecret, + timeout: Duration, + ) -> Result { + Self::build(base.as_ref(), Some(secret), timeout) } fn build( base: &str, internal_secret: Option, + timeout: Duration, ) -> Result { let base = reqwest::Url::parse(base.trim_end_matches('/')).map_err(|error| { CausalDispatchError::Internal(format!("invalid wait-path base URL: {error}")) @@ -359,8 +372,8 @@ impl HttpCommandHost { )); } let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(2)) - .timeout(Duration::from_secs(10)) + .connect_timeout(Duration::from_secs(2).min(timeout)) + .timeout(timeout) .redirect(Policy::none()) .build() .map_err(|error| { diff --git a/src/commit_builder/mod.rs b/src/commit_builder/mod.rs index f7741f792..501e1a26c 100644 --- a/src/commit_builder/mod.rs +++ b/src/commit_builder/mod.rs @@ -182,7 +182,7 @@ impl<'a, R> CommitBuilder<'a, R> { read_model_plans: self.read_model_plans, snapshots: Vec::new(), inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), }) .await } diff --git a/src/graphql/projection_delta/tests.rs b/src/graphql/projection_delta/tests.rs index bfd49d0f3..68cce2d7b 100644 --- a/src/graphql/projection_delta/tests.rs +++ b/src/graphql/projection_delta/tests.rs @@ -1097,6 +1097,7 @@ fn wait_path_sealing_retains_event_obligations_and_status_stays_pending_without_ let (protocol, contract, event) = wait_path_fixture(); let result = CausalDispatchResult { + replayed: false, payload: json!({"accepted": true}), receipt: CausalCommandReceiptSource { command_id: "wait-path-command".into(), @@ -1170,6 +1171,7 @@ fn wait_path_observation_proof_requires_the_persisted_semantic_program_identity( let (protocol, contract, event) = wait_path_fixture(); let result = CausalDispatchResult { + replayed: false, payload: json!({"accepted": true}), receipt: CausalCommandReceiptSource { command_id: "wait-path-identity-command".into(), @@ -1448,6 +1450,7 @@ fn wait_path_sealing_keeps_atomic_and_unselected_commands_without_metadata() { events: Vec| -> CausalDispatchResult { CausalDispatchResult { + replayed: false, payload: json!({"accepted": true}), receipt: CausalCommandReceiptSource { command_id: "wait-path-bypass".into(), diff --git a/src/in_memory_repo/repository.rs b/src/in_memory_repo/repository.rs index 7608907bb..c9592db09 100644 --- a/src/in_memory_repo/repository.rs +++ b/src/in_memory_repo/repository.rs @@ -62,6 +62,12 @@ pub struct InMemoryRepository { pub(super) projection_change_retention: ProjectionChangeRetention, #[cfg_attr(not(feature = "graphql"), allow(dead_code))] causal_storage_identity: CausalStorageIdentity, + /// Schedule rows per owning stream (`storage_key`), committed with the + /// owner's events. Stored, not dispatched: no + /// [`Capability::Scheduling`](crate::schedule::Capability::Scheduling). + schedules: Arc>>, + /// The store clock: `ctx.now()` and relative schedules. + clock: Arc, } /// In-memory outbox table handle. @@ -93,9 +99,32 @@ impl InMemoryRepository { causal_tables, projection_change_retention: ProjectionChangeRetention::default(), causal_storage_identity: CausalStorageIdentity::new(), + schedules: Arc::new(RwLock::new(HashMap::new())), + clock: Arc::new(crate::SystemClock), } } + /// Use `clock` as the store clock (`ctx.now()`, relative schedules), + /// e.g. a [`ManualClock`](crate::ManualClock) in tests. + pub fn with_clock(mut self, clock: impl crate::Clock + 'static) -> Self { + self.clock = Arc::new(clock); + self + } + + /// Armed schedules of the aggregate stream `owner`, earliest first. + pub fn armed_schedules( + &self, + owner: &StreamIdentity, + ) -> Result, RepositoryError> { + Ok(self + .schedules + .read() + .map_err(|_| RepositoryError::LockPoisoned("schedule read"))? + .get(&owner.storage_key()) + .map(crate::schedule::MemorySchedules::armed) + .unwrap_or_default()) + } + /// Configure the maximum newest projection changes retained per partition. /// /// The persisted compacted-through watermark remains authoritative when @@ -296,7 +325,9 @@ impl InMemoryRepository { mut completion: Option, direct_projection: Option, ) -> Result<(), InMemoryCommitError> { - crate::microsvc::cell_host::alarm::reject_alarm_writes(&batch.alarms)?; + let mut batch = batch; + let schedule_owner = crate::schedule::schedule_owner(&batch)?; + crate::schedule::resolve_schedule_writes(&mut batch.schedules, self.clock.now())?; let prepared = validate_commit_batch(&batch)?; if let Some(direct_projection) = &direct_projection { direct_projection.validate()?; @@ -496,7 +527,29 @@ impl InMemoryRepository { }) .transpose()?; + // Schedules: plan on a copy of the owner's table (may fail), install + // with the other writes below. + let mut schedule_storage = self + .schedules + .write() + .map_err(|_| RepositoryError::LockPoisoned("schedule write"))?; + let planned_schedules = match &schedule_owner { + Some(owner) => Some(( + owner.storage_key(), + schedule_storage + .get(&owner.storage_key()) + .cloned() + .unwrap_or_default() + .plan(&batch.schedules)?, + )), + None => None, + }; + // Nothing below can fail: mutate the live stores. + if let Some((owner, planned)) = planned_schedules { + schedule_storage.insert(owner, planned); + } + drop(schedule_storage); for append in prepared { storage .entry(append.identity.storage_key()) @@ -848,8 +901,11 @@ impl ReadModelWritePlanStore for InMemoryRepository { } impl RelationalReadModelQueryStore for InMemoryRepository { - fn scan_read_model(&self, request: crate::read_model::ReadModelScanRequest) - -> impl Future> + Send + '_ { + fn scan_read_model( + &self, + request: crate::read_model::ReadModelScanRequest, + ) -> impl Future> + Send + '_ + { self.model_store.scan_read_model(request) } @@ -1089,3 +1145,20 @@ mod tests { assert_eq!(repo.clear_inbox(), 0); } } + +/// Stores schedules, does not dispatch them (no worker yet); the store clock +/// is the injected [`Clock`](crate::Clock). +impl crate::microsvc::BackendHost for InMemoryRepository { + fn capabilities(&self) -> crate::schedule::Capabilities { + crate::schedule::Capabilities { + scheduling: false, + clock: true, + } + } + + fn host_now( + &self, + ) -> impl Future> + Send + '_ { + std::future::ready(Ok(self.clock.now())) + } +} diff --git a/src/lib.rs b/src/lib.rs index 7fa71b7fe..a516db0e9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,6 +57,7 @@ pub mod projection_protocol; pub mod queued_repo; pub mod read_model; pub mod saga; +pub mod schedule; pub mod snapshot; #[cfg(feature = "sqlite")] pub mod sqlite_repo; @@ -377,6 +378,7 @@ pub use saga::{ Saga, SagaAction, SagaCommand, SagaDefinition, SagaError, SagaMessage, SagaOutcome, SagaReply, SagaRunner, SagaStatus, SagaStep, }; +pub use schedule::{Capabilities, Capability, Schedule, ScheduleWrite}; // Outbox: commit concerns (aggregate + outbox in one commit). // diff --git a/src/microsvc/causal.rs b/src/microsvc/causal.rs index ff509b06a..48f0ff312 100644 --- a/src/microsvc/causal.rs +++ b/src/microsvc/causal.rs @@ -24,7 +24,7 @@ use crate::command::{ }; use crate::command_ledger::CausalGetStream; use crate::domain_event::{DomainEventCaptureError, DomainEventCommitGuardError}; -use crate::microsvc::cell_host::CellAlarmWrite; +use crate::microsvc::cell_host::ScheduleWrite; use crate::outbox::{OutboxMessage, PreparedDomainEvent}; use crate::projection::lower::{ DirectCandidate, LoweredProjectionPlan, ProjectionDescriptor, @@ -156,13 +156,13 @@ pub(crate) enum CausalWorkspaceError { DomainPublicationsAlreadyPrepared, DomainPublicationsNotPrepared, CommitBatchAlreadyPrepared, - Alarm(crate::microsvc::cell_host::CellAlarmError), + Schedule(crate::schedule::ScheduleError), } impl std::fmt::Display for CausalWorkspaceError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::Alarm(error) => write!(formatter, "cell alarm staging failed: {error}"), + Self::Schedule(error) => write!(formatter, "schedule staging failed: {error}"), Self::Repository(error) => write!(formatter, "causal aggregate load failed: {error}"), Self::Table(error) => write!(formatter, "causal read-model staging failed: {error}"), Self::DomainEventCapture(error) => { @@ -219,7 +219,7 @@ impl std::error::Error for CausalWorkspaceError { Self::DomainEventCapture(error) => Some(error), Self::DomainEventGuard(error) => Some(error), Self::Prepare(error) => Some(error), - Self::Alarm(error) => Some(error), + Self::Schedule(error) => Some(error), Self::Poisoned | Self::IdentityChanged { .. } | Self::CommittedVersionChanged { .. } @@ -284,7 +284,7 @@ struct WorkspaceState { outbox_messages: Vec, read_model_plans: Vec, snapshots: Vec, - alarms: Vec, + schedules: Vec, projection_staged: bool, modeled_direct_projection: Option, } @@ -297,7 +297,7 @@ impl Default for WorkspaceState { outbox_messages: Vec::new(), read_model_plans: Vec::new(), snapshots: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), projection_staged: false, modeled_direct_projection: None, } @@ -312,6 +312,7 @@ where { store: Box + 'repo>, state: Mutex>, + scheduling_declared: bool, } impl<'repo, A> CausalWorkspace<'repo, A> @@ -325,9 +326,18 @@ where Self { store: Box::new(AggregateStoreRef { repository }), state: Mutex::new(WorkspaceState::default()), + scheduling_declared: false, } } + /// Whether the route declared `Capability::Scheduling` (schedule writes + /// are refused otherwise, so a host that cannot dispatch them is caught + /// at registration). + pub(crate) fn with_scheduling_declared(mut self, declared: bool) -> Self { + self.scheduling_declared = declared; + self + } + /// Load a full stream through the explicit causal/non-locking capability. pub(crate) async fn load( &self, @@ -420,19 +430,24 @@ where Ok(()) } - /// Stage an alarm write. It commits with the command's events or not at - /// all; the batch is bounded by `MAX_CELL_ALARM_WRITES_PER_COMMAND`. - pub(crate) fn stage_alarm(&self, write: CellAlarmWrite) -> Result<(), CausalWorkspaceError> { + /// Stage a schedule write. It commits with the command's events or not at + /// all; the batch is bounded by `MAX_SCHEDULE_WRITES_PER_COMMAND`. + pub(crate) fn stage_schedule(&self, write: ScheduleWrite) -> Result<(), CausalWorkspaceError> { + if !self.scheduling_declared { + return Err(CausalWorkspaceError::Schedule( + crate::schedule::ScheduleError::NotDeclared, + )); + } let mut state = self .state .lock() .map_err(|_| CausalWorkspaceError::Poisoned)?; - if state.alarms.len() >= crate::microsvc::cell_host::MAX_CELL_ALARM_WRITES_PER_COMMAND { - return Err(CausalWorkspaceError::Alarm( - crate::microsvc::cell_host::CellAlarmError::TooManyWrites, + if state.schedules.len() >= crate::schedule::MAX_SCHEDULE_WRITES_PER_COMMAND { + return Err(CausalWorkspaceError::Schedule( + crate::schedule::ScheduleError::TooManyWrites, )); } - state.alarms.push(write); + state.schedules.push(write); Ok(()) } @@ -567,7 +582,7 @@ where outbox_messages: state.outbox_messages, read_model_plans: state.read_model_plans, snapshots: state.snapshots, - alarms: state.alarms, + schedules: state.schedules, domain_event_occurrences: Vec::new(), unpublished_direct_occurrences: HashSet::new(), modeled_direct_projection: state.modeled_direct_projection, @@ -585,7 +600,7 @@ pub(crate) struct CausalWorkspaceParts { outbox_messages: Vec, read_model_plans: Vec, snapshots: Vec, - alarms: Vec, + schedules: Vec, /// Exact ledger-causation-stamped outward occurrences in publication order. domain_event_occurrences: Vec, /// Captured occurrences consumed only by a direct modeled projection. @@ -807,7 +822,7 @@ where batch.outbox_messages = std::mem::take(&mut self.outbox_messages); batch.read_model_plans = std::mem::take(&mut self.read_model_plans); batch.snapshots = std::mem::take(&mut self.snapshots); - batch.alarms = std::mem::take(&mut self.alarms); + batch.schedules = std::mem::take(&mut self.schedules); Ok(batch) } diff --git a/src/microsvc/cell_host/alarm.rs b/src/microsvc/cell_host/alarm.rs index be2cba84f..ee70c684f 100644 --- a/src/microsvc/cell_host/alarm.rs +++ b/src/microsvc/cell_host/alarm.rs @@ -1,788 +1,22 @@ -//! Re-armable cell alarms. -//! -//! A cell command handler arms an alarm with its events: -//! -//! ```rust,ignore -//! ctx.arm_alarm(CellAlarm::at("tick", due, "matchmaker.tick", &TickInput { pool })?)?; -//! ``` -//! -//! The alarm row commits in the **same transaction** as the command's events, -//! ledger receipt and outbox rows, so a failed command leaves no alarm. At -//! `due` the cell dispatches `command` into itself with a trusted system -//! session ([`cell_alarm_session`]); that handler may arm the same name again, -//! which gives a periodic tick without a global cron. -//! -//! Semantics, identical on celld and on the in-process reference host: -//! -//! - One row per alarm **name**. Arming an existing name replaces its due -//! time, command and input and bumps its `generation`; cancelling disarms it. -//! - A fired alarm runs through the cell's command ledger with a command id -//! derived from `(name, generation)`. A crash after the handler committed but -//! before the alarm was settled replays the recorded receipt instead of -//! running the handler twice, so each arm has its effects at most once. -//! - Settlement only disarms the generation that fired. A handler that re-arms -//! its own name produced a newer generation, which stays armed. -//! - Rejections (domain errors, guard failures, bad input) are terminal for -//! that arm. Retryable failures (in-progress, internal) back off -//! exponentially from one second up to five minutes. +//! The cell host's side of [`crate::schedule`]: due schedules are the +//! cell's alarms. //! //! On celld the Durable Object has a single alarm slot, owned jointly with the //! outbox watchdog (`celld_outbox.rs`): every `setAlarm` targets the earliest -//! of the next domain alarm and the watchdog's next drain. Neither replaces the -//! other. - -use std::time::{Duration, SystemTime}; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use sha2::{Digest, Sha256}; - -use super::causal::{CellCommandIdentity, CellDispatchError}; -use crate::microsvc::session::{Session, ROLE_KEY, USER_ID_KEY}; -use crate::repository::RepositoryError; - -/// Role carried by the trusted session of a fired alarm. Commands that only -/// alarms may run declare `roles: ["cell-alarm"]` and -/// `guard: cell_alarm_guard`. -pub const CELL_ALARM_ROLE: &str = "cell-alarm"; -/// User id of the trusted session of a fired alarm. -pub const CELL_ALARM_USER_ID: &str = "system:cell-alarm"; -/// Session claim naming the alarm that fired. Set only by the cell host. -pub const CELL_ALARM_NAME_CLAIM: &str = "x-distributed-cell-alarm"; -/// Session claim with the fired alarm's due time (milliseconds since the Unix -/// epoch). Periodic handlers re-arm relative to it so ticks do not drift. -pub const CELL_ALARM_DUE_CLAIM: &str = "x-distributed-cell-alarm-due-ms"; -/// Session claim with the fired alarm's generation. -pub const CELL_ALARM_GENERATION_CLAIM: &str = "x-distributed-cell-alarm-generation"; -/// Session claim with the cell clock's time at dispatch (epoch ms). The cell -/// host overwrites it on every command, so callers cannot choose it; read it -/// with `CausalCommandContext::cell_now`. -pub const CELL_NOW_CLAIM: &str = "x-distributed-cell-now-ms"; -/// Command-ledger service id of fired alarms. -pub const CELL_ALARM_SERVICE_ID: &str = "distributed.cell-alarm"; -/// Longest alarm name. -pub const MAX_CELL_ALARM_NAME_BYTES: usize = 128; -/// Largest serialized alarm input. -pub const MAX_CELL_ALARM_INPUT_BYTES: usize = 16 * 1024; -/// Most alarm writes one command may stage. -pub const MAX_CELL_ALARM_WRITES_PER_COMMAND: usize = 16; -/// Most alarms one cell may hold armed at once. -pub const MAX_ARMED_CELL_ALARMS: usize = 1024; -/// Most due alarms fired per wake; the rest fire on the next wake. -pub const CELL_ALARM_FIRE_BATCH: usize = 32; -/// First retry delay after a retryable alarm failure. -pub const CELL_ALARM_RETRY_BASE: Duration = Duration::from_secs(1); -/// Longest retry delay after repeated retryable alarm failures. -pub const CELL_ALARM_RETRY_MAX: Duration = Duration::from_secs(300); - -/// Why an alarm could not be armed. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CellAlarmError { - InvalidName, - EmptyCommand, - InvalidInput, - DueOutOfRange, - TooManyWrites, - TooManyArmed, -} - -impl std::fmt::Display for CellAlarmError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::InvalidName => write!( - f, - "cell alarm name must be 1..={MAX_CELL_ALARM_NAME_BYTES} bytes of [A-Za-z0-9._:-]" - ), - Self::EmptyCommand => f.write_str("cell alarm command must not be empty"), - Self::InvalidInput => write!( - f, - "cell alarm input must be a JSON object of at most {MAX_CELL_ALARM_INPUT_BYTES} bytes" - ), - Self::DueOutOfRange => f.write_str("cell alarm due time is out of range"), - Self::TooManyWrites => write!( - f, - "a command may stage at most {MAX_CELL_ALARM_WRITES_PER_COMMAND} cell alarm writes" - ), - Self::TooManyArmed => write!( - f, - "a cell may hold at most {MAX_ARMED_CELL_ALARMS} armed alarms" - ), - } - } -} - -impl std::error::Error for CellAlarmError {} - -impl From for RepositoryError { - fn from(error: CellAlarmError) -> Self { - RepositoryError::Model(error.to_string()) - } -} - -/// When an alarm is due. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum AlarmDue { - /// Absolute epoch milliseconds. - At(i64), - /// Milliseconds after the commit, resolved by the cell's clock. - After(u64), -} - -/// An alarm a command arms: when due, dispatch `command` with `input`. -#[derive(Clone, Debug, PartialEq)] -pub struct CellAlarm { - name: String, - command: String, - input: Value, - due: AlarmDue, -} - -impl CellAlarm { - /// Arm `name` to dispatch `command` with `input` into this cell at `due`. - /// - /// Periodic handlers re-arm from the fired due time - /// (`ctx.alarm_due()`), so ticks do not drift with handler latency. - pub fn at( - name: impl Into, - due: SystemTime, - command: impl Into, - input: &impl Serialize, - ) -> Result { - Self::build(name, AlarmDue::At(system_time_to_ms(due)?), command, input) - } - - /// Arm `name` to fire `delay` after the command commits, measured by the - /// cell's clock (the Durable Object clock on celld, the injected - /// [`Clock`](crate::Clock) on the reference host). Domain handlers have no - /// wall clock of their own; this is how they schedule relative work. - pub fn after( - name: impl Into, - delay: Duration, - command: impl Into, - input: &impl Serialize, - ) -> Result { - let delay = u64::try_from(delay.as_millis()).map_err(|_| CellAlarmError::DueOutOfRange)?; - Self::build(name, AlarmDue::After(delay), command, input) - } - - fn build( - name: impl Into, - due: AlarmDue, - command: impl Into, - input: &impl Serialize, - ) -> Result { - let input = serde_json::to_value(input).map_err(|_| CellAlarmError::InvalidInput)?; - let alarm = Self { - name: name.into(), - command: command.into(), - input, - due, - }; - alarm.validate()?; - Ok(alarm) - } - - pub fn name(&self) -> &str { - &self.name - } - - pub fn command(&self) -> &str { - &self.command - } +//! of the next due schedule and the watchdog's next drain. Neither replaces +//! the other. Each cell stores its schedules in its own `cell_alarms` table +//! (one cell is one stream, so no owner columns); the in-process reference +//! host keeps a [`MemorySchedules`](crate::schedule::MemorySchedules) table. - pub fn input(&self) -> &Value { - &self.input - } +pub(crate) use crate::schedule::*; - /// Absolute due time, or `None` for a relative alarm not yet committed. - pub fn due(&self) -> Option { - match self.due { - AlarmDue::At(ms) => Some(ms_to_system_time(ms)), - AlarmDue::After(_) => None, - } - } - - /// Resolve a relative due time against the committing cell's clock. - pub(crate) fn resolve(&mut self, now: SystemTime) -> Result<(), CellAlarmError> { - if let AlarmDue::After(delay) = self.due { - let now = system_time_to_ms(now)?; - let due = now - .checked_add(i64::try_from(delay).map_err(|_| CellAlarmError::DueOutOfRange)?) - .ok_or(CellAlarmError::DueOutOfRange)?; - if due > MAX_JS_SAFE_MS { - return Err(CellAlarmError::DueOutOfRange); - } - self.due = AlarmDue::At(due); - } - Ok(()) - } - - /// Absolute due time; storage refuses an unresolved relative alarm. - pub(crate) fn due_at_ms(&self) -> Result { - match self.due { - AlarmDue::At(ms) => Ok(ms), - AlarmDue::After(_) => Err(RepositoryError::Model( - "relative cell alarm reached storage without being resolved".into(), - )), - } - } - - #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] - pub(crate) fn input_json(&self) -> Result { - serde_json::to_string(&self.input) - .map_err(|error| RepositoryError::Model(format!("cell alarm input: {error}"))) - } - - pub(crate) fn validate(&self) -> Result<(), CellAlarmError> { - validate_name(&self.name)?; - if self.command.trim().is_empty() { - return Err(CellAlarmError::EmptyCommand); - } - let size = serde_json::to_vec(&self.input) - .map_err(|_| CellAlarmError::InvalidInput)? - .len(); - if !self.input.is_object() || size > MAX_CELL_ALARM_INPUT_BYTES { - return Err(CellAlarmError::InvalidInput); - } - match self.due { - AlarmDue::At(ms) if !(0..=MAX_JS_SAFE_MS).contains(&ms) => { - Err(CellAlarmError::DueOutOfRange) - } - AlarmDue::After(delay) if delay > MAX_JS_SAFE_MS as u64 => { - Err(CellAlarmError::DueOutOfRange) - } - _ => Ok(()), - } - } -} - -/// Resolve every relative alarm of a batch against the cell clock. -pub(crate) fn resolve_alarm_writes( - writes: &mut [CellAlarmWrite], - now: SystemTime, -) -> Result<(), RepositoryError> { - for write in writes { - if let CellAlarmWrite::Arm(alarm) = write { - alarm.resolve(now)?; - } - } - Ok(()) -} - -/// Durable Object SQL binds integers through JS numbers. -const MAX_JS_SAFE_MS: i64 = 9_007_199_254_740_991; - -/// One alarm write staged by a command and committed with its events. -#[derive(Clone, Debug, PartialEq)] -pub enum CellAlarmWrite { - /// Arm (or re-arm) the alarm's name. - Arm(CellAlarm), - /// Disarm a name. Cancelling a name that is not armed is a no-op. - Cancel(String), -} - -impl CellAlarmWrite { - pub fn name(&self) -> &str { - match self { - Self::Arm(alarm) => alarm.name(), - Self::Cancel(name) => name, - } - } -} - -/// Validate a batch's alarm writes before any storage participant runs. -pub(crate) fn validate_alarm_writes(writes: &[CellAlarmWrite]) -> Result<(), RepositoryError> { - if writes.len() > MAX_CELL_ALARM_WRITES_PER_COMMAND { - return Err(CellAlarmError::TooManyWrites.into()); - } - for write in writes { - match write { - CellAlarmWrite::Arm(alarm) => alarm.validate()?, - CellAlarmWrite::Cancel(name) => validate_name(name)?, - } - } - Ok(()) -} - -/// Ordinary repositories have no alarm slot; only aggregate cells do. -pub(crate) fn reject_alarm_writes(writes: &[CellAlarmWrite]) -> Result<(), RepositoryError> { - if writes.is_empty() { - Ok(()) - } else { - Err(RepositoryError::Model( - "cell alarms require an aggregate cell host".into(), - )) - } -} - -/// An armed alarm as stored by the cell. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct ArmedCellAlarm { - pub name: String, - pub command: String, - pub input: Value, - pub due_at_ms: i64, - pub generation: u64, - pub attempts: u32, -} - -impl ArmedCellAlarm { - pub fn due(&self) -> SystemTime { - ms_to_system_time(self.due_at_ms) - } - - /// Stable command-ledger identity of this arm. - pub fn command_identity(&self) -> Result { - CellCommandIdentity::new( - CELL_ALARM_SERVICE_ID, - CELL_ALARM_SERVICE_ID, - alarm_command_id(&self.name, self.generation), - ) - } -} - -/// How a fired alarm settled. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum AlarmSettlement { - /// Ran (or replayed) or failed terminally: disarm this generation. - Done, - /// Retry this generation at `due_at_ms`. - Retry { due_at_ms: i64 }, -} - -/// What one wake did with the due alarms. -#[derive(Clone, Debug, Default, PartialEq)] -pub struct CellAlarmFireReport { - /// Alarms whose command committed (or replayed a committed receipt). - pub fired: Vec, - /// Alarms rescheduled after a retryable failure. - pub retried: Vec, - /// Alarms disarmed after a terminal failure. - pub failed: Vec, -} +use super::causal::CellDispatchError; -impl CellAlarmFireReport { - pub fn is_empty(&self) -> bool { - self.fired.is_empty() && self.retried.is_empty() && self.failed.is_empty() - } -} - -/// A fired alarm. -#[derive(Clone, Debug, PartialEq)] -pub struct FiredCellAlarm { - pub name: String, - pub generation: u64, - pub command: String, - /// The command had already committed; its receipt was replayed. - pub replayed: bool, - pub payload: Value, -} - -/// An alarm whose command failed. -#[derive(Clone, Debug, PartialEq)] -pub struct FailedCellAlarm { - pub name: String, - pub generation: u64, - pub command: String, - pub code: &'static str, - pub message: String, -} - -/// The trusted system session a fired alarm dispatches with. -pub fn cell_alarm_session(alarm: &ArmedCellAlarm) -> Session { - let mut session = Session::new(); - session.set(USER_ID_KEY, CELL_ALARM_USER_ID); - session.set(ROLE_KEY, CELL_ALARM_ROLE); - session.set(CELL_ALARM_NAME_CLAIM, alarm.name.clone()); - session.set(CELL_ALARM_DUE_CLAIM, alarm.due_at_ms.to_string()); - session.set(CELL_ALARM_GENERATION_CLAIM, alarm.generation.to_string()); - session -} - -/// The cell clock's time at dispatch, when the command runs in a cell. -pub fn cell_now(session: &Session) -> Option { - session - .get(CELL_NOW_CLAIM)? - .parse::() - .ok() - .filter(|ms| *ms >= 0) - .map(ms_to_system_time) -} - -/// Whether `session` is the trusted session of a fired alarm. -pub fn is_cell_alarm_session(session: &Session) -> bool { - session.user_id() == Some(CELL_ALARM_USER_ID) - && session.has_role(CELL_ALARM_ROLE) - && session.get(CELL_ALARM_NAME_CLAIM).is_some() -} - -/// Due time of the alarm that fired this command, when it was an alarm. -pub fn cell_alarm_due(session: &Session) -> Option { - if !is_cell_alarm_session(session) { - return None; - } - session - .get(CELL_ALARM_DUE_CLAIM)? - .parse::() - .ok() - .filter(|ms| *ms >= 0) - .map(ms_to_system_time) -} - -/// `portable_command!` guard admitting only fired alarms. -pub fn cell_alarm_guard(ctx: &crate::microsvc::CausalCommandContext<'_, A>) -> bool -where - A: crate::Aggregate + Send + Sync + 'static, -{ - is_cell_alarm_session(ctx.session()) -} - -/// Deterministic UUIDv7-shaped command id of one alarm arm. -pub(crate) fn alarm_command_id(name: &str, generation: u64) -> String { - let mut hasher = Sha256::new(); - hasher.update(CELL_ALARM_SERVICE_ID.as_bytes()); - hasher.update([0u8]); - hasher.update(name.as_bytes()); - hasher.update([0u8]); - hasher.update(generation.to_be_bytes()); - let digest = hasher.finalize(); - let mut bytes = [0u8; 16]; - bytes.copy_from_slice(&digest[..16]); - bytes[6] = (bytes[6] & 0x0f) | 0x70; - bytes[8] = (bytes[8] & 0x3f) | 0x80; - uuid::Uuid::from_bytes(bytes).hyphenated().to_string() -} - -/// Exponential retry delay for the `attempts`-th failure (0-based). -pub(crate) fn retry_delay(attempts: u32) -> Duration { - let factor = 1u32.checked_shl(attempts.min(16)).unwrap_or(u32::MAX); - CELL_ALARM_RETRY_BASE - .checked_mul(factor) - .unwrap_or(CELL_ALARM_RETRY_MAX) - .min(CELL_ALARM_RETRY_MAX) -} - -/// Whether a dispatch failure should be retried rather than disarm the alarm. +/// Whether a dispatch failure should be retried rather than disarm the +/// schedule. pub(crate) fn is_retryable(error: &CellDispatchError) -> bool { matches!( error, CellDispatchError::InProgress | CellDispatchError::Internal(_) ) } - -pub(crate) fn system_time_to_ms(time: SystemTime) -> Result { - let millis = time - .duration_since(SystemTime::UNIX_EPOCH) - .map_err(|_| CellAlarmError::DueOutOfRange)? - .as_millis(); - if millis > MAX_JS_SAFE_MS as u128 { - return Err(CellAlarmError::DueOutOfRange); - } - Ok(millis as i64) -} - -pub(crate) fn ms_to_system_time(ms: i64) -> SystemTime { - SystemTime::UNIX_EPOCH + Duration::from_millis(ms.max(0) as u64) -} - -fn validate_name(name: &str) -> Result<(), CellAlarmError> { - if name.is_empty() - || name.len() > MAX_CELL_ALARM_NAME_BYTES - || !name - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) - { - return Err(CellAlarmError::InvalidName); - } - Ok(()) -} - -/// The in-process reference host's alarm table, with the same semantics as -/// the SQL table used by Durable Object and native SQLite cells. -#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] -#[derive(Clone, Debug, Default)] -pub(crate) struct MemoryCellAlarms { - rows: std::collections::BTreeMap, -} - -#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] -#[derive(Clone, Debug)] -struct MemoryAlarmRow { - alarm: ArmedCellAlarm, - armed: bool, -} - -#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] -impl MemoryCellAlarms { - /// Apply one command's writes all-or-nothing, as the SQL transaction does. - pub(crate) fn plan(&self, writes: &[CellAlarmWrite]) -> Result { - let mut next = self.clone(); - for write in writes { - match write { - CellAlarmWrite::Arm(alarm) => { - let generation = next - .rows - .get(alarm.name()) - .map_or(1, |row| row.alarm.generation + 1); - next.rows.insert( - alarm.name().to_string(), - MemoryAlarmRow { - alarm: ArmedCellAlarm { - name: alarm.name().to_string(), - command: alarm.command().to_string(), - input: alarm.input().clone(), - due_at_ms: alarm.due_at_ms()?, - generation, - attempts: 0, - }, - armed: true, - }, - ); - } - CellAlarmWrite::Cancel(name) => { - if let Some(row) = next.rows.get_mut(name) { - row.armed = false; - } - } - } - } - if next.rows.values().filter(|row| row.armed).count() > MAX_ARMED_CELL_ALARMS { - return Err(CellAlarmError::TooManyArmed.into()); - } - Ok(next) - } - - pub(crate) fn armed(&self) -> Vec { - let mut armed = self - .rows - .values() - .filter(|row| row.armed) - .map(|row| row.alarm.clone()) - .collect::>(); - armed.sort_by(|a, b| (a.due_at_ms, &a.name).cmp(&(b.due_at_ms, &b.name))); - armed - } - - pub(crate) fn due(&self, now_ms: i64, limit: usize) -> Vec { - self.armed() - .into_iter() - .filter(|alarm| alarm.due_at_ms <= now_ms) - .take(limit) - .collect() - } - - pub(crate) fn next_due_ms(&self) -> Option { - self.rows - .values() - .filter(|row| row.armed) - .map(|row| row.alarm.due_at_ms) - .min() - } - - pub(crate) fn settle(&mut self, name: &str, generation: u64, settlement: &AlarmSettlement) { - let Some(row) = self.rows.get_mut(name) else { - return; - }; - if !row.armed || row.alarm.generation != generation { - return; - } - match settlement { - AlarmSettlement::Done => { - row.armed = false; - row.alarm.attempts = 0; - } - AlarmSettlement::Retry { due_at_ms } => { - row.alarm.attempts = row.alarm.attempts.saturating_add(1); - row.alarm.due_at_ms = *due_at_ms; - } - } - } - - /// Every row (armed or not), for reference-host restart exports. - pub(crate) fn export(&self) -> Vec { - self.rows - .values() - .map(|row| DurableCellAlarm { - alarm: row.alarm.clone(), - armed: row.armed, - }) - .collect() - } - - pub(crate) fn restore(rows: Vec) -> Self { - Self { - rows: rows - .into_iter() - .map(|row| { - ( - row.alarm.name.clone(), - MemoryAlarmRow { - alarm: row.alarm, - armed: row.armed, - }, - ) - }) - .collect(), - } - } -} - -/// One alarm row in a reference-host restart export. Disarmed rows are kept so -/// generations never repeat after a restore. -#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct DurableCellAlarm { - pub alarm: ArmedCellAlarm, - pub armed: bool, -} - -#[cfg(test)] -mod tests { - use super::*; - - fn alarm(name: &str, due_ms: u64) -> CellAlarm { - CellAlarm::at( - name, - SystemTime::UNIX_EPOCH + Duration::from_millis(due_ms), - "test.tick", - &serde_json::json!({ "n": 1 }), - ) - .unwrap() - } - - #[test] - fn validates_names_inputs_and_write_counts() { - let due = SystemTime::UNIX_EPOCH + Duration::from_secs(1); - assert_eq!( - CellAlarm::at("", due, "c", &serde_json::json!({})).unwrap_err(), - CellAlarmError::InvalidName - ); - assert_eq!( - CellAlarm::at("a b", due, "c", &serde_json::json!({})).unwrap_err(), - CellAlarmError::InvalidName - ); - assert_eq!( - CellAlarm::at("ok", due, " ", &serde_json::json!({})).unwrap_err(), - CellAlarmError::EmptyCommand - ); - assert_eq!( - CellAlarm::at("ok", due, "c", &1).unwrap_err(), - CellAlarmError::InvalidInput - ); - let big = serde_json::json!({ "x": "y".repeat(MAX_CELL_ALARM_INPUT_BYTES) }); - assert_eq!( - CellAlarm::at("ok", due, "c", &big).unwrap_err(), - CellAlarmError::InvalidInput - ); - let writes = (0..=MAX_CELL_ALARM_WRITES_PER_COMMAND) - .map(|index| CellAlarmWrite::Cancel(format!("a{index}"))) - .collect::>(); - assert!(validate_alarm_writes(&writes).is_err()); - assert!(validate_alarm_writes(&writes[1..]).is_ok()); - assert!(reject_alarm_writes(&writes).is_err()); - assert!(reject_alarm_writes(&[]).is_ok()); - } - - #[test] - fn command_ids_are_uuidv7_and_unique_per_generation() { - let first = alarm_command_id("tick", 1); - assert_eq!(first, alarm_command_id("tick", 1)); - assert_ne!(first, alarm_command_id("tick", 2)); - assert_ne!(first, alarm_command_id("tock", 1)); - let parsed = uuid::Uuid::parse_str(&first).unwrap(); - assert_eq!(parsed.get_version_num(), 7); - let armed = ArmedCellAlarm { - name: "tick".into(), - command: "c".into(), - input: serde_json::json!({}), - due_at_ms: 0, - generation: 1, - attempts: 0, - }; - assert_eq!(armed.command_identity().unwrap().command_id(), first); - } - - #[test] - fn retry_delay_doubles_and_caps() { - assert_eq!(retry_delay(0), Duration::from_secs(1)); - assert_eq!(retry_delay(3), Duration::from_secs(8)); - assert_eq!(retry_delay(40), CELL_ALARM_RETRY_MAX); - } - - #[test] - fn alarm_session_is_recognized_and_carries_due() { - let armed = ArmedCellAlarm { - name: "tick".into(), - command: "c".into(), - input: serde_json::json!({}), - due_at_ms: 1_234, - generation: 7, - attempts: 0, - }; - let session = cell_alarm_session(&armed); - assert!(is_cell_alarm_session(&session)); - assert_eq!( - cell_alarm_due(&session), - Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_234)) - ); - let mut forged = Session::new(); - forged.set(USER_ID_KEY, "player-1"); - forged.set(ROLE_KEY, CELL_ALARM_ROLE); - forged.set(CELL_ALARM_NAME_CLAIM, "tick"); - assert!(!is_cell_alarm_session(&forged)); - assert_eq!(cell_alarm_due(&forged), None); - } - - #[test] - fn memory_table_rearms_with_new_generation_and_settles_only_its_own() { - let table = MemoryCellAlarms::default(); - let table = table - .plan(&[CellAlarmWrite::Arm(alarm("tick", 10))]) - .unwrap(); - assert_eq!(table.armed()[0].generation, 1); - let mut table = table - .plan(&[CellAlarmWrite::Arm(alarm("tick", 20))]) - .unwrap(); - assert_eq!(table.armed()[0].generation, 2); - assert_eq!(table.next_due_ms(), Some(20)); - assert!(table.due(19, 10).is_empty()); - table.settle("tick", 1, &AlarmSettlement::Done); - assert_eq!(table.armed().len(), 1, "stale generation must not disarm"); - table.settle("tick", 2, &AlarmSettlement::Retry { due_at_ms: 50 }); - assert_eq!(table.armed()[0].attempts, 1); - assert_eq!(table.next_due_ms(), Some(50)); - table.settle("tick", 2, &AlarmSettlement::Done); - assert!(table.armed().is_empty()); - let table = table - .plan(&[CellAlarmWrite::Arm(alarm("tick", 60))]) - .unwrap(); - assert_eq!( - table.armed()[0].generation, - 3, - "generations never repeat after a disarm" - ); - let restored = MemoryCellAlarms::restore(table.export()); - assert_eq!(restored.armed(), table.armed()); - let cancelled = restored - .plan(&[CellAlarmWrite::Cancel("tick".into())]) - .unwrap(); - assert!(cancelled.armed().is_empty()); - } - - #[test] - fn memory_table_caps_armed_alarms() { - let mut table = MemoryCellAlarms::default(); - for chunk in 0..(MAX_ARMED_CELL_ALARMS / 16) { - let writes = (0..16) - .map(|index| CellAlarmWrite::Arm(alarm(&format!("a{chunk}-{index}"), 1))) - .collect::>(); - table = table.plan(&writes).unwrap(); - } - let error = table - .plan(&[CellAlarmWrite::Arm(alarm("overflow", 1))]) - .unwrap_err(); - assert!(error.to_string().contains("armed alarms")); - } -} diff --git a/src/microsvc/cell_host/causal.rs b/src/microsvc/cell_host/causal.rs index 532f85b10..e3bec2335 100644 --- a/src/microsvc/cell_host/causal.rs +++ b/src/microsvc/cell_host/causal.rs @@ -87,10 +87,9 @@ impl CellCommandIdentity { mut self, causation_id: impl AsRef, ) -> Result { - let causation_id = crate::command_ledger::CausationId::parse_stored( - causation_id.as_ref().to_string(), - ) - .map_err(internal_ledger_error)?; + let causation_id = + crate::command_ledger::CausationId::parse_stored(causation_id.as_ref().to_string()) + .map_err(internal_ledger_error)?; self.causation_id = Some(causation_id); Ok(self) } @@ -140,6 +139,22 @@ impl CellDispatchResult { pub fn replayed(&self) -> bool { self.replayed } + + /// The standard wait-path response body (`{payload, receipt: {commandId, + /// causationId, state, replayed}, events}`) every command host decodes + /// (`CausalDispatchResult::from_wait_path_wire`). + pub fn wait_path_wire(&self) -> Value { + serde_json::json!({ + "payload": self.payload, + "receipt": { + "commandId": self.command_id, + "causationId": self.causation_id, + "state": self.state, + "replayed": self.replayed, + }, + "events": self.events, + }) + } } /// Stable error vocabulary for the feature-free cell wait path. diff --git a/src/microsvc/cell_host/cell.rs b/src/microsvc/cell_host/cell.rs index 488ff8b43..9fcbb54dc 100644 --- a/src/microsvc/cell_host/cell.rs +++ b/src/microsvc/cell_host/cell.rs @@ -6,7 +6,7 @@ use std::time::{Duration, SystemTime}; use serde_json::Value; -use super::alarm::{AlarmSettlement, ArmedCellAlarm, CellAlarmFireReport}; +use super::alarm::{ArmedSchedule, ScheduleFireReport, ScheduleSettlement}; use super::causal::{CellCommandIdentity, CellDispatchError, CellDispatchResult}; use super::store::CellStreamStore; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] @@ -212,7 +212,6 @@ where input: Value, session: Session, ) -> Result { - let session = self.stamp_now(session); #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] let _command = self .begin_command() @@ -239,7 +238,6 @@ where input: Value, session: Session, ) -> Result { - let session = self.stamp_now(session); #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] let _command = self .begin_command() @@ -254,21 +252,13 @@ where result } - /// Overwrite the cell-clock claim: handlers read the cell's time - /// (`ctx.cell_now()`), never a caller's. - fn stamp_now(&self, mut session: Session) -> Session { - let now = super::alarm::system_time_to_ms(self.routes.repo().repo().now()).unwrap_or(0); - session.set(super::alarm::CELL_NOW_CLAIM, now.to_string()); - session - } - /// Armed alarms of this cell, earliest first. - pub fn armed_alarms(&self) -> Result, RepositoryError> { - self.routes.repo().repo().armed_alarms() + pub fn armed_schedules(&self) -> Result, RepositoryError> { + self.routes.repo().repo().armed_schedules() } /// Due time of this cell's earliest armed alarm. - pub fn next_alarm_due(&self) -> Result, RepositoryError> { + pub fn next_schedule_due(&self) -> Result, RepositoryError> { Ok(self .routes .repo() @@ -283,33 +273,33 @@ where /// [`ManualClock`](crate::ManualClock) ([`CellStreamStore::with_clock`]), /// advance it, and call this, exactly as the celld alarm handler does on /// every Durable Object wake. - pub async fn fire_due_alarms(&self) -> Result { - self.fire_due_alarms_at(self.routes.repo().repo().now()) + pub async fn fire_due_schedules(&self) -> Result { + self.fire_due_schedules_at(self.routes.repo().repo().now()) .await } - /// Fire up to [`CELL_ALARM_FIRE_BATCH`](super::CELL_ALARM_FIRE_BATCH) + /// Fire up to [`SCHEDULE_FIRE_BATCH`](crate::schedule::SCHEDULE_FIRE_BATCH) /// alarms due at or before `now`, earliest first. /// /// Each one dispatches its command through this cell's command ledger with /// a stable `(name, generation)` command id and the trusted alarm session. /// The due set is read once, so a handler that re-arms in the past fires on /// the next wake, not in a loop. - pub async fn fire_due_alarms_at( + pub async fn fire_due_schedules_at( &self, now: SystemTime, - ) -> Result { + ) -> Result { let store = self.routes.repo().repo(); let now_ms = super::alarm::system_time_to_ms(now)?; - let mut report = CellAlarmFireReport::default(); - for alarm in store.due_alarms(now_ms, super::CELL_ALARM_FIRE_BATCH)? { + let mut report = ScheduleFireReport::default(); + for alarm in store.due_alarms(now_ms, crate::schedule::SCHEDULE_FIRE_BATCH)? { let result = match alarm.command_identity() { Ok(identity) => { self.dispatch_idempotent( &alarm.command, &identity, alarm.input.clone(), - super::cell_alarm_session(&alarm), + super::schedule_session(&alarm), ) .await } @@ -317,18 +307,18 @@ where }; let settlement = match result { Ok(dispatch) => { - report.fired.push(super::FiredCellAlarm { + report.fired.push(super::FiredSchedule { name: alarm.name.clone(), generation: alarm.generation, command: alarm.command.clone(), replayed: dispatch.replayed(), payload: dispatch.payload().clone(), }); - AlarmSettlement::Done + ScheduleSettlement::Done } Err(error) => { let retry = super::alarm::is_retryable(&error); - let failed = super::FailedCellAlarm { + let failed = super::FailedSchedule { name: alarm.name.clone(), generation: alarm.generation, command: alarm.command.clone(), @@ -340,14 +330,14 @@ where }; if retry { report.retried.push(failed); - AlarmSettlement::Retry { + ScheduleSettlement::Retry { due_at_ms: now_ms.saturating_add( super::alarm::retry_delay(alarm.attempts).as_millis() as i64, ), } } else { report.failed.push(failed); - AlarmSettlement::Done + ScheduleSettlement::Done } } }; @@ -462,7 +452,7 @@ where &self, env: &worker::Env, ) -> Result { - let next_alarm = self.next_alarm_due().map_err(alarm_transport_error)?; + let next_alarm = self.next_schedule_due().map_err(alarm_transport_error)?; self.outbox_config()? .drain(self, env, &self.storage, next_alarm) .await @@ -485,11 +475,14 @@ where env: &worker::Env, ) -> Result { let alarms = self - .fire_due_alarms() + .fire_due_schedules() .await .map_err(alarm_transport_error)?; let drain = self.drain_outbox(env).await?; - Ok(super::celld_outbox::CelldAlarmOutcome { alarms, drain }) + Ok(super::celld_outbox::CelldAlarmOutcome { + schedules: alarms, + drain, + }) } /// A command may have armed an alarm due before the prearmed watchdog @@ -500,7 +493,7 @@ where let Ok(outbox) = self.outbox_config() else { return; }; - let next_alarm = match self.next_alarm_due() { + let next_alarm = match self.next_schedule_due() { Ok(next_alarm) => next_alarm, Err(error) => { worker::console_error!("cell alarm lookup failed after command: {error}"); @@ -535,7 +528,7 @@ where async fn begin_command( &self, ) -> Result { - let next_alarm = self.next_alarm_due().map_err(alarm_transport_error)?; + let next_alarm = self.next_schedule_due().map_err(alarm_transport_error)?; self.outbox_config()? .begin_command( &self.activity, diff --git a/src/microsvc/cell_host/celld_outbox.rs b/src/microsvc/cell_host/celld_outbox.rs index ea33c6e64..1abac6758 100644 --- a/src/microsvc/cell_host/celld_outbox.rs +++ b/src/microsvc/cell_host/celld_outbox.rs @@ -58,7 +58,7 @@ pub struct CelldOutboxDrainOutcome { #[derive(Debug, Default)] pub struct CelldAlarmOutcome { /// Domain alarms fired, retried or failed by this wake. - pub alarms: super::CellAlarmFireReport, + pub schedules: crate::schedule::ScheduleFireReport, /// The outbox drain that followed. pub drain: CelldOutboxDrainOutcome, } diff --git a/src/microsvc/cell_host/command.rs b/src/microsvc/cell_host/command.rs index 9c7b44df3..99f2729a4 100644 --- a/src/microsvc/cell_host/command.rs +++ b/src/microsvc/cell_host/command.rs @@ -11,16 +11,16 @@ use serde_json::Value; use super::InternalHttpSecret; use crate::command_dispatch::{ - validate_principal_session, validate_principal_session_if_present, CommandHost, - HttpCommandHost, LocalCommandHost, + validate_principal_session, validate_principal_session_if_present, CellRequestContext, + CommandHost, HttpCommandHost, LocalCommandHost, TrustedRequestMetadata, }; +use crate::command_ledger::ExternalDispatchBinding; use crate::graphql::identity::VerifiedPrincipal; use crate::graphql::protocol::ProtocolResponseAccumulator; use crate::microsvc::{ - CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, ExternalCausalReservation, - Service, Session, + CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, + ExternalCausalReservation, Service, Session, }; -use crate::command_ledger::ExternalDispatchBinding; /// One aggregate's cell wait-path: command names, URL kind, shard id, payload. #[derive(Clone, Copy)] @@ -48,11 +48,33 @@ impl CelldRoute { } } -/// Routes selected commands to celld; everything else stays on [`LocalCommandHost`]. +type CellShardFn = Arc Option + Send + Sync>; +type CellPayloadFn = fn(command: &str, input: &Value, remote: &Value, session: &Session) -> Value; + +/// One registered cell route: commands, cell kind, shard of (input, session). +#[derive(Clone)] +struct CellRoute { + commands: Vec, + kind: String, + shard: CellShardFn, + payload: Option, +} + +/// Routes selected commands to celld cells. +/// +/// - [`new`](Self::new): the GraphQL wait-path host. A named local +/// [`Service`] reserves each routed command in its ledger (status queries, +/// projection sealing); unrouted commands run on that service. +/// - [`remote`](Self::remote): a remote-only host for trusted internal +/// callers (a zone server's fact sink): routed commands go straight to the +/// cell, whose own command ledger makes them idempotent; anything unrouted +/// is refused. pub struct CelldCommandHost { http: HttpCommandHost, - local: LocalCommandHost, - routes: Vec, + local: Option, + service_id: Option, + routes: Vec, + forwarded_prefixes: Vec, } impl CelldCommandHost { @@ -65,30 +87,183 @@ impl CelldCommandHost { let http = HttpCommandHost::new_internal(&celld_url, internal_secret)?; Ok(Self { http, - local: LocalCommandHost::new(service), + local: Some(LocalCommandHost::new(service)), + service_id: None, + routes: Vec::new(), + forwarded_prefixes: Vec::new(), + }) + } + + /// A remote-only host under command-ledger `service_id` (see the type + /// docs), with a per-request `timeout`. + pub fn remote( + celld_url: impl Into, + service_id: impl Into, + internal_secret: InternalHttpSecret, + timeout: std::time::Duration, + ) -> Result { + let service_id = service_id.into(); + if service_id.trim().is_empty() { + return Err(CausalDispatchError::Internal( + "a remote celld command host needs a service id".into(), + )); + } + let celld_url = celld_url.into().trim_end_matches('/').to_string(); + let http = + HttpCommandHost::new_internal_with_timeout(&celld_url, internal_secret, timeout)?; + Ok(Self { + http, + local: None, + service_id: Some(service_id), routes: Vec::new(), + forwarded_prefixes: Vec::new(), }) } pub fn route(mut self, route: CelldRoute) -> Self { - self.routes.push(route); + let shard = route.shard; + self.routes.push(CellRoute { + commands: route + .commands + .iter() + .map(|command| command.to_string()) + .collect(), + kind: route.kind.to_string(), + shard: Arc::new(move |input, _| shard(input)), + payload: Some(route.payload), + }); self } - fn route_for(&self, command: &str) -> Option<&CelldRoute> { + /// Route `command` to `{kind}/{shard(input, session)}/{command}` and + /// return the cell's payload unchanged. A later route for the same + /// command replaces the earlier. + pub fn cell_route( + mut self, + command: impl Into, + kind: impl Into, + shard: impl Fn(&Value, &Session) -> Option + Send + Sync + 'static, + ) -> Self { + self.add_cell_route(command, kind, shard); + self + } + + /// [`cell_route`](Self::cell_route) on a host being configured in place. + pub fn add_cell_route( + &mut self, + command: impl Into, + kind: impl Into, + shard: impl Fn(&Value, &Session) -> Option + Send + Sync + 'static, + ) { + let command = command.into(); + for route in &mut self.routes { + route.commands.retain(|existing| existing != &command); + } + self.routes.retain(|route| !route.commands.is_empty()); + self.routes.push(CellRoute { + commands: vec![command], + kind: kind.into(), + shard: Arc::new(shard), + payload: None, + }); + } + + /// Forward the session variables whose names start with `prefix` (for + /// example a game fact's `x-game-*` stamp) to the cell as trusted + /// metadata headers. Only for callers that build the session from + /// already-trusted facts: public request sessions stay identity-only. + /// Framework identity and transport headers can never be forwarded. + pub fn forward_session_prefix( + mut self, + prefix: impl Into, + ) -> Result { + let prefix = prefix.into(); + // A probe header under the prefix must be a legal, non-reserved name. + TrustedRequestMetadata::new().try_insert(&format!("{prefix}probe"), "probe")?; + self.forwarded_prefixes.push(prefix); + Ok(self) + } + + fn route_for(&self, command: &str) -> Option<&CellRoute> { self.routes .iter() - .find(|route| route.commands.contains(&command)) + .find(|route| route.commands.iter().any(|existing| existing == command)) } fn service_id(&self) -> Result<&str, CausalDispatchError> { - self.local.service().name().ok_or_else(|| { - CausalDispatchError::Internal( - "celld command host requires a named executable service".into(), - ) - }) + if let Some(service_id) = &self.service_id { + return Ok(service_id); + } + self.local + .as_ref() + .and_then(|local| local.service().name()) + .ok_or_else(|| { + CausalDispatchError::Internal( + "celld command host requires a named executable service".into(), + ) + }) + } + + fn forwarded_metadata( + &self, + session: &Session, + ) -> Result { + let mut metadata = TrustedRequestMetadata::new(); + if self.forwarded_prefixes.is_empty() { + return Ok(metadata); + } + let mut names: Vec<&String> = session + .variables() + .keys() + .filter(|name| { + self.forwarded_prefixes + .iter() + .any(|prefix| name.starts_with(prefix.as_str())) + }) + .collect(); + names.sort(); + for name in names { + if let Some(value) = session.get(name) { + metadata.try_insert(name, value)?; + } + } + Ok(metadata) } + /// Remote-only dispatch: straight to the cell, idempotent by its ledger. + #[allow(clippy::too_many_arguments)] + async fn invoke_remote( + &self, + route: &CellRoute, + shard: &str, + command: &str, + command_id: &str, + input: Value, + session: &Session, + principal: &VerifiedPrincipal, + ) -> Result { + let service_id = self.service_id()?.to_string(); + let context = + CellRequestContext::new(&service_id, principal.partition_for_service(&service_id)) + .with_trusted_metadata(self.forwarded_metadata(session)?); + let http = self.http.retarget_segments(&[&route.kind, shard])?; + let (status, body) = http + .post_cell_wait_path_with_context(command, command_id, input.clone(), session, &context) + .await?; + if status >= 400 { + return Err(remote_dispatch_error(status, &body)); + } + let remote = CausalDispatchResult::from_wait_path_wire(body).map_err(|error| { + CausalDispatchError::Internal(format!("wait-path decode: {error:?}")) + })?; + Ok(match route.payload { + Some(payload) => { + let mapped = payload(command, &input, remote.payload(), session); + remote.with_payload(mapped) + } + None => remote, + }) + } } fn remote_dispatch_error(status: u16, body: &Value) -> CausalDispatchError { @@ -136,13 +311,19 @@ impl CommandHost for CelldCommandHost { protocol: Option, ) -> Result { validate_principal_session(&session, &principal)?; - let Some(route) = self.route_for(command).copied() else { - return self - .local - .invoke(command, command_id, input, session, principal, protocol) - .await; + let Some(route) = self.route_for(command).cloned() else { + return match &self.local { + Some(local) => { + local + .invoke(command, command_id, input, session, principal, protocol) + .await + } + None => Err(CausalDispatchError::BadRequest(format!( + "no celld route for command `{command}`" + ))), + }; }; - let shard = (route.shard)(&input) + let shard = (route.shard)(&input, &session) .filter(|value| !value.is_empty()) .ok_or_else(|| { CausalDispatchError::BadRequest(format!( @@ -150,12 +331,18 @@ impl CommandHost for CelldCommandHost { route.kind )) })?; + let Some(local) = &self.local else { + return self + .invoke_remote( + &route, &shard, command, command_id, input, &session, &principal, + ) + .await; + }; let service_id = self.service_id()?.to_string(); let principal_partition = principal.partition_for_service(&service_id); - let binding = ExternalDispatchBinding::new(route.kind, &shard) + let binding = ExternalDispatchBinding::new(&route.kind, &shard) .map_err(|error| CausalDispatchError::BadRequest(error.to_string()))?; - let reservation = self - .local + let reservation = local .service() .reserve_external_causal( command, @@ -170,11 +357,10 @@ impl CommandHost for CelldCommandHost { ExternalCausalReservation::Acquired(attempt) => attempt, ExternalCausalReservation::Replay(replay) => return Ok(replay), }; - let http = match self.http.retarget_segments(&[route.kind, &shard]) { + let http = match self.http.retarget_segments(&[&route.kind, &shard]) { Ok(http) => http, Err(error) => { - let _ = self - .local + let _ = local .service() .abandon_external_causal(command, attempt) .await; @@ -195,8 +381,7 @@ impl CommandHost for CelldCommandHost { { Ok(response) => response, Err(error) => { - let _ = self - .local + let _ = local .service() .abandon_external_causal(command, attempt) .await; @@ -205,8 +390,7 @@ impl CommandHost for CelldCommandHost { }; if status >= 400 { let error = remote_dispatch_error(status, &body); - let _ = self - .local + let _ = local .service() .abandon_external_causal(command, attempt) .await; @@ -215,8 +399,7 @@ impl CommandHost for CelldCommandHost { let remote = match CausalDispatchResult::from_wait_path_wire(body) { Ok(remote) => remote, Err(error) => { - let _ = self - .local + let _ = local .service() .abandon_external_causal(command, attempt) .await; @@ -226,25 +409,27 @@ impl CommandHost for CelldCommandHost { } }; if let Err(error) = attempt.validate_remote(&remote) { - let _ = self - .local + let _ = local .service() .abandon_external_causal(command, attempt) .await; return Err(error); } - let payload = (route.payload)(command, &input, remote.payload(), &session); - let mut remote = remote.with_payload(payload); + let mut remote = match route.payload { + Some(payload) => { + let mapped = payload(command, &input, remote.payload(), &session); + remote.with_payload(mapped) + } + None => remote, + }; if let Some(protocol) = protocol { - remote = match self - .local + remote = match local .service() .seal_wait_path_dispatch(command, &protocol, remote) { Ok(remote) => remote, Err(error) => { - let _ = self - .local + let _ = local .service() .abandon_external_causal(command, attempt) .await; @@ -252,7 +437,7 @@ impl CommandHost for CelldCommandHost { } }; } - self.local + local .service() .complete_external_causal(command, attempt, &remote) .await?; @@ -267,8 +452,9 @@ impl CommandHost for CelldCommandHost { protocol: Option, ) -> Result { validate_principal_session_if_present(session, &principal)?; - self.local - .status(command_id, session, principal, protocol) - .await + match &self.local { + Some(local) => local.status(command_id, session, principal, protocol).await, + None => Ok(CausalCommandPublicStatus::unknown(command_id)), + } } } diff --git a/src/microsvc/cell_host/mod.rs b/src/microsvc/cell_host/mod.rs index 7e5ed4cde..d178d2432 100644 --- a/src/microsvc/cell_host/mod.rs +++ b/src/microsvc/cell_host/mod.rs @@ -35,16 +35,11 @@ mod sql_store; mod store; mod wire; -#[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] -pub use alarm::DurableCellAlarm; -pub use alarm::{ - cell_alarm_due, cell_alarm_guard, cell_alarm_session, cell_now, is_cell_alarm_session, - ArmedCellAlarm, CellAlarm, CellAlarmError, CellAlarmFireReport, CellAlarmWrite, - FailedCellAlarm, FiredCellAlarm, CELL_ALARM_DUE_CLAIM, CELL_ALARM_FIRE_BATCH, - CELL_ALARM_GENERATION_CLAIM, CELL_ALARM_NAME_CLAIM, CELL_ALARM_RETRY_BASE, - CELL_ALARM_RETRY_MAX, CELL_ALARM_ROLE, CELL_ALARM_SERVICE_ID, CELL_ALARM_USER_ID, - CELL_NOW_CLAIM, MAX_ARMED_CELL_ALARMS, MAX_CELL_ALARM_INPUT_BYTES, MAX_CELL_ALARM_NAME_BYTES, - MAX_CELL_ALARM_WRITES_PER_COMMAND, +// Schedule types under their old cell-host paths, for the host's own code. +#[allow(unused_imports)] +pub(crate) use crate::schedule::{ + schedule_session, ArmedSchedule, FailedSchedule, FiredSchedule, Schedule, ScheduleError, + ScheduleFireReport, ScheduleWrite, HOST_NOW_CLAIM, MAX_SCHEDULE_WRITES_PER_COMMAND, }; pub use causal::{ CellCommandIdentity, CellDispatchError, CellDispatchResult, CELL_CAUSATION_ID_HEADER, diff --git a/src/microsvc/cell_host/sql_store.rs b/src/microsvc/cell_host/sql_store.rs index dd651a96e..057025621 100644 --- a/src/microsvc/cell_host/sql_store.rs +++ b/src/microsvc/cell_host/sql_store.rs @@ -31,7 +31,7 @@ use crate::repository::{ }; use crate::snapshot::SnapshotRecord; -use super::alarm::{AlarmSettlement, ArmedCellAlarm}; +use super::alarm::{ArmedSchedule, ScheduleSettlement}; // Durable Object SQL limits bindings per statement, not per transaction. // Both shared insert planners split large commands within the same transaction. @@ -120,7 +120,7 @@ impl CellSqlRepository { )); } let prepared = validate_commit_batch(&batch)?; - super::alarm::validate_alarm_writes(&batch.alarms)?; + super::alarm::validate_schedule_writes(&batch.schedules)?; self.connection.transaction(|executor| { finish_sql(async { if let Some(completion) = &completion { @@ -168,8 +168,8 @@ impl CellSqlRepository { } // Alarm rows join the same transaction: a failed command (or a // lost ledger lease below) leaves no alarm armed. - if !batch.alarms.is_empty() { - alarms::apply(executor, &batch.alarms).await?; + if !batch.schedules.is_empty() { + alarms::apply(executor, &batch.schedules).await?; } // Native sealed read-model row: same transaction, so a reducer // error or lost ledger lease rolls the row back with the events. @@ -190,7 +190,7 @@ impl CellSqlRepository { impl CellSqlRepository { /// Armed alarms, earliest first. - pub(super) fn armed_alarms(&self) -> Result, RepositoryError> { + pub(super) fn armed_alarms(&self) -> Result, RepositoryError> { self.connection .read(|executor| finish_sql(alarms::armed(executor, None, usize::MAX))) } @@ -200,7 +200,7 @@ impl CellSqlRepository { &self, now_ms: i64, limit: usize, - ) -> Result, RepositoryError> { + ) -> Result, RepositoryError> { self.connection .read(|executor| finish_sql(alarms::armed(executor, Some(now_ms), limit))) } @@ -216,7 +216,7 @@ impl CellSqlRepository { &self, name: &str, generation: u64, - settlement: &AlarmSettlement, + settlement: &ScheduleSettlement, ) -> Result<(), RepositoryError> { self.connection.transaction(|executor| { finish_sql(alarms::settle(executor, name, generation, settlement)) @@ -428,20 +428,18 @@ impl CellSqlOutboxStore { /// native SQLite cells; the reference host mirrors these semantics in memory. mod alarms { use super::*; - use crate::microsvc::cell_host::alarm::{ - CellAlarmError, CellAlarmWrite, MAX_ARMED_CELL_ALARMS, - }; + use crate::microsvc::cell_host::alarm::{ScheduleError, ScheduleWrite, MAX_ARMED_SCHEDULES}; const SELECT: &str = "SELECT name, command, input, due_at_ms, generation, attempts FROM cell_alarms WHERE armed = 1"; pub(super) async fn apply( executor: &mut impl SqlExecutor, - writes: &[CellAlarmWrite], + writes: &[ScheduleWrite], ) -> Result<(), RepositoryError> { for write in writes { match write { - CellAlarmWrite::Arm(alarm) => { + ScheduleWrite::Arm(alarm) => { let input = alarm.input_json()?; executor .execute( @@ -457,7 +455,7 @@ mod alarms { ) .await?; } - CellAlarmWrite::Cancel(name) => { + ScheduleWrite::Cancel(name) => { executor .execute( Statement::new( @@ -479,8 +477,8 @@ mod alarms { .map(|row| row.integer("armed")) .transpose()? .unwrap_or(0); - if armed > MAX_ARMED_CELL_ALARMS as i64 { - return Err(CellAlarmError::TooManyArmed.into()); + if armed > MAX_ARMED_SCHEDULES as i64 { + return Err(ScheduleError::TooManyArmed.into()); } Ok(()) } @@ -489,7 +487,7 @@ mod alarms { executor: &mut impl SqlExecutor, due_by_ms: Option, limit: usize, - ) -> Result, RepositoryError> { + ) -> Result, RepositoryError> { let limit = (limit as u64).min(9_007_199_254_740_991) as i64; let mut statement = Statement::new(SELECT); if let Some(due_by_ms) = due_by_ms { @@ -505,7 +503,7 @@ mod alarms { rows.iter() .map(|row| { let input = row.text("input")?; - Ok(ArmedCellAlarm { + Ok(ArmedSchedule { name: row.text("name")?, command: row.text("command")?, input: serde_json::from_str(&input).map_err(|error| { @@ -540,15 +538,15 @@ mod alarms { executor: &mut impl SqlExecutor, name: &str, generation: u64, - settlement: &AlarmSettlement, + settlement: &ScheduleSettlement, ) -> Result<(), RepositoryError> { let generation = i64::try_from(generation) .map_err(|_| RepositoryError::Model("cell alarm generation overflow".into()))?; let statement = match settlement { - AlarmSettlement::Done => { + ScheduleSettlement::Done => { Statement::new("UPDATE cell_alarms SET armed = 0, attempts = 0 WHERE name = ") } - AlarmSettlement::Retry { due_at_ms } => { + ScheduleSettlement::Retry { due_at_ms } => { Statement::new("UPDATE cell_alarms SET attempts = attempts + 1, due_at_ms = ") .bind((*due_at_ms).into()) .sql(" WHERE name = ") diff --git a/src/microsvc/cell_host/store.rs b/src/microsvc/cell_host/store.rs index dee9e7136..a1a4e19c5 100644 --- a/src/microsvc/cell_host/store.rs +++ b/src/microsvc/cell_host/store.rs @@ -112,7 +112,7 @@ pub struct DurableAggregateCellState { pub sealed_row: Option, /// Alarm rows, including disarmed ones so generations never repeat. #[serde(default)] - pub alarms: Vec, + pub alarms: Vec, } /// Private command-side repository for one cell instance (`{aggregate_type}:{shard}`). @@ -141,7 +141,7 @@ pub struct CellStreamStore { sealed_row_reducer: Option, /// Reference-host alarm table (SQL cells keep alarms in their database). #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] - alarms: Arc>, + alarms: Arc>, } impl CellStreamStore { @@ -271,10 +271,10 @@ impl CellStreamStore { /// Replace the cell clock (default: the system clock). The reference host /// takes a [`ManualClock`](crate::ManualClock) so alarm tests are - /// deterministic; it resolves [`CellAlarm::after`](super::CellAlarm::after) - /// at commit and is what [`AggregateCell::fire_due_alarms`] reads. + /// deterministic; it resolves [`Schedule::after`](super::Schedule::after) + /// at commit and is what [`AggregateCell::fire_due_schedules`] reads. /// - /// [`AggregateCell::fire_due_alarms`]: super::AggregateCell::fire_due_alarms + /// [`AggregateCell::fire_due_schedules`]: super::AggregateCell::fire_due_schedules pub fn with_clock(mut self, clock: impl crate::Clock + 'static) -> Self { self.clock = Arc::new(clock); self @@ -499,14 +499,14 @@ impl CellStreamStore { .lock() .map_err(|_| RepositoryError::Model("cell sealed row lock poisoned".into()))?; *sealed = state.sealed_row; - *self.lock_alarms()? = super::alarm::MemoryCellAlarms::restore(state.alarms); + *self.lock_alarms()? = super::alarm::MemorySchedules::restore(state.alarms); Ok(()) } #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] fn lock_alarms( &self, - ) -> Result, RepositoryError> { + ) -> Result, RepositoryError> { self.alarms .lock() .map_err(|_| RepositoryError::Model("cell alarm table lock poisoned".into())) @@ -525,7 +525,7 @@ impl CellStreamStore { } /// Armed alarms, earliest first. - pub fn armed_alarms(&self) -> Result, RepositoryError> { + pub fn armed_schedules(&self) -> Result, RepositoryError> { #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] if let Some(sql) = self.alarm_sql() { return sql.armed_alarms(); @@ -541,7 +541,7 @@ impl CellStreamStore { &self, now_ms: i64, limit: usize, - ) -> Result, RepositoryError> { + ) -> Result, RepositoryError> { #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] if let Some(sql) = self.alarm_sql() { return sql.due_alarms(now_ms, limit); @@ -568,7 +568,7 @@ impl CellStreamStore { &self, name: &str, generation: u64, - settlement: &super::alarm::AlarmSettlement, + settlement: &super::alarm::ScheduleSettlement, ) -> Result<(), RepositoryError> { #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] if let Some(sql) = self.alarm_sql() { @@ -585,19 +585,19 @@ impl CellStreamStore { fn plan_memory_alarms( &self, batch: &mut CommitBatch<'_>, - ) -> Result, RepositoryError> { - if self.alarm_sql().is_some() || batch.alarms.is_empty() { + ) -> Result, RepositoryError> { + if self.alarm_sql().is_some() || batch.schedules.is_empty() { return Ok(None); } - let writes = std::mem::take(&mut batch.alarms); - super::alarm::validate_alarm_writes(&writes)?; + let writes = std::mem::take(&mut batch.schedules); + super::alarm::validate_schedule_writes(&writes)?; self.lock_alarms()?.plan(&writes).map(Some) } #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] fn install_memory_alarms( &self, - planned: Option, + planned: Option, ) -> Result<(), RepositoryError> { if let Some(planned) = planned { *self.lock_alarms()? = planned; @@ -755,6 +755,21 @@ impl CommandLedgerStore for CellStreamStore { } } +/// A cell dispatches its due schedules (the Durable Object alarm on celld, +/// `AggregateCell::fire_due_schedules` on the reference host) and stamps the +/// cell clock. +impl crate::microsvc::BackendHost for CellStreamStore { + fn capabilities(&self) -> crate::schedule::Capabilities { + crate::schedule::Capabilities::ALL + } + + fn host_now( + &self, + ) -> impl Future> + Send + '_ { + std::future::ready(Ok(self.now())) + } +} + impl TransactionalCommit for CellStreamStore { fn commit_batch<'a>( &'a self, @@ -763,7 +778,8 @@ impl TransactionalCommit for CellStreamStore { let mut batch = batch; async move { self.ensure_batch(&batch)?; - super::alarm::resolve_alarm_writes(&mut batch.alarms, self.clock.now())?; + crate::schedule::schedule_owner(&batch)?; + super::alarm::resolve_schedule_writes(&mut batch.schedules, self.clock.now())?; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] let sealed = self.reduce_memory_sealed_row(&batch)?; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] @@ -789,7 +805,8 @@ impl CausalTransactionalCommit for CellStreamStore { async move { self.ensure_batch(&batch.domain) .map_err(CommandLedgerError::Storage)?; - super::alarm::resolve_alarm_writes(&mut batch.domain.alarms, self.clock.now()) + crate::schedule::schedule_owner(&batch.domain).map_err(CommandLedgerError::Storage)?; + super::alarm::resolve_schedule_writes(&mut batch.domain.schedules, self.clock.now()) .map_err(CommandLedgerError::Storage)?; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] let sealed = self diff --git a/src/microsvc/cell_host/tests/alarms.rs b/src/microsvc/cell_host/tests/alarms.rs index db2748adc..dbfd1e39e 100644 --- a/src/microsvc/cell_host/tests/alarms.rs +++ b/src/microsvc/cell_host/tests/alarms.rs @@ -1,7 +1,8 @@ -//! Re-armable cell alarms on the in-process reference host, driven by a +//! Schedules on the in-process reference cell host, driven by a //! `ManualClock`. The same command/ledger/settlement path runs on celld; only //! the wake source differs (Durable Object alarm vs. the test calling -//! `fire_due_alarms`). +//! `fire_due_schedules`). Also: schedule rows on the in-memory repository, +//! `ctx.now()`, and the capability check at registration. use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, SystemTime}; @@ -9,16 +10,17 @@ use std::time::{Duration, SystemTime}; use serde::{Deserialize, Serialize}; use serde_json::json; -use super::super::{ - cell_alarm_guard, cell_alarm_session, AggregateCell, CellAlarm, CellAlarmWrite, - CellCommandIdentity, CellStreamStore, CELL_ALARM_RETRY_BASE, -}; +use super::super::{AggregateCell, CellCommandIdentity, CellStreamStore}; use super::owner_session; use crate::command::{typed_command, PreparedCommand, Succeeded}; use crate::entity::Entity; use crate::microsvc::service::{CausalCommandContext, PortableCommand, Routes}; use crate::microsvc::HandlerError; use crate::repository::{CommitBatch, StreamIdentity, TransactionalCommit}; +use crate::schedule::{ + schedule_guard, schedule_session, Capability, Schedule, ScheduleWrite, HOST_NOW_CLAIM, + SCHEDULE_RETRY_BASE, +}; use crate::{sourced, Clock, InMemoryRepository, ManualClock}; #[derive(Clone, Default, Serialize, Deserialize, crate::Snapshot)] @@ -62,6 +64,8 @@ struct TickInput { struct TickerPayload { id: String, ticks: u32, + /// `ctx.now()` of the start command (epoch ms), 0 otherwise. + started_at_ms: u64, } const TICK: &str = "ticker.tick"; @@ -76,6 +80,8 @@ where .typed_command(typed_command::>( "ticker.start", )) + .requires(Capability::Scheduling) + .requires(Capability::Clock) .guarded( |ctx: &CausalCommandContext<'_, Ticker>| ctx.session().user_id().is_some(), handle_start, @@ -92,8 +98,8 @@ async fn handle_start( ticker .start(input.id.clone(), input.limit, input.period_ms) .map_err(|error| HandlerError::Rejected(error.to_string()))?; - ctx.arm_alarm( - CellAlarm::after( + ctx.schedule( + Schedule::after( "tick", Duration::from_millis(input.period_ms), &input.tick_command, @@ -106,9 +112,14 @@ async fn handle_start( if input.fail_after_arming { return Err(HandlerError::Rejected("start refused after arming".into())); } + let started_at_ms = ctx + .now()? + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis() as u64); repo.commit(ticker)?.succeeded(TickerPayload { id: input.id, ticks: 0, + started_at_ms, }) } @@ -120,7 +131,8 @@ where fn install(self, routes: Routes) -> Routes { routes .typed_command(typed_command::>(TICK)) - .guarded(cell_alarm_guard::, handle_tick) + .requires(Capability::Scheduling) + .guarded(schedule_guard::, handle_tick) } } @@ -138,9 +150,10 @@ async fn handle_tick( .map_err(|error| HandlerError::Rejected(error.to_string()))?; if ticker.ticks < ticker.limit { // Re-arm from the fired due time, not wall time: no drift. - let due = ctx.alarm_due().expect("alarm session") + Duration::from_millis(ticker.period_ms); - ctx.arm_alarm( - CellAlarm::at("tick", due, TICK, &input) + let due = ctx.scheduled_due().expect("schedule session") + + Duration::from_millis(ticker.period_ms); + ctx.schedule( + Schedule::at("tick", due, TICK, &input) .map_err(|error| HandlerError::Rejected(error.to_string()))?, )?; } @@ -148,6 +161,7 @@ async fn handle_tick( repo.commit(ticker)?.succeeded(TickerPayload { id: input.id, ticks, + started_at_ms: 0, }) } @@ -161,6 +175,7 @@ where .typed_command(typed_command::>( "ticker.stop", )) + .requires(Capability::Scheduling) .guarded( |ctx: &CausalCommandContext<'_, Ticker>| ctx.session().user_id().is_some(), handle_stop, @@ -177,11 +192,12 @@ async fn handle_stop( .get(&input.id) .await? .ok_or_else(|| HandlerError::Rejected("no ticker".into()))?; - ctx.cancel_alarm("tick")?; + ctx.cancel_schedule("tick")?; let ticks = ticker.ticks; repo.commit(ticker)?.succeeded(TickerPayload { id: input.id, ticks, + started_at_ms: 0, }) } @@ -214,7 +230,8 @@ where .typed_command(typed_command::>( "ticker.flaky", )) - .guarded(cell_alarm_guard::, handle_flaky) + .requires(Capability::Scheduling) + .guarded(schedule_guard::, handle_flaky) } } @@ -276,17 +293,17 @@ async fn alarm_commits_with_the_events_and_is_absent_when_the_command_fails() { ) .await; assert!(refused.is_err()); - assert!(cell.armed_alarms().unwrap().is_empty()); + assert!(cell.armed_schedules().unwrap().is_empty()); assert!(cell.load().await.unwrap().is_none()); start(&cell, json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; - let armed = cell.armed_alarms().unwrap(); + let armed = cell.armed_schedules().unwrap(); assert_eq!(armed.len(), 1); assert_eq!(armed[0].name, "tick"); assert_eq!(armed[0].command, TICK); assert_eq!(armed[0].generation, 1); assert_eq!( - cell.next_alarm_due().unwrap(), + cell.next_schedule_due().unwrap(), Some(clock.now() + Duration::from_millis(1000)), "a relative alarm resolves against the cell clock at commit" ); @@ -299,16 +316,16 @@ async fn alarm_fires_once_per_arm_with_the_trusted_session() { start(&cell, json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; clock.advance(Duration::from_millis(999)); - assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(cell.fire_due_schedules().await.unwrap().is_empty()); clock.advance(Duration::from_millis(1)); - let report = cell.fire_due_alarms().await.unwrap(); + let report = cell.fire_due_schedules().await.unwrap(); assert_eq!(report.fired.len(), 1); assert!(!report.fired[0].replayed); assert_eq!(report.fired[0].payload["ticks"], 1); clock.advance(Duration::from_secs(60)); - assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(cell.fire_due_schedules().await.unwrap().is_empty()); assert_eq!(ticks(&cell).await, 1); - assert!(cell.armed_alarms().unwrap().is_empty()); + assert!(cell.armed_schedules().unwrap().is_empty()); // The alarm-only command refuses ordinary callers. let forged = cell @@ -328,21 +345,21 @@ async fn rearming_in_the_handler_ticks_one_hundred_times_without_drift() { let clock = clock(); let cell = ticker_cell(&clock); start(&cell, json!({ "id": "t1", "limit": 100, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" })).await; - let first_due = cell.next_alarm_due().unwrap().unwrap(); + let first_due = cell.next_schedule_due().unwrap().unwrap(); for tick in 1..=100u64 { let expected_due = first_due + Duration::from_millis((tick - 1) * 1000); - assert_eq!(cell.next_alarm_due().unwrap(), Some(expected_due)); + assert_eq!(cell.next_schedule_due().unwrap(), Some(expected_due)); // Wake late; the next tick still re-arms from the due time. clock.set(expected_due + Duration::from_millis(250)); - let report = cell.fire_due_alarms().await.unwrap(); + let report = cell.fire_due_schedules().await.unwrap(); assert_eq!(report.fired.len(), 1, "tick {tick}"); assert_eq!(report.fired[0].generation, tick); assert!(report.failed.is_empty() && report.retried.is_empty()); } assert_eq!(ticks(&cell).await, 100); - assert!(cell.armed_alarms().unwrap().is_empty()); + assert!(cell.armed_schedules().unwrap().is_empty()); clock.advance(Duration::from_secs(3600)); - assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(cell.fire_due_schedules().await.unwrap().is_empty()); assert_eq!(ticks(&cell).await, 100); } @@ -354,22 +371,22 @@ async fn a_fired_command_that_committed_before_settlement_replays_instead_of_rer clock.advance(Duration::from_secs(1)); // Crash window: the alarm command committed but its row was not settled. - let alarm = cell.armed_alarms().unwrap().remove(0); + let alarm = cell.armed_schedules().unwrap().remove(0); cell.dispatch_idempotent( &alarm.command, &alarm.command_identity().unwrap(), alarm.input.clone(), - cell_alarm_session(&alarm), + schedule_session(&alarm), ) .await .unwrap(); assert_eq!(ticks(&cell).await, 1); - let report = cell.fire_due_alarms().await.unwrap(); + let report = cell.fire_due_schedules().await.unwrap(); assert_eq!(report.fired.len(), 1); assert!(report.fired[0].replayed, "the ledger replays the receipt"); assert_eq!(ticks(&cell).await, 1, "the handler did not run twice"); - assert!(cell.armed_alarms().unwrap().is_empty()); + assert!(cell.armed_schedules().unwrap().is_empty()); } #[tokio::test] @@ -385,9 +402,9 @@ async fn cancel_disarms_and_rejections_are_terminal() { ) .await .unwrap(); - assert!(cell.armed_alarms().unwrap().is_empty()); + assert!(cell.armed_schedules().unwrap().is_empty()); clock.advance(Duration::from_secs(10)); - assert!(cell.fire_due_alarms().await.unwrap().is_empty()); + assert!(cell.fire_due_schedules().await.unwrap().is_empty()); let other = ticker_cell(&clock); start( @@ -396,11 +413,11 @@ async fn cancel_disarms_and_rejections_are_terminal() { ) .await; clock.advance(Duration::from_secs(1)); - let report = other.fire_due_alarms().await.unwrap(); + let report = other.fire_due_schedules().await.unwrap(); assert!(report.fired.is_empty()); assert_eq!(report.failed.len(), 1); assert_eq!(report.failed[0].code, "BAD_REQUEST"); - assert!(other.armed_alarms().unwrap().is_empty()); + assert!(other.armed_schedules().unwrap().is_empty()); } #[tokio::test] @@ -414,23 +431,23 @@ async fn retryable_failures_back_off_and_then_fire() { ) .await; clock.advance(Duration::from_secs(1)); - let first = cell.fire_due_alarms().await.unwrap(); + let first = cell.fire_due_schedules().await.unwrap(); assert_eq!(first.retried.len(), 1); assert_eq!(first.retried[0].code, "INTERNAL"); - let armed = cell.armed_alarms().unwrap(); + let armed = cell.armed_schedules().unwrap(); assert_eq!(armed[0].attempts, 1); - assert_eq!(armed[0].due(), clock.now() + CELL_ALARM_RETRY_BASE); + assert_eq!(armed[0].due(), clock.now() + SCHEDULE_RETRY_BASE); - clock.advance(CELL_ALARM_RETRY_BASE); - let second = cell.fire_due_alarms().await.unwrap(); + clock.advance(SCHEDULE_RETRY_BASE); + let second = cell.fire_due_schedules().await.unwrap(); assert_eq!(second.retried.len(), 1); assert_eq!( - cell.armed_alarms().unwrap()[0].due(), - clock.now() + CELL_ALARM_RETRY_BASE * 2, + cell.armed_schedules().unwrap()[0].due(), + clock.now() + SCHEDULE_RETRY_BASE * 2, "backoff doubles" ); - clock.advance(CELL_ALARM_RETRY_BASE * 2); - let third = cell.fire_due_alarms().await.unwrap(); + clock.advance(SCHEDULE_RETRY_BASE * 2); + let third = cell.fire_due_schedules().await.unwrap(); assert_eq!(third.fired.len(), 1); assert_eq!(ticks(&cell).await, 1); } @@ -446,19 +463,61 @@ async fn armed_alarms_survive_a_reference_host_restart() { let restarted = ticker_cell(&clock); restarted.restore_durable_state(exported).unwrap(); clock.advance(Duration::from_secs(1)); - assert_eq!(restarted.fire_due_alarms().await.unwrap().fired.len(), 1); + assert_eq!(restarted.fire_due_schedules().await.unwrap().fired.len(), 1); clock.advance(Duration::from_secs(1)); - assert_eq!(restarted.fire_due_alarms().await.unwrap().fired.len(), 1); + assert_eq!(restarted.fire_due_schedules().await.unwrap().fired.len(), 1); assert_eq!(ticks(&restarted).await, 2); } #[tokio::test] -async fn repositories_without_an_alarm_slot_reject_alarm_writes() { - let repo = InMemoryRepository::new(); - let mut batch = CommitBatch::empty(); - batch.alarms.push(CellAlarmWrite::Cancel("tick".into())); - let error = repo.commit_batch(batch).await.unwrap_err(); - assert!(error.to_string().contains("aggregate cell"), "{error}"); +async fn the_in_memory_repository_stores_schedules_with_their_one_owner() { + let clock = clock(); + let repo = InMemoryRepository::new().with_clock(clock.clone()); + let owner = StreamIdentity::new("Ticker", "t9").unwrap(); + let mut ticker = Ticker::default(); + ticker.start("t9".into(), 1, 1000).unwrap(); + let mut batch = CommitBatch::new(vec![crate::repository::StreamWrite::new( + owner.clone(), + &mut ticker.entity, + )]); + batch.schedules.push(ScheduleWrite::Arm( + Schedule::after( + "tick", + Duration::from_secs(5), + TICK, + &TickInput { id: "t9".into() }, + ) + .unwrap(), + )); + repo.commit_batch(batch).await.unwrap(); + let armed = repo.armed_schedules(&owner).unwrap(); + assert_eq!(armed.len(), 1); + assert_eq!(armed[0].generation, 1); + assert_eq!( + armed[0].due(), + clock.now() + Duration::from_secs(5), + "a relative schedule resolves against the repository clock" + ); + + // A schedule needs exactly one owning stream. + let mut orphan = CommitBatch::empty(); + orphan.schedules.push(ScheduleWrite::Cancel("tick".into())); + let error = repo.commit_batch(orphan).await.unwrap_err(); + assert!( + error.to_string().contains("exactly one aggregate"), + "{error}" + ); +} + +#[test] +#[should_panic( + expected = "command `ticker.start` requires scheduling that its backend does not provide" +)] +fn a_scheduling_route_on_a_backend_that_cannot_dispatch_fails_at_registration() { + let routes = Routes::new().with_repo(crate::aggregate::AggregateRepository::new( + InMemoryRepository::new(), + )); + let _ = Start.install(routes); } #[tokio::test] @@ -466,8 +525,8 @@ async fn handlers_see_the_cell_clock_and_callers_cannot_forge_it() { let clock = clock(); let cell = ticker_cell(&clock); let mut forged = owner_session(); - forged.set(super::super::CELL_NOW_CLAIM, "1"); - cell.dispatch_idempotent( + forged.set(HOST_NOW_CLAIM, "1"); + let started = cell.dispatch_idempotent( "ticker.start", &identity("0190a000-0000-7000-8000-000000000009"), json!({ "id": "t1", "limit": 1, "period_ms": 1000, "fail_after_arming": false, "tick_command": "ticker.tick" }), @@ -475,16 +534,21 @@ async fn handlers_see_the_cell_clock_and_callers_cannot_forge_it() { ) .await .unwrap(); + assert_eq!( + started.payload()["started_at_ms"], + START_MS, + "ctx.now() is the cell clock, not the forged claim" + ); // `after(period)` resolved by the cell clock, not the forged claim. assert_eq!( - cell.next_alarm_due().unwrap(), + cell.next_schedule_due().unwrap(), Some(clock.now() + Duration::from_millis(1000)) ); let mut session = crate::microsvc::Session::new(); - assert_eq!(super::super::cell_now(&session), None); - session.set(super::super::CELL_NOW_CLAIM, "1234"); + assert_eq!(crate::schedule::host_now(&session), None); + session.set(HOST_NOW_CLAIM, "1234"); assert_eq!( - super::super::cell_now(&session), + crate::schedule::host_now(&session), Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1234)) ); } diff --git a/src/microsvc/dependencies.rs b/src/microsvc/dependencies.rs index e9870e831..13bb0621e 100644 --- a/src/microsvc/dependencies.rs +++ b/src/microsvc/dependencies.rs @@ -25,6 +25,7 @@ pub trait HasRepo { #[allow(private_bounds)] pub trait CausalRepositoryBackend: CausalGetStream + + BackendHost + CommandLedgerStore + CausalTransactionalCommit + CausalRepositoryIdentity @@ -38,6 +39,7 @@ pub trait CausalRepositoryBackend: impl CausalRepositoryBackend for T where T: CausalGetStream + + BackendHost + CommandLedgerStore + CausalTransactionalCommit + CausalRepositoryIdentity @@ -174,6 +176,22 @@ pub trait CausalRouteDependencies { -> &AggregateRepository; } +/// What a command backend provides to the commands it hosts +/// ([`crate::schedule::Capabilities`]) and its clock. Implemented by the +/// framework's repository adapters; checked when a route is registered. +pub(crate) trait BackendHost: Send + Sync { + /// The backend's capabilities. + fn capabilities(&self) -> crate::schedule::Capabilities; + + /// The backend's clock now (store clock, Durable Object clock, database + /// clock). + fn host_now( + &self, + ) -> impl std::future::Future> + + Send + + '_; +} + /// Dependency capability for services that expose a read-model store. pub trait HasReadModelStore { type ReadModelStore; diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index d7e5703fb..0344a10da 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -59,7 +59,7 @@ mod causal; pub mod cell_host; mod context; mod dependencies; -pub(crate) use dependencies::CausalHostProjections; +pub(crate) use dependencies::{BackendHost, CausalHostProjections}; mod descriptor; mod error; pub(crate) mod lifecycle; @@ -105,9 +105,9 @@ pub use runtime::{DEFAULT_MAX_PUBLISH_ATTEMPTS, DEFAULT_PUBLISH_LEASE}; #[cfg(all(feature = "graphql", test))] pub(crate) use service::CausalCommandProjectionEvidence; #[cfg(feature = "graphql")] -pub use service::GraphqlServiceBindError; -#[cfg(feature = "graphql")] pub(crate) use service::ExternalCausalReservation; +#[cfg(feature = "graphql")] +pub use service::GraphqlServiceBindError; pub use service::{ direct_read_model, invoke_transition, require_loaded, CausalCommandContext, CausalCommitBuilder, CausalRepository, CommandRequest, CommandResponse, DeliveryKind, diff --git a/src/microsvc/service/causal.rs b/src/microsvc/service/causal.rs index 7c70703bc..43971e138 100644 --- a/src/microsvc/service/causal.rs +++ b/src/microsvc/service/causal.rs @@ -219,6 +219,9 @@ pub struct CausalDispatchResult { /// Domain-event evidence used to seal wait-path projection metadata. /// Transport delivery remains owned by the aggregate cell and Queue. pub(crate) projection_events: Vec, + /// The command ledger replayed an earlier receipt (the command id was + /// already settled); the handler did not run again. + pub(crate) replayed: bool, } /// Gateway-side capability for one externally dispatched command. The attempt @@ -359,6 +362,18 @@ impl CausalDispatchResult { &self.receipt.command_id } + /// Whether this result replays an earlier settled receipt. + pub fn replayed(&self) -> bool { + self.replayed + } + + /// The receipt of the attempt that just committed it (read back from + /// the ledger), not a replay. + pub(crate) fn fresh(mut self) -> Self { + self.replayed = false; + self + } + /// Ledger causation id assigned on accept. pub fn causation_id(&self) -> &str { &self.receipt.causation_id @@ -510,7 +525,7 @@ impl CausalDispatchResult { let state = crate::command_ledger::CommandLedgerState::parse(&wire.receipt.state).map_err( |err| CausalDispatchError::Internal(format!("wait-path receipt state: {err}")), )?; - let _ = wire.receipt.replayed; + let replayed = wire.receipt.replayed.unwrap_or(false); let projection_events = crate::microsvc::cell_host::parse_cell_projection_events(&serde_json::json!({ "events": wire.events @@ -523,6 +538,7 @@ impl CausalDispatchResult { Ok(Self { payload: wire.payload, projection_events, + replayed, receipt: CausalCommandReceiptSource { command_id: wire.receipt.command_id, command_name: String::new(), @@ -740,6 +756,7 @@ pub(super) fn replay_result( payload: receipt.outcome.clone(), receipt, projection_events: Vec::new(), + replayed: true, }) } CommandLedgerState::Rejected => replay_rejection(replay.outcome), @@ -844,7 +861,10 @@ where .await .map_err(internal_ledger_error)? { - CommandLookup::Replay(replay) => replay_result(consistency, replay), + // The receipt this attempt just wrote: not a replay. + CommandLookup::Replay(replay) => { + replay_result(consistency, replay).map(CausalDispatchResult::fresh) + } CommandLookup::Expired => Err(CausalDispatchError::Expired), CommandLookup::InProgress { .. } | CommandLookup::RetryableUnknown { .. } @@ -903,7 +923,9 @@ where .lookup_command(fence.key(), CommandLookupScope::Attempt(&fence)) .await { - Ok(CommandLookup::Replay(replay)) => replay_result(consistency, replay), + Ok(CommandLookup::Replay(replay)) => { + replay_result(consistency, replay).map(CausalDispatchResult::fresh) + } Ok(CommandLookup::Expired) => Err(CausalDispatchError::Expired), Ok(CommandLookup::RetryableUnknown { .. }) => Err(CausalDispatchError::Internal(detail)), Ok(CommandLookup::InProgress { .. }) => { diff --git a/src/microsvc/service/handlers.rs b/src/microsvc/service/handlers.rs index 58c16c3bd..f544c49f4 100644 --- a/src/microsvc/service/handlers.rs +++ b/src/microsvc/service/handlers.rs @@ -686,37 +686,50 @@ where CausalCommitBuilder::empty(self).outbox(message) } - /// Arm (or re-arm) a cell alarm with this command's effects. + /// Schedule (or re-schedule) work for the aggregate this command + /// commits ([`crate::schedule`]). /// - /// The alarm row commits in the same transaction as the command's events - /// and receipt; if the command fails, nothing is armed. At the alarm's due - /// time the hosting cell dispatches its command into itself with the - /// trusted session from [`crate::cell_host::cell_alarm_session`]. Only - /// aggregate cells accept alarms: other hosts reject the commit. - pub fn arm_alarm(&self, alarm: crate::cell_host::CellAlarm) -> Result<(), HandlerError> { + /// The schedule row commits in the same transaction as the command's + /// events and receipt, on every backend; if the command fails, nothing is + /// scheduled. The command must commit exactly one aggregate, which owns + /// it. At the due time a host with [`Capability::Scheduling`] dispatches + /// its command into that aggregate with the trusted + /// [`schedule_session`](crate::schedule::schedule_session); declare the + /// route with `.requires(Capability::Scheduling)` so a host that cannot + /// dispatch refuses it at service construction. + /// + /// [`Capability::Scheduling`]: crate::schedule::Capability::Scheduling + pub fn schedule(&self, schedule: crate::schedule::Schedule) -> Result<(), HandlerError> { self.workspace - .stage_alarm(crate::cell_host::CellAlarmWrite::Arm(alarm)) + .stage_schedule(crate::schedule::ScheduleWrite::Arm(schedule)) .map_err(workspace_handler_error) } - /// Disarm a cell alarm with this command's effects (no-op when unarmed). - pub fn cancel_alarm(&self, name: impl Into) -> Result<(), HandlerError> { + /// Cancel a schedule of this command's aggregate with its effects (no-op + /// when it is not armed). + pub fn cancel_schedule(&self, name: impl Into) -> Result<(), HandlerError> { self.workspace - .stage_alarm(crate::cell_host::CellAlarmWrite::Cancel(name.into())) + .stage_schedule(crate::schedule::ScheduleWrite::Cancel(name.into())) .map_err(workspace_handler_error) } - /// The cell clock's time when this command was dispatched (the Durable - /// Object clock on celld, the store clock — e.g. a `ManualClock` — on the - /// reference host). `None` outside an aggregate cell. - pub fn cell_now(&self) -> Option { - crate::cell_host::cell_now(self.session) + /// The host clock when this command was dispatched: the Durable Object + /// clock on celld, the store clock on the reference host and the + /// in-memory repository (e.g. a `ManualClock`), the database clock on + /// SQL. Never a caller's time. Requires the route to declare + /// `.requires(Capability::Clock)`; otherwise an error says so. + pub fn now(&self) -> Result { + crate::schedule::host_now(self.session).ok_or_else(|| { + HandlerError::Rejected( + "ctx.now() needs a route declared with .requires(Capability::Clock)".into(), + ) + }) } - /// Due time of the alarm that fired this command; `None` for any command - /// not dispatched by a cell alarm. - pub fn alarm_due(&self) -> Option { - crate::cell_host::cell_alarm_due(self.session) + /// Due time of the schedule that fired this command; `None` for any + /// command not dispatched by a schedule. + pub fn scheduled_due(&self) -> Option { + crate::schedule::scheduled_due(self.session) } /// Low-level outbox staging retained for framework internals. diff --git a/src/microsvc/service/routes.rs b/src/microsvc/service/routes.rs index a1171ee10..5d615912d 100644 --- a/src/microsvc/service/routes.rs +++ b/src/microsvc/service/routes.rs @@ -1,17 +1,10 @@ #[cfg(feature = "graphql")] -use std::collections::BTreeMap; -use std::collections::{BTreeSet, HashMap}; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -use std::time::Duration; -#[cfg(feature = "graphql")] use super::causal::{ abandon_causal_attempt, causal_handler_error_code, commit_causal_rejection, ensure_causal_grant, evaluate_causal_command_status, internal_ledger_error, load_committed_dispatch_result, recover_causal_commit_error, replay_result, - CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, - ExternalCausalAttempt, ExternalCausalReservation, + CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, ExternalCausalAttempt, + ExternalCausalReservation, }; use super::handlers::{ boxed_causal_guard, boxed_handler, boxed_prepared_handler, CausalCommandContext, CausalGuardFn, @@ -75,6 +68,13 @@ use crate::outbox_worker::{ use crate::projection_protocol::{CompiledProjectionTopology, ProjectorTopologyId}; use crate::repository::{StreamIdentity, TransactionalCommit}; use serde_json::Value; +#[cfg(feature = "graphql")] +use std::collections::BTreeMap; +use std::collections::{BTreeSet, HashMap}; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; /// How a handler expects the transport to deliver matching messages. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -300,6 +300,12 @@ where contract: TypedCommandContract, guard: Option>>, handle: Arc>, + /// The route declared [`Capability::Clock`](crate::schedule::Capability): + /// the dispatcher stamps the host clock into its session (`ctx.now()`). + requires_clock: bool, + /// The route declared [`Capability::Scheduling`](crate::schedule::Capability): + /// only then may its handler stage schedule writes. + requires_scheduling: bool, /// Retryable, fail-closed bootstrap for the bound projector's complete /// model/table ownership inventory. `get_or_try_init` leaves the cell empty /// after a transient registration failure. @@ -317,11 +323,15 @@ where contract: TypedCommandContract, guard: Option>>, handle: Arc>, + requires_clock: bool, + requires_scheduling: bool, ) -> Self { Self { contract, guard, handle, + requires_clock, + requires_scheduling, #[cfg(feature = "graphql")] direct_projection_bootstrap: tokio::sync::OnceCell::new(), _types: std::marker::PhantomData, @@ -329,6 +339,26 @@ where } } +/// The session a command runs with: any caller-supplied host-clock claim +/// removed, the backend's clock stamped when the route declared +/// [`Capability::Clock`](crate::schedule::Capability::Clock). +async fn stamped_session( + requires_clock: bool, + repository: &R, + mut session: Session, +) -> Result +where + R: crate::microsvc::dependencies::BackendHost, +{ + let now = if requires_clock { + Some(repository.host_now().await?) + } else { + None + }; + crate::schedule::stamp_host_now(&mut session, now); + Ok(session) +} + pub(super) type OutboxConfigurator = fn(&mut D, DynBusPublisher, String, Duration, u32, Option); @@ -596,6 +626,7 @@ pub struct TypedRouteBuilder { route_name: &'static str, contract: TypedCommandContract, mount: Option, + requires: Vec, _types: std::marker::PhantomData K>, } @@ -685,6 +716,20 @@ where self.mount = Some(mount); self } + + /// Declare a host capability this command needs + /// ([`Capability::Scheduling`](crate::schedule::Capability::Scheduling) + /// for `ctx.schedule`, [`Capability::Clock`](crate::schedule::Capability::Clock) + /// for `ctx.now()`). Registering the route on a backend without it panics + /// with the missing capabilities, at service construction rather than at + /// the first commit. + #[must_use] + pub fn requires(mut self, capability: crate::schedule::Capability) -> Self { + if !self.requires.contains(&capability) { + self.requires.push(capability); + } + self + } } impl TypedRouteBuilder @@ -705,6 +750,7 @@ where self.mount, None, boxed_prepared_handler(handler), + self.requires, ) } @@ -721,6 +767,7 @@ where self.mount, Some(guard), boxed_prepared_handler(handler), + self.requires, ) } @@ -1143,6 +1190,7 @@ impl Routes { route_name, contract, mount: None, + requires: Vec::new(), _types: std::marker::PhantomData, } } @@ -1357,6 +1405,7 @@ impl Routes { declared_mount: Option, guard: Option>>, handle: Arc>, + requires: Vec, ) -> Self where D: CausalRouteDependencies, @@ -1368,6 +1417,21 @@ impl Routes { route_name, contract.name, "typed command route and contract ids must match" ); + let capabilities = crate::microsvc::dependencies::BackendHost::capabilities( + self.dependencies.__causal_aggregate_repository().repo(), + ); + let missing = capabilities.missing(&requires); + assert!( + missing.is_empty(), + "command `{route_name}` requires {} that its backend does not provide \ + (backend capabilities: {capabilities:?})", + missing + .iter() + .map(ToString::to_string) + .collect::>() + .join(" and "), + ); + let requires_clock = requires.contains(&crate::schedule::Capability::Clock); let by_name = self.handlers.entry(MessageKind::Command).or_default(); assert!( !by_name.contains_key(route_name), @@ -1392,7 +1456,13 @@ impl Routes { by_name.insert( route_name.to_string(), RegisteredHandler::Causal(Box::new( - RegisteredCausalHandler::::new(contract, guard, handle), + RegisteredCausalHandler::::new( + contract, + guard, + handle, + requires_clock, + requires.contains(&crate::schedule::Capability::Scheduling), + ), )), ); mount @@ -1644,6 +1714,9 @@ where let aggregate_repository = dependencies.__causal_aggregate_repository(); let repository = aggregate_repository.repo(); + let session = stamped_session(self.requires_clock, repository, session) + .await + .map_err(|error| CellDispatchError::Internal(error.to_string()))?; let attempt = match repository .reserve_command(reservation) .await @@ -1684,7 +1757,8 @@ where metadata, }; - let workspace = CausalWorkspace::new(aggregate_repository); + let workspace = CausalWorkspace::new(aggregate_repository) + .with_scheduling_declared(self.requires_scheduling); let context = CausalCommandContext::new(&message, &session, &workspace); if self.guard.as_ref().is_some_and(|guard| !guard(&context)) { return crate::microsvc::cell_host::causal::commit_rejection( @@ -1903,6 +1977,9 @@ where let aggregate_repository = dependencies.__causal_aggregate_repository(); let repository = aggregate_repository.repo(); + let session = stamped_session(self.requires_clock, repository, session) + .await + .map_err(|error| CausalDispatchError::Internal(error.to_string()))?; if let Some(target) = direct_projection_target.as_ref() { let (topology, ownership) = target.registration(); self.direct_projection_bootstrap @@ -1958,7 +2035,8 @@ where metadata, }; - let workspace = CausalWorkspace::new(aggregate_repository); + let workspace = CausalWorkspace::new(aggregate_repository) + .with_scheduling_declared(self.requires_scheduling); let context = CausalCommandContext::new(&message, &session, &workspace); if self.guard.as_ref().is_some_and(|guard| !guard(&context)) { return commit_causal_rejection( @@ -2475,7 +2553,12 @@ where .map_err(|error| HandlerError::DecodeFailed(error.to_string()))?; let message = Message::new(self.contract.name.clone(), MessageKind::Command, payload); let aggregate_repository = dependencies.__causal_aggregate_repository(); - let workspace = CausalWorkspace::new(aggregate_repository); + let session = + stamped_session(self.requires_clock, aggregate_repository.repo(), session) + .await + .map_err(HandlerError::Repository)?; + let workspace = CausalWorkspace::new(aggregate_repository) + .with_scheduling_declared(self.requires_scheduling); let context = CausalCommandContext::new(&message, &session, &workspace); if self.guard.as_ref().is_some_and(|guard| !guard(&context)) { return Err(HandlerError::GuardRejected(self.contract.name.clone())); diff --git a/src/microsvc/service/tests.rs b/src/microsvc/service/tests.rs index 8fdf19e5b..4c9578234 100644 --- a/src/microsvc/service/tests.rs +++ b/src/microsvc/service/tests.rs @@ -13,6 +13,8 @@ use crate::command::{typed_command, PreparedCommand, Succeeded}; use crate::command::{Atomic, CommandConsistency}; use crate::command::{CommandInputType, CommandOutputType, CommandTypeDef, CommandTypeField}; #[cfg(feature = "graphql")] +use crate::command_dispatch::SharedCommandHost; +#[cfg(feature = "graphql")] use crate::command_ledger::{ AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, CausalTransactionalCommit, CommandLedgerError, CommandLedgerKey, CommandLedgerState, @@ -20,15 +22,13 @@ use crate::command_ledger::{ ExternalCommandCompletion, ReservationOutcome, }; #[cfg(feature = "graphql")] -use crate::command_dispatch::SharedCommandHost; -#[cfg(feature = "graphql")] use crate::graphql::identity::VerifiedPrincipal; #[cfg(feature = "graphql")] use crate::graphql::{SurfaceDirectProjection, SurfaceProjector}; #[cfg(feature = "graphql")] -use crate::microsvc::HasOutboxStore; -#[cfg(feature = "graphql")] use crate::microsvc::cell_host::{CelldCommandHost, InternalHttpSecret}; +#[cfg(feature = "graphql")] +use crate::microsvc::HasOutboxStore; use crate::microsvc::{ CommandRequest, Context, HandlerError, RepoReadModelDependencies, Routes, Service, Session, }; @@ -1055,6 +1055,21 @@ impl CommandLedgerStore for AmbiguousCommitRepository { } } +#[cfg(feature = "graphql")] +impl crate::microsvc::BackendHost for AmbiguousCommitRepository { + fn capabilities(&self) -> crate::schedule::Capabilities { + self.inner.capabilities() + } + + fn host_now( + &self, + ) -> impl std::future::Future> + + Send + + '_ { + self.inner.host_now() + } +} + #[cfg(feature = "graphql")] impl crate::repository::TransactionalCommit for AmbiguousCommitRepository { fn commit_batch<'a>( @@ -1467,7 +1482,9 @@ async fn causal_dispatch_receipt_and_status_use_the_exact_durable_replay() { .await .expect("response-loss retry should recover the same receipt source"); - assert_eq!(first, replay); + assert!(!first.replayed(), "the first dispatch ran the handler"); + assert!(replay.replayed(), "the retry replays the durable receipt"); + assert_eq!(first, replay.clone().fresh()); assert_eq!(first.payload, json!({ "id": "todo-receipt" })); assert_eq!(first.receipt.command_id, command_id); assert_eq!(first.receipt.state, CommandLedgerState::Succeeded); @@ -2880,7 +2897,9 @@ async fn graphql_succeeded_status_evaluates_retained_projection_evidence() { .len(), 1 ); - assert!(before_celld_envelope["command"].get("observations").is_none()); + assert!(before_celld_envelope["command"] + .get("observations") + .is_none()); let pending = repository .outbox_store() diff --git a/src/microsvc/session.rs b/src/microsvc/session.rs index 019e979c1..1efd863fc 100644 --- a/src/microsvc/session.rs +++ b/src/microsvc/session.rs @@ -136,7 +136,12 @@ impl Session { self.variables.insert(key.into(), value.into()); } - /// Check if a session variable exists. + /// Remove a variable, returning its value. + pub fn remove(&mut self, key: &str) -> Option { + self.variables.remove(key) + } + + /// Whether `key` is set. pub fn has(&self, key: &str) -> bool { self.variables.contains_key(key) } diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index fadc13fff..ce0a8f2e7 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -276,7 +276,7 @@ where read_model_plans: self.read_model_plans, snapshots, inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), }) .await?; if let Some(version) = snapshot_version { diff --git a/src/queued_repo/repository.rs b/src/queued_repo/repository.rs index 66bc0f6cc..325c06072 100644 --- a/src/queued_repo/repository.rs +++ b/src/queued_repo/repository.rs @@ -603,8 +603,11 @@ where R: RelationalReadModelQueryStore, L: LockManager, { - fn scan_read_model(&self, request: crate::read_model::ReadModelScanRequest) - -> impl Future> + Send + '_ { + fn scan_read_model( + &self, + request: crate::read_model::ReadModelScanRequest, + ) -> impl Future> + Send + '_ + { self.inner.scan_read_model(request) } @@ -832,3 +835,20 @@ mod tests { repository.abort(&identity).await.unwrap(); } } + +impl crate::microsvc::BackendHost for QueuedRepository +where + R: crate::microsvc::BackendHost, + L: LockManager, +{ + fn capabilities(&self) -> crate::schedule::Capabilities { + self.inner.capabilities() + } + + fn host_now( + &self, + ) -> impl Future> + Send + '_ + { + self.inner.host_now() + } +} diff --git a/src/repository/sql.rs b/src/repository/sql.rs index e6482c910..6d5fd560b 100644 --- a/src/repository/sql.rs +++ b/src/repository/sql.rs @@ -12,6 +12,8 @@ use std::time::{Duration, SystemTime}; pub(crate) mod ledger; pub(crate) mod outbox; +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub(crate) mod schedules; use crate::entity::{Entity, EventRecord, BITCODE_PAYLOAD_CODEC}; use crate::snapshot::SnapshotRecord; diff --git a/src/repository/sql/schedules.rs b/src/repository/sql/schedules.rs new file mode 100644 index 000000000..811a5c896 --- /dev/null +++ b/src/repository/sql/schedules.rs @@ -0,0 +1,109 @@ +//! Schedule rows of a shared SQL repository (`aggregate_schedules`): one +//! table for every aggregate, keyed by the owning stream and the schedule +//! name, written in the command's transaction. Cells keep their own +//! single-owner `cell_alarms` table. + +use super::{SqlExecutor, SqlRow, Statement}; +use crate::repository::RepositoryError; +use crate::schedule::{ArmedSchedule, ScheduleError, ScheduleWrite, MAX_ARMED_SCHEDULES}; +use crate::StreamIdentity; + +/// Apply one command's resolved schedule writes for `owner`. +pub(crate) async fn apply( + executor: &mut impl SqlExecutor, + owner: &StreamIdentity, + writes: &[ScheduleWrite], +) -> Result<(), RepositoryError> { + for write in writes { + match write { + ScheduleWrite::Arm(schedule) => { + let input = schedule.input_json()?; + executor + .execute( + Statement::new("INSERT INTO aggregate_schedules (aggregate_type, aggregate_id, name, command, input, due_at_ms, generation, attempts, armed) VALUES (") + .bind(owner.aggregate_type().into()) + .sql(", ") + .bind(owner.aggregate_id().into()) + .sql(", ") + .bind(schedule.name().into()) + .sql(", ") + .bind(schedule.command().into()) + .sql(", ") + .bind(input.as_str().into()) + .sql(", ") + .bind(schedule.due_at_ms()?.into()) + .sql(", 1, 0, 1) ON CONFLICT (aggregate_type, aggregate_id, name) DO UPDATE SET command = excluded.command, input = excluded.input, due_at_ms = excluded.due_at_ms, generation = aggregate_schedules.generation + 1, attempts = 0, armed = 1"), + ) + .await?; + } + ScheduleWrite::Cancel(name) => { + executor + .execute( + Statement::new( + "UPDATE aggregate_schedules SET armed = 0, attempts = 0 WHERE aggregate_type = ", + ) + .bind(owner.aggregate_type().into()) + .sql(" AND aggregate_id = ") + .bind(owner.aggregate_id().into()) + .sql(" AND name = ") + .bind(name.as_str().into()), + ) + .await?; + } + } + } + let rows = executor + .query( + Statement::new( + "SELECT COUNT(*) AS armed FROM aggregate_schedules WHERE armed = 1 AND aggregate_type = ", + ) + .bind(owner.aggregate_type().into()) + .sql(" AND aggregate_id = ") + .bind(owner.aggregate_id().into()), + ) + .await?; + let armed = rows + .first() + .map(|row| row.integer("armed")) + .transpose()? + .unwrap_or(0); + if armed > MAX_ARMED_SCHEDULES as i64 { + return Err(ScheduleError::TooManyArmed.into()); + } + Ok(()) +} + +/// Armed schedules of `owner`, earliest first. +pub(crate) async fn armed( + executor: &mut impl SqlExecutor, + owner: &StreamIdentity, +) -> Result, RepositoryError> { + let rows = executor + .query( + Statement::new( + "SELECT name, command, input, due_at_ms, generation, attempts FROM aggregate_schedules WHERE armed = 1 AND aggregate_type = ", + ) + .bind(owner.aggregate_type().into()) + .sql(" AND aggregate_id = ") + .bind(owner.aggregate_id().into()) + .sql(" ORDER BY due_at_ms, name"), + ) + .await?; + rows.iter() + .map(|row| { + let input = row.text("input")?; + Ok(ArmedSchedule { + name: row.text("name")?, + command: row.text("command")?, + input: serde_json::from_str(&input).map_err(|error| { + RepositoryError::Model(format!("schedule input is invalid JSON: {error}")) + })?, + due_at_ms: row.integer("due_at_ms")?, + generation: u64::try_from(row.integer("generation")?).map_err(|_| { + RepositoryError::Model("schedule generation is negative".into()) + })?, + attempts: u32::try_from(row.integer("attempts")?).unwrap_or(u32::MAX), + }) + }) + .collect() +} diff --git a/src/repository/traits.rs b/src/repository/traits.rs index bebecad3f..86445a903 100644 --- a/src/repository/traits.rs +++ b/src/repository/traits.rs @@ -39,9 +39,9 @@ pub struct CommitBatch<'a> { /// Consumer inbox receipts to record in the same transaction (the optional /// effectively-once effect fence). Empty for the default idempotent path. pub inbox_receipts: Vec, - /// Re-armable cell alarm writes. Only aggregate cells accept them; every - /// other repository rejects a batch that carries any. - pub alarms: Vec, + /// Schedule writes ([`crate::schedule`]), owned by the batch's one + /// stream and stored in the same transaction on every backend. + pub schedules: Vec, } impl<'a> CommitBatch<'a> { @@ -52,7 +52,7 @@ impl<'a> CommitBatch<'a> { read_model_plans: Vec::new(), snapshots: Vec::new(), inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), } } @@ -205,8 +205,13 @@ pub trait RelationalReadModelQueryStore: Send + Sync { fn scan_read_model( &self, _request: crate::read_model::ReadModelScanRequest, - ) -> impl Future> + Send + '_ { - async { Err(TableStoreError::Metadata("bounded read-model scan is unsupported by this adapter".into())) } + ) -> impl Future> + Send + '_ + { + async { + Err(TableStoreError::Metadata( + "bounded read-model scan is unsupported by this adapter".into(), + )) + } } fn load_graph( diff --git a/src/schedule.rs b/src/schedule.rs new file mode 100644 index 000000000..b547cd390 --- /dev/null +++ b/src/schedule.rs @@ -0,0 +1,922 @@ +//! Portable aggregate schedules, the host clock and backend capabilities. +//! +//! A command handler schedules work for its own aggregate with its events: +//! +//! ```rust,ignore +//! let now = ctx.now()?; +//! ctx.schedule(Schedule::at("expiry", now + ttl, "zone_lease.expire", &ExpireInput { zone_id })?)?; +//! ``` +//! +//! The schedule row commits in the **same transaction** as the command's +//! events, ledger receipt and outbox rows, on every +//! [`TransactionalCommit`](crate::TransactionalCommit) backend, so a failed +//! command leaves no schedule. The batch must commit exactly one aggregate +//! stream, which owns the schedule. At `due` the host dispatches `command` +//! into that aggregate with the trusted [`schedule_session`]; the handler +//! may schedule the same name again (a periodic tick without a global cron). +//! +//! Semantics, identical on every backend that dispatches schedules: +//! +//! - One row per (owner, **name**). Scheduling an existing name replaces its +//! due time, command and input and bumps its `generation`; cancelling +//! disarms it. +//! - A fired schedule runs through the command ledger with a command id +//! derived from `(name, generation)`, so each arm has its effects at most +//! once even when the host crashes between the commit and the settlement. +//! - Settlement only disarms the generation that fired; a handler that +//! re-schedules its own name produced a newer generation, which stays. +//! - Rejections are terminal for that arm; retryable failures back off +//! exponentially from one second up to five minutes. +//! +//! **Backends declare [`Capabilities`].** Every backend stores schedule rows +//! (cells in their own `cell_alarms` table, the in-memory repository in a +//! per-stream table, SQL in `aggregate_schedules`); only some *dispatch* +//! them ([`Capability::Scheduling`]: aggregate cells today — the Durable +//! Object alarm on celld, `fire_due_schedules` on the reference host). Every +//! backend has a clock ([`Capability::Clock`]): the cell store clock (the DO +//! clock on celld), the in-memory repository's injected +//! [`Clock`](crate::Clock), the database clock on SQL. Routes declare what +//! they need (`.requires(Capability::Scheduling)`); registering one on a host +//! without it fails at service construction, never at the first commit. + +use std::time::{Duration, SystemTime}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::microsvc::Session; +use crate::repository::RepositoryError; +use crate::{ROLE_KEY, USER_ID_KEY}; + +/// Role carried by the trusted session of a fired schedule. Commands that +/// only schedules may run are guarded with [`schedule_guard`]. +pub const SCHEDULE_ROLE: &str = "schedule"; +/// User id of the trusted session of a fired schedule. +pub const SCHEDULE_USER_ID: &str = "system:schedule"; +/// Session claim naming the schedule that fired. Set only by the host. +pub const SCHEDULE_NAME_CLAIM: &str = "x-distributed-schedule"; +/// Session claim with the fired schedule's due time (epoch ms). Periodic +/// handlers re-schedule relative to it so ticks do not drift. +pub const SCHEDULE_DUE_CLAIM: &str = "x-distributed-schedule-due-ms"; +/// Session claim with the fired schedule's generation. +pub const SCHEDULE_GENERATION_CLAIM: &str = "x-distributed-schedule-generation"; +/// Session claim with the host clock's time at dispatch (epoch ms). The +/// dispatcher removes any caller value and sets it for routes that declare +/// [`Capability::Clock`]; read it with `CausalCommandContext::now`. +pub const HOST_NOW_CLAIM: &str = "x-distributed-now-ms"; +/// Command-ledger service id of fired schedules. +pub const SCHEDULE_SERVICE_ID: &str = "distributed.schedule"; +/// Longest schedule name. +pub const MAX_SCHEDULE_NAME_BYTES: usize = 128; +/// Largest serialized schedule input. +pub const MAX_SCHEDULE_INPUT_BYTES: usize = 16 * 1024; +/// Most schedule writes one command may stage. +pub const MAX_SCHEDULE_WRITES_PER_COMMAND: usize = 16; +/// Most schedules one aggregate may hold armed at once. +pub const MAX_ARMED_SCHEDULES: usize = 1024; +/// Most due schedules fired per wake; the rest fire on the next wake. +pub const SCHEDULE_FIRE_BATCH: usize = 32; +/// First retry delay after a retryable schedule failure. +pub const SCHEDULE_RETRY_BASE: Duration = Duration::from_secs(1); +/// Longest retry delay after repeated retryable schedule failures. +pub const SCHEDULE_RETRY_MAX: Duration = Duration::from_secs(300); + +/// Something a host may or may not provide to the commands it runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Capability { + /// The host dispatches due schedules (`ctx.schedule`) back into their + /// aggregate. + Scheduling, + /// The host stamps its clock into each command (`ctx.now()`). + Clock, +} + +impl std::fmt::Display for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Scheduling => "scheduling", + Self::Clock => "clock", + }) + } +} + +/// What a backend provides ([`Capability`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Capabilities { + /// Due schedules are dispatched. + pub scheduling: bool, + /// A clock is stamped into commands. + pub clock: bool, +} + +impl Capabilities { + /// Nothing. + pub const NONE: Self = Self { + scheduling: false, + clock: false, + }; + /// Everything (aggregate cells). + pub const ALL: Self = Self { + scheduling: true, + clock: true, + }; + + /// Whether `capability` is provided. + pub fn has(&self, capability: Capability) -> bool { + match capability { + Capability::Scheduling => self.scheduling, + Capability::Clock => self.clock, + } + } + + /// The members of `required` this backend lacks. + pub fn missing(&self, required: &[Capability]) -> Vec { + let mut missing: Vec = required + .iter() + .copied() + .filter(|capability| !self.has(*capability)) + .collect(); + missing.sort(); + missing.dedup(); + missing + } +} + +/// Why a schedule could not be armed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ScheduleError { + InvalidName, + EmptyCommand, + InvalidInput, + DueOutOfRange, + TooManyWrites, + TooManyArmed, + /// The batch committed no aggregate, or several: a schedule belongs to + /// exactly one. + NoSingleOwner, + /// The route did not declare `.requires(Capability::Scheduling)`. + NotDeclared, +} + +impl std::fmt::Display for ScheduleError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidName => write!( + f, + "schedule name must be 1..={MAX_SCHEDULE_NAME_BYTES} bytes of [A-Za-z0-9._:-]" + ), + Self::EmptyCommand => f.write_str("schedule command must not be empty"), + Self::InvalidInput => write!( + f, + "schedule input must be a JSON object of at most {MAX_SCHEDULE_INPUT_BYTES} bytes" + ), + Self::DueOutOfRange => f.write_str("schedule due time is out of range"), + Self::TooManyWrites => write!( + f, + "a command may stage at most {MAX_SCHEDULE_WRITES_PER_COMMAND} schedule writes" + ), + Self::TooManyArmed => write!( + f, + "an aggregate may hold at most {MAX_ARMED_SCHEDULES} armed schedules" + ), + Self::NotDeclared => f.write_str( + "ctx.schedule needs a route declared with .requires(Capability::Scheduling)", + ), + Self::NoSingleOwner => f.write_str( + "a command that schedules must commit exactly one aggregate, which owns the schedule", + ), + } + } +} + +impl std::error::Error for ScheduleError {} + +impl From for RepositoryError { + fn from(error: ScheduleError) -> Self { + RepositoryError::Model(error.to_string()) + } +} + +/// When a schedule is due. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ScheduleDue { + /// Absolute epoch milliseconds. + At(i64), + /// Milliseconds after the commit, resolved by the backend's clock. + After(u64), +} + +/// Work a command schedules for its aggregate: when due, dispatch `command` +/// with `input` into it. +#[derive(Clone, Debug, PartialEq)] +pub struct Schedule { + name: String, + command: String, + input: Value, + due: ScheduleDue, +} + +impl Schedule { + /// Schedule `name` to dispatch `command` with `input` at `due`. + /// + /// Periodic handlers re-schedule from the fired due time + /// (`ctx.scheduled_due()`), so ticks do not drift with handler latency. + pub fn at( + name: impl Into, + due: SystemTime, + command: impl Into, + input: &impl Serialize, + ) -> Result { + Self::build( + name, + ScheduleDue::At(system_time_to_ms(due)?), + command, + input, + ) + } + + /// Schedule `name` to fire `delay` after the command commits, measured by + /// the backend's clock (the Durable Object clock on celld, the injected + /// [`Clock`](crate::Clock) in memory, the database clock on SQL). + pub fn after( + name: impl Into, + delay: Duration, + command: impl Into, + input: &impl Serialize, + ) -> Result { + let delay = u64::try_from(delay.as_millis()).map_err(|_| ScheduleError::DueOutOfRange)?; + Self::build(name, ScheduleDue::After(delay), command, input) + } + + fn build( + name: impl Into, + due: ScheduleDue, + command: impl Into, + input: &impl Serialize, + ) -> Result { + let input = serde_json::to_value(input).map_err(|_| ScheduleError::InvalidInput)?; + let schedule = Self { + name: name.into(), + command: command.into(), + input, + due, + }; + schedule.validate()?; + Ok(schedule) + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn command(&self) -> &str { + &self.command + } + + pub fn input(&self) -> &Value { + &self.input + } + + /// Absolute due time, or `None` for a relative schedule not yet + /// committed. + pub fn due(&self) -> Option { + match self.due { + ScheduleDue::At(ms) => Some(ms_to_system_time(ms)), + ScheduleDue::After(_) => None, + } + } + + /// The relative delay, for backends that resolve it in SQL. + pub(crate) fn relative_delay_ms(&self) -> Option { + match self.due { + ScheduleDue::After(delay) => Some(delay), + ScheduleDue::At(_) => None, + } + } + + /// Resolve a relative due time against the committing backend's clock. + pub(crate) fn resolve(&mut self, now: SystemTime) -> Result<(), ScheduleError> { + if let ScheduleDue::After(delay) = self.due { + let now = system_time_to_ms(now)?; + let due = now + .checked_add(i64::try_from(delay).map_err(|_| ScheduleError::DueOutOfRange)?) + .ok_or(ScheduleError::DueOutOfRange)?; + if due > MAX_JS_SAFE_MS { + return Err(ScheduleError::DueOutOfRange); + } + self.due = ScheduleDue::At(due); + } + Ok(()) + } + + /// Absolute due time; storage refuses an unresolved relative schedule. + pub(crate) fn due_at_ms(&self) -> Result { + match self.due { + ScheduleDue::At(ms) => Ok(ms), + ScheduleDue::After(_) => Err(RepositoryError::Model( + "relative schedule reached storage without being resolved".into(), + )), + } + } + + pub(crate) fn input_json(&self) -> Result { + serde_json::to_string(&self.input) + .map_err(|error| RepositoryError::Model(format!("schedule input: {error}"))) + } + + pub(crate) fn validate(&self) -> Result<(), ScheduleError> { + validate_name(&self.name)?; + if self.command.trim().is_empty() { + return Err(ScheduleError::EmptyCommand); + } + let size = serde_json::to_vec(&self.input) + .map_err(|_| ScheduleError::InvalidInput)? + .len(); + if !self.input.is_object() || size > MAX_SCHEDULE_INPUT_BYTES { + return Err(ScheduleError::InvalidInput); + } + match self.due { + ScheduleDue::At(ms) if !(0..=MAX_JS_SAFE_MS).contains(&ms) => { + Err(ScheduleError::DueOutOfRange) + } + ScheduleDue::After(delay) if delay > MAX_JS_SAFE_MS as u64 => { + Err(ScheduleError::DueOutOfRange) + } + _ => Ok(()), + } + } +} + +/// Resolve every relative schedule of a batch against a backend clock. +pub(crate) fn resolve_schedule_writes( + writes: &mut [ScheduleWrite], + now: SystemTime, +) -> Result<(), RepositoryError> { + for write in writes { + if let ScheduleWrite::Arm(schedule) = write { + schedule.resolve(now)?; + } + } + Ok(()) +} + +/// Durable Object SQL binds integers through JS numbers. +const MAX_JS_SAFE_MS: i64 = 9_007_199_254_740_991; + +/// One schedule write staged by a command and committed with its events. +#[derive(Clone, Debug, PartialEq)] +pub enum ScheduleWrite { + /// Schedule (or re-schedule) the name. + Arm(Schedule), + /// Disarm a name. Cancelling a name that is not armed is a no-op. + Cancel(String), +} + +impl ScheduleWrite { + pub fn name(&self) -> &str { + match self { + Self::Arm(schedule) => schedule.name(), + Self::Cancel(name) => name, + } + } +} + +/// Validate a batch's schedule writes before any storage participant runs. +pub(crate) fn validate_schedule_writes(writes: &[ScheduleWrite]) -> Result<(), RepositoryError> { + if writes.len() > MAX_SCHEDULE_WRITES_PER_COMMAND { + return Err(ScheduleError::TooManyWrites.into()); + } + for write in writes { + match write { + ScheduleWrite::Arm(schedule) => schedule.validate()?, + ScheduleWrite::Cancel(name) => validate_name(name)?, + } + } + Ok(()) +} + +/// The one stream of a batch that owns its schedule writes (None when the +/// batch has none). +pub(crate) fn schedule_owner( + batch: &crate::repository::CommitBatch<'_>, +) -> Result, RepositoryError> { + if batch.schedules.is_empty() { + return Ok(None); + } + validate_schedule_writes(&batch.schedules)?; + match batch.streams.as_slice() { + [stream] => Ok(Some(stream.identity.clone())), + _ => Err(ScheduleError::NoSingleOwner.into()), + } +} + +/// An armed schedule as stored. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ArmedSchedule { + pub name: String, + pub command: String, + pub input: Value, + pub due_at_ms: i64, + pub generation: u64, + pub attempts: u32, +} + +impl ArmedSchedule { + pub fn due(&self) -> SystemTime { + ms_to_system_time(self.due_at_ms) + } + + /// Stable command-ledger identity of this arm. + pub fn command_identity( + &self, + ) -> Result< + crate::microsvc::cell_host::CellCommandIdentity, + crate::microsvc::cell_host::CellDispatchError, + > { + crate::microsvc::cell_host::CellCommandIdentity::new( + SCHEDULE_SERVICE_ID, + SCHEDULE_SERVICE_ID, + schedule_command_id(&self.name, self.generation), + ) + } +} + +/// How a fired schedule settled. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum ScheduleSettlement { + /// Ran (or replayed) or failed terminally: disarm this generation. + Done, + /// Retry this generation at `due_at_ms`. + Retry { due_at_ms: i64 }, +} + +/// What one wake did with the due schedules. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ScheduleFireReport { + /// Schedules whose command committed (or replayed a committed receipt). + pub fired: Vec, + /// Schedules rescheduled after a retryable failure. + pub retried: Vec, + /// Schedules disarmed after a terminal failure. + pub failed: Vec, +} + +impl ScheduleFireReport { + pub fn is_empty(&self) -> bool { + self.fired.is_empty() && self.retried.is_empty() && self.failed.is_empty() + } +} + +/// A fired schedule. +#[derive(Clone, Debug, PartialEq)] +pub struct FiredSchedule { + pub name: String, + pub generation: u64, + pub command: String, + /// The command had already committed; its receipt was replayed. + pub replayed: bool, + pub payload: Value, +} + +/// A schedule whose command failed. +#[derive(Clone, Debug, PartialEq)] +pub struct FailedSchedule { + pub name: String, + pub generation: u64, + pub command: String, + pub code: &'static str, + pub message: String, +} + +/// The trusted system session a fired schedule dispatches with. +pub fn schedule_session(schedule: &ArmedSchedule) -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, SCHEDULE_USER_ID); + session.set(ROLE_KEY, SCHEDULE_ROLE); + session.set(SCHEDULE_NAME_CLAIM, schedule.name.clone()); + session.set(SCHEDULE_DUE_CLAIM, schedule.due_at_ms.to_string()); + session.set(SCHEDULE_GENERATION_CLAIM, schedule.generation.to_string()); + session +} + +/// The host clock's time at dispatch, when the dispatcher stamped it. +pub fn host_now(session: &Session) -> Option { + session + .get(HOST_NOW_CLAIM)? + .parse::() + .ok() + .filter(|ms| *ms >= 0) + .map(ms_to_system_time) +} + +/// Overwrite (or clear) the host-clock claim of a session about to run a +/// command: callers never choose it. +pub(crate) fn stamp_host_now(session: &mut Session, now: Option) { + session.remove(HOST_NOW_CLAIM); + if let Some(now) = now { + let ms = system_time_to_ms(now).unwrap_or(0); + session.set(HOST_NOW_CLAIM, ms.to_string()); + } +} + +/// Whether `session` is the trusted session of a fired schedule. +pub fn is_schedule_session(session: &Session) -> bool { + session.user_id() == Some(SCHEDULE_USER_ID) + && session.has_role(SCHEDULE_ROLE) + && session.get(SCHEDULE_NAME_CLAIM).is_some() +} + +/// Due time of the schedule that fired this command, when it was one. +pub fn scheduled_due(session: &Session) -> Option { + if !is_schedule_session(session) { + return None; + } + session + .get(SCHEDULE_DUE_CLAIM)? + .parse::() + .ok() + .filter(|ms| *ms >= 0) + .map(ms_to_system_time) +} + +/// Route guard admitting only fired schedules. +pub fn schedule_guard(ctx: &crate::microsvc::CausalCommandContext<'_, A>) -> bool +where + A: crate::Aggregate + Send + Sync + 'static, +{ + is_schedule_session(ctx.session()) +} + +/// Deterministic UUIDv7-shaped command id of one schedule arm. +pub(crate) fn schedule_command_id(name: &str, generation: u64) -> String { + let mut hasher = Sha256::new(); + hasher.update(SCHEDULE_SERVICE_ID.as_bytes()); + hasher.update([0u8]); + hasher.update(name.as_bytes()); + hasher.update([0u8]); + hasher.update(generation.to_be_bytes()); + let digest = hasher.finalize(); + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&digest[..16]); + bytes[6] = (bytes[6] & 0x0f) | 0x70; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + uuid::Uuid::from_bytes(bytes).hyphenated().to_string() +} + +/// Exponential retry delay for the `attempts`-th failure (0-based). +pub(crate) fn retry_delay(attempts: u32) -> Duration { + let factor = 1u32.checked_shl(attempts.min(16)).unwrap_or(u32::MAX); + SCHEDULE_RETRY_BASE + .checked_mul(factor) + .unwrap_or(SCHEDULE_RETRY_MAX) + .min(SCHEDULE_RETRY_MAX) +} + +pub(crate) fn system_time_to_ms(time: SystemTime) -> Result { + let millis = time + .duration_since(SystemTime::UNIX_EPOCH) + .map_err(|_| ScheduleError::DueOutOfRange)? + .as_millis(); + if millis > MAX_JS_SAFE_MS as u128 { + return Err(ScheduleError::DueOutOfRange); + } + Ok(millis as i64) +} + +pub(crate) fn ms_to_system_time(ms: i64) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_millis(ms.max(0) as u64) +} + +fn validate_name(name: &str) -> Result<(), ScheduleError> { + if name.is_empty() + || name.len() > MAX_SCHEDULE_NAME_BYTES + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) + { + return Err(ScheduleError::InvalidName); + } + Ok(()) +} + +/// One owner's schedule table in memory (the in-memory repository and the +/// in-process reference cell host), with the same semantics as the SQL +/// tables. +#[derive(Clone, Debug, Default)] +pub(crate) struct MemorySchedules { + rows: std::collections::BTreeMap, +} + +#[derive(Clone, Debug)] +struct MemoryScheduleRow { + schedule: ArmedSchedule, + armed: bool, +} + +impl MemorySchedules { + /// Apply one command's writes all-or-nothing, as the SQL transaction does. + pub(crate) fn plan(&self, writes: &[ScheduleWrite]) -> Result { + let mut next = self.clone(); + for write in writes { + match write { + ScheduleWrite::Arm(schedule) => { + let generation = next + .rows + .get(schedule.name()) + .map_or(1, |row| row.schedule.generation + 1); + next.rows.insert( + schedule.name().to_string(), + MemoryScheduleRow { + schedule: ArmedSchedule { + name: schedule.name().to_string(), + command: schedule.command().to_string(), + input: schedule.input().clone(), + due_at_ms: schedule.due_at_ms()?, + generation, + attempts: 0, + }, + armed: true, + }, + ); + } + ScheduleWrite::Cancel(name) => { + if let Some(row) = next.rows.get_mut(name) { + row.armed = false; + } + } + } + } + if next.rows.values().filter(|row| row.armed).count() > MAX_ARMED_SCHEDULES { + return Err(ScheduleError::TooManyArmed.into()); + } + Ok(next) + } + + pub(crate) fn armed(&self) -> Vec { + let mut armed = self + .rows + .values() + .filter(|row| row.armed) + .map(|row| row.schedule.clone()) + .collect::>(); + armed.sort_by(|a, b| (a.due_at_ms, &a.name).cmp(&(b.due_at_ms, &b.name))); + armed + } + + pub(crate) fn due(&self, now_ms: i64, limit: usize) -> Vec { + self.armed() + .into_iter() + .filter(|schedule| schedule.due_at_ms <= now_ms) + .take(limit) + .collect() + } + + pub(crate) fn next_due_ms(&self) -> Option { + self.rows + .values() + .filter(|row| row.armed) + .map(|row| row.schedule.due_at_ms) + .min() + } + + pub(crate) fn settle(&mut self, name: &str, generation: u64, settlement: &ScheduleSettlement) { + let Some(row) = self.rows.get_mut(name) else { + return; + }; + if !row.armed || row.schedule.generation != generation { + return; + } + match settlement { + ScheduleSettlement::Done => { + row.armed = false; + row.schedule.attempts = 0; + } + ScheduleSettlement::Retry { due_at_ms } => { + row.schedule.attempts = row.schedule.attempts.saturating_add(1); + row.schedule.due_at_ms = *due_at_ms; + } + } + } + + /// Every row (armed or not), for reference-host restart exports. + pub(crate) fn export(&self) -> Vec { + self.rows + .values() + .map(|row| DurableSchedule { + schedule: row.schedule.clone(), + armed: row.armed, + }) + .collect() + } + + pub(crate) fn restore(rows: Vec) -> Self { + Self { + rows: rows + .into_iter() + .map(|row| { + ( + row.schedule.name.clone(), + MemoryScheduleRow { + schedule: row.schedule, + armed: row.armed, + }, + ) + }) + .collect(), + } + } +} + +/// One schedule row in a reference-host restart export. Disarmed rows are +/// kept so generations never repeat after a restore. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DurableSchedule { + pub schedule: ArmedSchedule, + pub armed: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn schedule(name: &str, due_ms: u64) -> Schedule { + Schedule::at( + name, + SystemTime::UNIX_EPOCH + Duration::from_millis(due_ms), + "test.tick", + &serde_json::json!({ "n": 1 }), + ) + .unwrap() + } + + #[test] + fn validates_names_inputs_and_write_counts() { + let due = SystemTime::UNIX_EPOCH + Duration::from_secs(1); + assert_eq!( + Schedule::at("", due, "c", &serde_json::json!({})).unwrap_err(), + ScheduleError::InvalidName + ); + assert_eq!( + Schedule::at("a b", due, "c", &serde_json::json!({})).unwrap_err(), + ScheduleError::InvalidName + ); + assert_eq!( + Schedule::at("ok", due, " ", &serde_json::json!({})).unwrap_err(), + ScheduleError::EmptyCommand + ); + assert_eq!( + Schedule::at("ok", due, "c", &1).unwrap_err(), + ScheduleError::InvalidInput + ); + let big = serde_json::json!({ "x": "y".repeat(MAX_SCHEDULE_INPUT_BYTES) }); + assert_eq!( + Schedule::at("ok", due, "c", &big).unwrap_err(), + ScheduleError::InvalidInput + ); + let writes = (0..=MAX_SCHEDULE_WRITES_PER_COMMAND) + .map(|index| ScheduleWrite::Cancel(format!("a{index}"))) + .collect::>(); + assert!(validate_schedule_writes(&writes).is_err()); + assert!(validate_schedule_writes(&writes[1..]).is_ok()); + } + + #[test] + fn capabilities_report_what_is_missing() { + let sql = Capabilities { + scheduling: false, + clock: true, + }; + assert_eq!( + sql.missing(&[ + Capability::Clock, + Capability::Scheduling, + Capability::Scheduling + ]), + vec![Capability::Scheduling] + ); + assert!(Capabilities::ALL + .missing(&[Capability::Scheduling, Capability::Clock]) + .is_empty()); + assert_eq!(Capability::Scheduling.to_string(), "scheduling"); + } + + #[test] + fn command_ids_are_uuidv7_and_unique_per_generation() { + let first = schedule_command_id("tick", 1); + assert_eq!(first, schedule_command_id("tick", 1)); + assert_ne!(first, schedule_command_id("tick", 2)); + assert_ne!(first, schedule_command_id("tock", 1)); + let parsed = uuid::Uuid::parse_str(&first).unwrap(); + assert_eq!(parsed.get_version_num(), 7); + let armed = ArmedSchedule { + name: "tick".into(), + command: "c".into(), + input: serde_json::json!({}), + due_at_ms: 0, + generation: 1, + attempts: 0, + }; + assert_eq!(armed.command_identity().unwrap().command_id(), first); + } + + #[test] + fn retry_delay_doubles_and_caps() { + assert_eq!(retry_delay(0), Duration::from_secs(1)); + assert_eq!(retry_delay(3), Duration::from_secs(8)); + assert_eq!(retry_delay(40), SCHEDULE_RETRY_MAX); + } + + #[test] + fn schedule_session_is_recognized_and_carries_due() { + let armed = ArmedSchedule { + name: "tick".into(), + command: "c".into(), + input: serde_json::json!({}), + due_at_ms: 1_234, + generation: 7, + attempts: 0, + }; + let session = schedule_session(&armed); + assert!(is_schedule_session(&session)); + assert_eq!( + scheduled_due(&session), + Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_234)) + ); + let mut forged = Session::new(); + forged.set(USER_ID_KEY, "player-1"); + forged.set(ROLE_KEY, SCHEDULE_ROLE); + forged.set(SCHEDULE_NAME_CLAIM, "tick"); + assert!(!is_schedule_session(&forged)); + assert_eq!(scheduled_due(&forged), None); + } + + #[test] + fn host_now_is_overwritten_never_taken_from_the_caller() { + let mut session = Session::new(); + session.set(HOST_NOW_CLAIM, "1"); + stamp_host_now(&mut session, None); + assert_eq!(host_now(&session), None, "an undeclared clock is cleared"); + stamp_host_now( + &mut session, + Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_234)), + ); + assert_eq!( + host_now(&session), + Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_234)) + ); + } + + #[test] + fn memory_table_reschedules_with_new_generation_and_settles_only_its_own() { + let table = MemorySchedules::default(); + let table = table + .plan(&[ScheduleWrite::Arm(schedule("tick", 10))]) + .unwrap(); + assert_eq!(table.armed()[0].generation, 1); + let mut table = table + .plan(&[ScheduleWrite::Arm(schedule("tick", 20))]) + .unwrap(); + assert_eq!(table.armed()[0].generation, 2); + assert_eq!(table.next_due_ms(), Some(20)); + assert!(table.due(19, 10).is_empty()); + table.settle("tick", 1, &ScheduleSettlement::Done); + assert_eq!(table.armed().len(), 1, "stale generation must not disarm"); + table.settle("tick", 2, &ScheduleSettlement::Retry { due_at_ms: 50 }); + assert_eq!(table.armed()[0].attempts, 1); + assert_eq!(table.next_due_ms(), Some(50)); + table.settle("tick", 2, &ScheduleSettlement::Done); + assert!(table.armed().is_empty()); + let table = table + .plan(&[ScheduleWrite::Arm(schedule("tick", 60))]) + .unwrap(); + assert_eq!( + table.armed()[0].generation, + 3, + "generations never repeat after a disarm" + ); + let restored = MemorySchedules::restore(table.export()); + assert_eq!(restored.armed(), table.armed()); + let cancelled = restored + .plan(&[ScheduleWrite::Cancel("tick".into())]) + .unwrap(); + assert!(cancelled.armed().is_empty()); + } + + #[test] + fn memory_table_caps_armed_schedules() { + let mut table = MemorySchedules::default(); + for chunk in 0..(MAX_ARMED_SCHEDULES / 16) { + let writes = (0..16) + .map(|index| ScheduleWrite::Arm(schedule(&format!("a{chunk}-{index}"), 1))) + .collect::>(); + table = table.plan(&writes).unwrap(); + } + let error = table + .plan(&[ScheduleWrite::Arm(schedule("overflow", 1))]) + .unwrap_err(); + assert!(error.to_string().contains("armed schedules")); + } +} diff --git a/src/sqlx_repo/repo/backend.rs b/src/sqlx_repo/repo/backend.rs index 2ac2c2e8d..948e3e83a 100644 --- a/src/sqlx_repo/repo/backend.rs +++ b/src/sqlx_repo/repo/backend.rs @@ -47,7 +47,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); assert_eq!( descriptions, vec![ @@ -61,7 +61,8 @@ mod tests { "external command binding", "projection delivery aliases", "coordination leases", - "cell alarms" + "cell alarms", + "aggregate schedules" ] ); assert_eq!( @@ -111,6 +112,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/sqlite/0011_cell_alarms.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/sqlite/0012_aggregate_schedules.sql" + )), ] ); } @@ -130,7 +135,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); assert_eq!( descriptions, vec![ @@ -144,7 +149,8 @@ mod tests { "external command binding", "projection delivery aliases", "coordination leases", - "cell alarms" + "cell alarms", + "aggregate schedules" ] ); assert_eq!( @@ -194,6 +200,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/postgres/0011_cell_alarms.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/postgres/0012_aggregate_schedules.sql" + )), ] ); } diff --git a/src/sqlx_repo/repo/commit.rs b/src/sqlx_repo/repo/commit.rs index 84dc20a62..45be32c17 100644 --- a/src/sqlx_repo/repo/commit.rs +++ b/src/sqlx_repo/repo/commit.rs @@ -43,7 +43,8 @@ where for<'q> Option<&'q str>: Encode<'q, DB> + Type, for<'r> &'r str: sqlx::ColumnIndex, { - crate::microsvc::cell_host::alarm::reject_alarm_writes(&batch.alarms)?; + let mut batch = batch; + let schedule_owner = crate::schedule::schedule_owner(&batch)?; let prepared = validate_commit_batch(&batch)?; for plan in &batch.read_model_plans { validate_sql_write_plan(plan).map_err(RepositoryError::from)?; @@ -101,6 +102,27 @@ where insert_events_in_tx(&repository.pool, &mut tx, &prepared).await?; insert_outbox_messages_in_tx(&mut tx, &batch.outbox_messages).await?; + if let Some(owner) = &schedule_owner { + // Relative schedules resolve against the database clock, inside the + // transaction that stores them. + if batch + .schedules + .iter() + .any(|write| matches!(write, crate::schedule::ScheduleWrite::Arm(schedule) if schedule.relative_delay_ms().is_some())) + { + let now = crate::repository::sql::ledger::now( + &mut executor::ConnectionExecutor::(&mut *tx), + ) + .await?; + crate::schedule::resolve_schedule_writes(&mut batch.schedules, now)?; + } + crate::repository::sql::schedules::apply( + &mut executor::ConnectionExecutor::(&mut *tx), + owner, + &batch.schedules, + ) + .await?; + } let mut changed_tables = std::collections::BTreeSet::new(); for plan in batch.read_model_plans { @@ -593,3 +615,71 @@ where Ok(versions) } + +/// SQL stores schedules but does not dispatch them yet (the schedule worker +/// is a later addition); its clock is the database clock. +impl crate::microsvc::BackendHost for SqlxRepository +where + DB: SqlxRepoBackend, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type + sqlx::Decode<'q, DB>, + for<'q> String: Type + sqlx::Decode<'q, DB>, + for<'q> Vec: Type + sqlx::Decode<'q, DB>, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, + for<'r> &'r str: sqlx::ColumnIndex, +{ + fn capabilities(&self) -> crate::schedule::Capabilities { + crate::schedule::Capabilities { + scheduling: false, + clock: true, + } + } + + fn host_now(&self) -> impl Future> + Send + '_ { + async move { + let mut connection = + self.pool.acquire().await.map_err(|err| { + repository_storage_error::("acquire clock connection", err) + })?; + crate::repository::sql::ledger::now(&mut executor::ConnectionExecutor::( + &mut *connection, + )) + .await + .map_err(|error| match error { + CommandLedgerError::Storage(error) => error, + other => RepositoryError::Model(other.to_string()), + }) + } + } +} + +impl SqlxRepository +where + DB: SqlxRepoBackend, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type + sqlx::Decode<'q, DB>, + for<'q> String: Type + sqlx::Decode<'q, DB>, + for<'q> Vec: Type + sqlx::Decode<'q, DB>, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, + for<'r> &'r str: sqlx::ColumnIndex, +{ + /// Armed schedules of the aggregate stream `owner`, earliest first. + pub async fn armed_schedules( + &self, + owner: &crate::StreamIdentity, + ) -> Result, RepositoryError> { + let mut connection = + self.pool.acquire().await.map_err(|err| { + repository_storage_error::("acquire schedule connection", err) + })?; + crate::repository::sql::schedules::armed( + &mut executor::ConnectionExecutor::(&mut *connection), + owner, + ) + .await + } +} diff --git a/tests/aggregate_schedules.rs b/tests/aggregate_schedules.rs new file mode 100644 index 000000000..3b2e592cb --- /dev/null +++ b/tests/aggregate_schedules.rs @@ -0,0 +1,83 @@ +//! Schedule rows on the shared SQL repository: stored in the command's +//! transaction, owned by the batch's one stream, relative due times resolved +//! by the database clock, re-arming bumps the generation, cancelling +//! disarms. (Dispatching them is a later SQL schedule worker; SQL does not +//! declare `Capability::Scheduling` yet.) + +#![cfg(feature = "sqlite")] + +use std::time::{Duration, SystemTime}; + +use distributed::repository::{CommitBatch, StreamWrite}; +use distributed::schedule::{Schedule, ScheduleWrite}; +use distributed::{Entity, SqliteRepository, StreamIdentity, TransactionalCommit}; + +fn event(entity: &mut Entity, id: &str) { + if entity.id().is_empty() { + entity.set_id(id); + } + entity.digest_empty("ticker.ticked").expect("record event"); +} + +#[tokio::test] +async fn sqlite_stores_schedules_with_their_owner_in_the_commit() { + let repo = SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .unwrap(); + let owner = StreamIdentity::new("Ticker", "t1").unwrap(); + let mut entity = Entity::new(); + event(&mut entity, "t1"); + let mut batch = CommitBatch::new(vec![StreamWrite::new(owner.clone(), &mut entity)]); + batch.schedules.push(ScheduleWrite::Arm( + Schedule::after( + "tick", + Duration::from_secs(30), + "ticker.tick", + &serde_json::json!({ "id": "t1" }), + ) + .unwrap(), + )); + let before = SystemTime::now(); + repo.commit_batch(batch).await.unwrap(); + let armed = repo.armed_schedules(&owner).await.unwrap(); + assert_eq!(armed.len(), 1); + assert_eq!(armed[0].command, "ticker.tick"); + assert_eq!(armed[0].generation, 1); + let due = armed[0].due(); + assert!( + due >= before + Duration::from_secs(29) + && due <= SystemTime::now() + Duration::from_secs(31), + "relative due resolved by the database clock: {due:?}" + ); + + // Re-arm (generation 2) and cancel in later commits of the same owner. + event(&mut entity, "t1"); + let mut batch = CommitBatch::new(vec![StreamWrite::new(owner.clone(), &mut entity)]); + batch.schedules.push(ScheduleWrite::Arm( + Schedule::at( + "tick", + SystemTime::UNIX_EPOCH + Duration::from_secs(2_000_000_000), + "ticker.tick", + &serde_json::json!({ "id": "t1" }), + ) + .unwrap(), + )); + repo.commit_batch(batch).await.unwrap(); + assert_eq!(repo.armed_schedules(&owner).await.unwrap()[0].generation, 2); + event(&mut entity, "t1"); + let mut batch = CommitBatch::new(vec![StreamWrite::new(owner.clone(), &mut entity)]); + batch.schedules.push(ScheduleWrite::Cancel("tick".into())); + repo.commit_batch(batch).await.unwrap(); + assert!(repo.armed_schedules(&owner).await.unwrap().is_empty()); + + // Another owner's table is separate; a schedule needs exactly one owner. + let other = StreamIdentity::new("Ticker", "t2").unwrap(); + assert!(repo.armed_schedules(&other).await.unwrap().is_empty()); + let mut orphan = CommitBatch::empty(); + orphan.schedules.push(ScheduleWrite::Cancel("tick".into())); + let error = repo.commit_batch(orphan).await.unwrap_err(); + assert!( + error.to_string().contains("exactly one aggregate"), + "{error}" + ); +} diff --git a/tests/celld/alarm-worker/src/lib.rs b/tests/celld/alarm-worker/src/lib.rs index b9b4efce3..576b0b923 100644 --- a/tests/celld/alarm-worker/src/lib.rs +++ b/tests/celld/alarm-worker/src/lib.rs @@ -13,15 +13,16 @@ use std::time::Duration; use distributed::cell_host::{ - cell_alarm_guard, AggregateCell, CellAlarm, CellCommandIdentity, CellDispatchError, - CellWaitPathRequest, CelldOutbox, InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, - CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, + AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, + InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, }; use distributed::command::{typed_command, PreparedCommand, Succeeded}; use distributed::microsvc::{ CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, Session, ROLE_KEY, USER_ID_KEY, }; +use distributed::schedule::{schedule_guard, Capability, Schedule}; use distributed::{sourced, Entity}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -97,6 +98,7 @@ where .typed_command(typed_command::>( "ticker.start", )) + .requires(Capability::Scheduling) .guarded(is_user, handle_start) } } @@ -113,8 +115,8 @@ async fn handle_start( ticker .start(input.id.clone(), input.limit, input.period_ms) .map_err(rejected)?; - ctx.arm_alarm( - CellAlarm::after( + ctx.schedule( + Schedule::after( "tick", Duration::from_millis(input.period_ms), TICK, @@ -141,7 +143,8 @@ where fn install(self, routes: Routes) -> Routes { routes .typed_command(typed_command::>(TICK)) - .guarded(cell_alarm_guard::, handle_tick) + .requires(Capability::Scheduling) + .guarded(schedule_guard::, handle_tick) } } @@ -157,10 +160,10 @@ async fn handle_tick( ticker.tick().map_err(rejected)?; if ticker.ticks < ticker.limit { let due = ctx - .alarm_due() + .scheduled_due() .ok_or_else(|| rejected("tick outside an alarm"))? + Duration::from_millis(ticker.period_ms); - ctx.arm_alarm(CellAlarm::at("tick", due, TICK, &input).map_err(rejected)?)?; + ctx.schedule(Schedule::at("tick", due, TICK, &input).map_err(rejected)?)?; } let ticks = ticker.ticks; repo.publish_events() @@ -256,7 +259,7 @@ impl DurableObject for TickerCell { .alarm(&self.env) .await .map_err(|error| Error::RustError(error.to_string()))?; - for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + for failed in wake.schedules.failed.iter().chain(&wake.schedules.retried) { worker::console_error!( "cell alarm {} ({}) failed: {} {}", failed.name, @@ -286,7 +289,7 @@ impl TickerCell { .map_err(|error| Error::RustError(error.to_string()))?; let armed = self .cell - .armed_alarms() + .armed_schedules() .map_err(|error| Error::RustError(error.to_string()))?; let backlog = self .cell @@ -433,7 +436,7 @@ fn request_identity( } /// Only the verified user and roles cross the HTTP boundary; the alarm claims -/// that `cell_alarm_guard` checks are set by the cell host alone. +/// that `schedule_guard` checks are set by the cell host alone. fn request_session(req: &Request) -> Session { let mut session = Session::new(); for key in [USER_ID_KEY, ROLE_KEY] { diff --git a/tests/celld/fact-worker/src/lib.rs b/tests/celld/fact-worker/src/lib.rs index fcb695261..c55a769e5 100644 --- a/tests/celld/fact-worker/src/lib.rs +++ b/tests/celld/fact-worker/src/lib.rs @@ -299,13 +299,7 @@ where worker::console_error!("post-commit drain deferred: {error}"); } match result { - Ok(dispatch) => json_status( - json!({ - "payload": dispatch.payload(), - "receipt": { "commandId": dispatch.command_id(), "replayed": dispatch.replayed() }, - }), - 200, - ), + Ok(dispatch) => json_status(dispatch.wait_path_wire(), 200), Err(error) => error_response(error), } } @@ -344,7 +338,7 @@ where .map(|event| event.event_name.clone()) .collect::>(); let armed = cell - .armed_alarms() + .armed_schedules() .map_err(|error| Error::RustError(error.to_string()))? .len(); json_status( @@ -457,7 +451,7 @@ impl DurableObject for ZoneLeaseCell { .alarm(&self.env) .await .map_err(|error| Error::RustError(error.to_string()))?; - for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + for failed in wake.schedules.failed.iter().chain(&wake.schedules.retried) { worker::console_error!("lease alarm failed: {} {}", failed.code, failed.message); } Response::ok("ok") diff --git a/tests/celld/main.rs b/tests/celld/main.rs index 368eae7b3..ea1b28bbb 100644 --- a/tests/celld/main.rs +++ b/tests/celld/main.rs @@ -122,8 +122,9 @@ fn alarm_worker_arms_cell_alarms_and_shares_the_durable_object_alarm() { assert_eq!(spec["migrations"][0]["new_sqlite_classes"][0], "TickerCell"); assert_eq!(spec["queues"]["producers"][0]["binding"], "OUTBOX"); let source = std::fs::read_to_string(dir.join("src/lib.rs")).expect("lib.rs"); - assert!(source.contains("ctx.arm_alarm(")); - assert!(source.contains("cell_alarm_guard::")); + assert!(source.contains("ctx.schedule(")); + assert!(source.contains("schedule_guard::")); + assert!(source.contains(".requires(Capability::Scheduling)")); assert!(source.contains(".alarm(&self.env)")); assert!(source.contains("with_celld_outbox(outbox)")); let harness = std::fs::read_to_string(concat!( diff --git a/tests/celld/worker/src/lib.rs b/tests/celld/worker/src/lib.rs index 23678b3b4..48c03704b 100644 --- a/tests/celld/worker/src/lib.rs +++ b/tests/celld/worker/src/lib.rs @@ -8,8 +8,8 @@ use chat_domain::{post, ChatMessage, ChatMessageState}; use distributed::cell_host::{ AggregateCell, CellCommandIdentity, CellDispatchError, CellDispatchResult, CellWaitPathRequest, - CelldOutbox, InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, - CELL_CAUSATION_ID_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, + CelldOutbox, InternalHttpSecret, CELL_CAUSATION_ID_HEADER, CELL_INTERNAL_SECRET_ENV, + CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, }; use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; use serde::de::DeserializeOwned; @@ -52,7 +52,7 @@ where .alarm(env) .await .map_err(|error| Error::RustError(error.to_string()))?; - for failed in wake.alarms.failed.iter().chain(&wake.alarms.retried) { + for failed in wake.schedules.failed.iter().chain(&wake.schedules.retried) { worker::console_error!( "cell alarm {} on {} failed: {} {}", failed.name, @@ -431,7 +431,11 @@ fn optional_internal_header( .map_err(|error| { CellDispatchError::Internal(format!("could not read internal cell header: {error}")) }) - .map(|value| value.map(|value| value.trim().to_string()).filter(|value| !value.is_empty())) + .map(|value| { + value + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + }) } fn wait_path_ok( diff --git a/tests/persistent_repository_conformance/read_models.rs b/tests/persistent_repository_conformance/read_models.rs index 1ebaa1085..4e9c6e713 100644 --- a/tests/persistent_repository_conformance/read_models.rs +++ b/tests/persistent_repository_conformance/read_models.rs @@ -179,7 +179,7 @@ where let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), streams: vec![StreamWrite::new(identity, stale.entity_mut())], outbox_messages: Vec::new(), read_model_plans: vec![read_models diff --git a/tests/persistent_repository_conformance/scenario.rs b/tests/persistent_repository_conformance/scenario.rs index a45414f19..76ecb2eee 100644 --- a/tests/persistent_repository_conformance/scenario.rs +++ b/tests/persistent_repository_conformance/scenario.rs @@ -164,7 +164,7 @@ where let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), streams: vec![ StreamWrite::new(stale_identity, stale.entity_mut()), StreamWrite::new(checkout_identity.clone(), checkout.entity_mut()), diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index d6f7fb385..e77d6efed 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -234,7 +234,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_and_preserve_checks() { .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 11); + assert_eq!(latest_version, 12); let invalid_service = sqlx::query( r#" @@ -337,7 +337,7 @@ async fn read_model_failure_mid_plan_rolls_back_events_and_outbox() { let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), streams: vec![StreamWrite::new(identity.clone(), projection.entity_mut())], outbox_messages: vec![outbox_message], read_model_plans: vec![read_models.into_write_plan().unwrap()], diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index f233321ce..e44a89fc4 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -111,7 +111,7 @@ async fn outbox_backlog_stats_order_sqlite_text_timestamps_numerically() { read_model_plans: Vec::new(), snapshots: Vec::new(), inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), }) .await .expect("outbox-only batch should commit"); @@ -202,7 +202,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_without_schema_drift() .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 11); + assert_eq!(latest_version, 12); let created_at_type: String = sqlx::query_scalar( "SELECT typeof(created_at) FROM command_ledger WHERE service_id = 'service'", @@ -318,7 +318,7 @@ async fn optimistic_conflict_rolls_back_other_stream_and_read_model_plan() { let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), streams: vec![ StreamWrite::new(stale_identity.clone(), stale.entity_mut()), StreamWrite::new(other_identity.clone(), other.entity_mut()), @@ -409,7 +409,7 @@ async fn read_model_failure_mid_plan_rolls_back_events_and_outbox() { let err = repo .commit_batch(CommitBatch { inbox_receipts: Vec::new(), - alarms: Vec::new(), + schedules: Vec::new(), streams: vec![StreamWrite::new(identity.clone(), projection.entity_mut())], outbox_messages: vec![outbox_message], read_model_plans: vec![read_models.into_write_plan().unwrap()], From edae35c75cb8f2581d972e473168d3ae4d32ee7b Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:30:15 -0500 Subject: [PATCH 22/40] fix(e2e)!: pin celld 0.6.2 and move the profile off :18080 The e2e-ui celld profile defaulted CELLD_HTTP_PORT to 18080, which Zitadel's edge from `make up` already binds, so `make up && make up-celld-nats` failed. celld now defaults to 18880 in the e2e-ui, e2e-celld and tests/celld Makefiles. One celld version, in /.celld-version (0.6.2), checked by scripts/celld.sh (check prints the upgrade command; install is what CI runs). run.sh, the e2e-ui profile and every workflow (including the previously unpinned live job) use it. The binary is CELLD_BIN everywhere; the CELLD alias in alarms.mjs and the Rust support is gone. Also from A8: drop the dead .make-ui/.make-runner pid handling and the unused DATABASE_URL default, fix the "UI :5180" comment, pass --context to every kubectl call in dual-worktree-suite.sh, and replace the zitadel-ingestor sample query that used undeclared reverse joins. BREAKING CHANGE: the celld binary env var is CELLD_BIN (CELLD is no longer read); the profile's default celld port is 18880. [[tasks/gr-p40-examples-e2e-coverage]] --- .celld-version | 1 + .github/workflows/integration-celld.yaml | 13 ++--- .github/workflows/integration-game-demos.yaml | 7 +-- distributed_game/tests/support/celld.rs | 4 +- docs/coordination-primitives.md | 2 +- examples/game/run.sh | 20 ++------ scripts/celld.sh | 49 +++++++++++++++++++ src/bus/celld_queue.rs | 2 +- tests/celld/Makefile | 9 ++-- tests/celld/README.md | 9 ++-- tests/celld/alarms.mjs | 4 +- tests/celld/storage-conformance.mjs | 2 +- tests/e2e-celld/Makefile | 14 ++---- tests/e2e-celld/README.md | 5 +- tests/e2e-ui/Makefile | 42 +++++++--------- tests/e2e-ui/README.md | 2 +- tests/e2e-ui/celld-nats-profile/README.md | 18 ++++--- .../celld-nats-profile/docker-compose.yml | 2 +- tests/e2e-ui/docs/zitadel-ingestor.md | 20 +++++--- tests/e2e-ui/scripts/dual-worktree-suite.sh | 8 +-- 20 files changed, 136 insertions(+), 97 deletions(-) create mode 100644 .celld-version create mode 100755 scripts/celld.sh diff --git a/.celld-version b/.celld-version new file mode 100644 index 000000000..b61604874 --- /dev/null +++ b/.celld-version @@ -0,0 +1 @@ +0.6.2 diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index d61c02ad4..b939e220f 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -52,8 +52,9 @@ jobs: sudo apt-get update && sudo apt-get install -y sqlite3 npm install -g esbuild cargo install worker-build --locked - # Pinned: the zone-state R2 path and the fact routing are proven on it. - curl -fsSL https://celld.dev/install.sh | CELLD_VERSION=v0.6.2 sh + # The pinned celld (/.celld-version) is the one the zone-state R2 + # path, fact routing and every profile below are proven on. + scripts/celld.sh install echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Build isolated fault-probe Worker working-directory: tests/celld/worker @@ -115,7 +116,7 @@ jobs: run: cargo test --workspace --verbose live: - name: celld 0.4 live (local store + Queue + NATS) + name: celld live (local store + Queue + NATS) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -154,9 +155,9 @@ jobs: - name: Install worker-build run: cargo install worker-build --locked || cargo install worker-build - - name: Install celld CLI + - name: Install celld CLI (pinned in /.celld-version) run: | - curl -fsSL https://celld.dev/install.sh | sh + scripts/celld.sh install echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Bring up Postgres + Zitadel and bootstrap OIDC @@ -183,7 +184,7 @@ jobs: working-directory: tests/e2e-ui run: npx playwright install chromium --with-deps - - name: Bring up celld 0.4 + Queue relay + NATS + - name: Bring up celld + Queue relay + NATS run: | command -v celld command -v worker-build diff --git a/.github/workflows/integration-game-demos.yaml b/.github/workflows/integration-game-demos.yaml index a94fbe118..f2ee5012a 100644 --- a/.github/workflows/integration-game-demos.yaml +++ b/.github/workflows/integration-game-demos.yaml @@ -8,7 +8,7 @@ name: Game demos smoke # examples/game/server` (GraphQL client, svelte-check, vite build), then # Playwright against the production build, a real Mob Pit server and the real # mobpit-service read model. Mob Pit's durable cells run on celld: the tests -# start their own `celld dev` (CELLD_BIN, pinned below; no Docker) serving +# start their own `celld dev` (CELLD_BIN, pinned in /.celld-version; no Docker) serving # examples/game/cells, built with worker-build. on: workflow_call: @@ -19,8 +19,6 @@ jobs: runs-on: ubuntu-latest env: CARGO_TERM_COLOR: always - # Pinned celld for Mob Pit's cells (run.sh requires 0.6.2+). - CELLD_VERSION: v0.6.2 # The shared ticket rate limiter's Lua script and the leaderboard store # (conformance suite, projector, GraphQL boards) run against a real Redis. services: @@ -56,9 +54,8 @@ jobs: shared-key: game-demos - name: Install celld (pinned) and worker-build run: | - curl -fsSL https://celld.dev/install.sh | CELLD_VERSION="$CELLD_VERSION" sh + scripts/celld.sh install echo "CELLD_BIN=$HOME/.local/bin/celld" >> "$GITHUB_ENV" - "$HOME/.local/bin/celld" --version cargo install worker-build --locked - name: Install SvelteKit demo dependencies and Chromium working-directory: examples/game/sveltekit diff --git a/distributed_game/tests/support/celld.rs b/distributed_game/tests/support/celld.rs index 4157bdd47..1efd2fffa 100644 --- a/distributed_game/tests/support/celld.rs +++ b/distributed_game/tests/support/celld.rs @@ -79,9 +79,7 @@ impl Celld { /// celld on PATH, or no esbuild for its bundler). pub fn start(&mut self) -> Result<(), String> { self.generation += 1; - let binary = std::env::var("CELLD_BIN") - .or_else(|_| std::env::var("CELLD")) - .unwrap_or_else(|_| "celld".into()); + let binary = std::env::var("CELLD_BIN").unwrap_or_else(|_| "celld".into()); let help = Command::new(&binary) .args(["dev", "--help"]) .output() diff --git a/docs/coordination-primitives.md b/docs/coordination-primitives.md index d5d168282..2a6fbf87b 100644 --- a/docs/coordination-primitives.md +++ b/docs/coordination-primitives.md @@ -275,7 +275,7 @@ cargo test --features workers-rs --lib cell_host ``` `tests/celld/alarms.mjs` owns its own `celld dev` process (no Docker; set -`CELLD` to choose the binary) and proves the alarm row commits atomically, +`CELLD_BIN` to choose the binary) and proves the alarm row commits atomically, periodic re-arm at the alarm cadence, that the watchdog and domain alarms do not starve each other, and exactly-once firing across a celld restart. diff --git a/examples/game/run.sh b/examples/game/run.sh index 7fdf79a15..b04ff4270 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -37,7 +37,8 @@ # multi-host demo's zone leases and hibernation snapshots) run on celld, # never natively; state persists in .data//celld (Mob Pit: # MOBPIT_DATA_DIR/celld). CELLD_BIN names the binary (default: `celld` on -# PATH; 0.6.2 or newer is required), CELLD_PORT its port (default +# PATH; the version pinned in .celld-version, checked by +# scripts/celld.sh), CELLD_PORT its port (default # GAME_DEMOS_PORT + 3; cluster demos GAME_DEMOS_PORT + 4), # CELLD_ESBUILD its bundler (default: the client's esbuild). The worker is # built with worker-build (`cargo install worker-build --locked`). @@ -55,7 +56,6 @@ DEMOS=(arena mobpit leaderboard gallery freeze grid ripple) # Single-process demos whose durable state lives in celld cells (the # multi-host ones always run on celld too: zone leases and hibernation). CELLD_DEMOS=(mobpit leaderboard freeze) -CELLD_MIN_VERSION="0.6.2" export MOBPIT_DATA_DIR="${MOBPIT_DATA_DIR:-$HERE/.data/mobpit}" # Multi-host demos: `game-demos --host-id A|B --peer …`. CLUSTER_DEMOS=(grid ripple) @@ -95,20 +95,10 @@ ensure_sveltekit() { fi } -# ensure_celld: resolve CELLD_BIN (>= CELLD_MIN_VERSION) and CELLD_ESBUILD. +# ensure_celld: resolve CELLD_BIN (the pinned celld, `.celld-version`) and +# CELLD_ESBUILD. scripts/celld.sh prints the upgrade command on a mismatch. ensure_celld() { - CELLD_BIN="${CELLD_BIN:-$(command -v celld || true)}" - if [[ -z "$CELLD_BIN" || ! -x "$CELLD_BIN" ]]; then - echo "celld not found: set CELLD_BIN to a celld $CELLD_MIN_VERSION+ binary" >&2 - echo " (install: curl -fsSL https://celld.dev/install.sh | CELLD_VERSION=v$CELLD_MIN_VERSION sh)" >&2 - exit 1 - fi - local version - version="$("$CELLD_BIN" --version 2>/dev/null | sed -n 's/^celld \([0-9][0-9.]*\).*/\1/p' | head -1)" - if [[ -z "$version" ]] || [[ "$(printf '%s\n%s\n' "$CELLD_MIN_VERSION" "$version" | sort -V | head -1)" != "$CELLD_MIN_VERSION" ]]; then - echo "$CELLD_BIN is celld ${version:-(unknown)}; Mob Pit needs celld $CELLD_MIN_VERSION or newer (set CELLD_BIN)" >&2 - exit 1 - fi + CELLD_BIN="$("$REPO/scripts/celld.sh" check "${CELLD_BIN:-}")" || exit 1 if [[ -z "${CELLD_ESBUILD:-}" ]]; then ensure_client CELLD_ESBUILD="$CLIENT/node_modules/.bin/esbuild" diff --git a/scripts/celld.sh b/scripts/celld.sh new file mode 100755 index 000000000..ddd97e085 --- /dev/null +++ b/scripts/celld.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# The one celld version this repository is tested against lives in +# `.celld-version` at the repository root. Every Makefile, run.sh and CI job +# reads it through this script. +# +# scripts/celld.sh version print the pinned version (e.g. 0.6.2) +# scripts/celld.sh check [BIN] verify BIN (default: $CELLD_BIN, else +# `celld` on PATH) is the pinned version or +# newer; print its absolute path +# scripts/celld.sh install install exactly the pinned version into +# ~/.local/bin (CI; overwrites that binary) +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PINNED="$(tr -d '[:space:]' < "$ROOT/.celld-version")" +INSTALL_HINT="curl -fsSL https://celld.dev/install.sh | CELLD_VERSION=v$PINNED sh" + +case "${1:-}" in +version) + echo "$PINNED" + ;; +check) + bin="${2:-${CELLD_BIN:-}}" + [[ -n "$bin" ]] || bin="$(command -v celld || true)" + if [[ -z "$bin" ]] || ! command -v "$bin" >/dev/null 2>&1; then + echo "celld not found: set CELLD_BIN to a celld $PINNED binary" >&2 + echo " install: $INSTALL_HINT" >&2 + echo " (or install elsewhere and point CELLD_BIN at it)" >&2 + exit 1 + fi + bin="$(command -v "$bin")" + version="$("$bin" --version 2>/dev/null | sed -n 's/^celld \([0-9][0-9.]*\).*/\1/p' | head -1)" + if [[ -z "$version" ]] || [[ "$(printf '%s\n%s\n' "$PINNED" "$version" | sort -V | head -1)" != "$PINNED" ]]; then + echo "$bin is celld ${version:-(unknown version)}; this repository needs celld $PINNED" >&2 + echo " upgrade: $INSTALL_HINT" >&2 + echo " or keep the old binary and set CELLD_BIN=/path/to/celld-$PINNED" >&2 + exit 1 + fi + echo "$bin" + ;; +install) + curl -fsSL https://celld.dev/install.sh | CELLD_VERSION="v$PINNED" sh + "$HOME/.local/bin/celld" --version + ;; +*) + sed -n '2,12p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 + exit 2 + ;; +esac diff --git a/src/bus/celld_queue.rs b/src/bus/celld_queue.rs index 7758abbfa..a8ba721ca 100644 --- a/src/bus/celld_queue.rs +++ b/src/bus/celld_queue.rs @@ -21,7 +21,7 @@ use std::sync::Arc; /// Version of the JSON body stored in celld Queue. pub const CELLD_QUEUE_ENVELOPE_VERSION: u16 = 1; -/// celld 0.4 Queue's maximum serialized message body. +/// celld Queue's maximum serialized message body. pub const CELLD_QUEUE_MAX_BODY_BYTES: usize = 128 * 1024; /// Conventional authenticated native relay endpoint used by Queue consumers. diff --git a/tests/celld/Makefile b/tests/celld/Makefile index cf3887961..11b175c2b 100644 --- a/tests/celld/Makefile +++ b/tests/celld/Makefile @@ -1,10 +1,12 @@ -# Thin entrypoint for the canonical celld 0.4 + Queue + NATS local profile. +# Thin entrypoint for the canonical celld + Queue + NATS local profile (celld +# version pinned in /.celld-version; set CELLD_BIN to use a non-PATH binary). # State persists in worker/.celld/dev across reloads and ordinary shutdowns. .PHONY: up reload logs down test help PROFILE_DIR := ../e2e-ui -CELLD_HTTP_PORT ?= 18080 +CELLD_HTTP_PORT ?= 18880 +CELLD_BIN ?= celld CELLD_URL ?= http://127.0.0.1:$(CELLD_HTTP_PORT) NATS_PORT ?= 14222 NATS_URL ?= nats://127.0.0.1:$(NATS_PORT) @@ -12,6 +14,7 @@ DISTRIBUTED_INTERNAL_SECRET ?= test-only-internal-secret-change-me-2026 export DISTRIBUTED_INTERNAL_SECRET PROFILE_VARS = \ + CELLD_BIN="$(CELLD_BIN)" \ CELLD_HTTP_PORT="$(CELLD_HTTP_PORT)" \ CELLD_URL="$(CELLD_URL)" \ NATS_PORT="$(NATS_PORT)" \ @@ -33,7 +36,7 @@ test: $(MAKE) -C $(PROFILE_DIR) test-celld $(PROFILE_VARS) help: - @echo "celld 0.4 + Queue + NATS profile" + @echo "celld + Queue + NATS profile (CELLD_BIN, version in /.celld-version)" @echo " make up build/register both Workers and start the shared local profile" @echo " make reload rebuild and restart without deleting worker/.celld/dev" @echo " make logs follow aggregate and relay registration logs" diff --git a/tests/celld/README.md b/tests/celld/README.md index d87295bb1..7c04021bb 100644 --- a/tests/celld/README.md +++ b/tests/celld/README.md @@ -1,4 +1,4 @@ -# celld 0.4 aggregate cells, Queue, and bus relay +# celld aggregate cells, Queue, and bus relay This fixture exercises the write side of the Distributed flow: @@ -45,7 +45,7 @@ on an isolated `celld dev` (including a crash and restart): ```sh (cd tests/celld/alarm-worker && worker-build --release) -CELLD=celld node tests/celld/alarms.mjs +CELLD_BIN=celld node tests/celld/alarms.mjs ``` The Queue lives in another cell: its acceptance is **not** part of the @@ -154,7 +154,8 @@ correctness. ## Local celld -Prerequisites: Docker (for NATS), celld 0.4, `worker-build`, and the +Prerequisites: Docker (for NATS), celld (the version in `/.celld-version`, +checked by `scripts/celld.sh`; `CELLD_BIN` selects the binary), `worker-build`, and the `wasm32-unknown-unknown` Rust target. ```sh @@ -163,7 +164,7 @@ make -C tests/celld test make -C tests/celld down ``` -celld 0.4 development mode supplies its own local object store; Azurite, MinIO, +celld development mode supplies its own local object store; Azurite, MinIO, and an external S3 endpoint are not used. The canonical profile stores cell and Queue data in `tests/celld/worker/.celld/dev`. `make down` and `make reload` preserve that directory. diff --git a/tests/celld/alarms.mjs b/tests/celld/alarms.mjs index a2b1a5a90..e9aae1a97 100644 --- a/tests/celld/alarms.mjs +++ b/tests/celld/alarms.mjs @@ -37,9 +37,9 @@ const headers = { "x-roles": "user", }; const nonce = Date.now().toString(36); -// `CELLD` selects the binary (default: `celld` on PATH). celld 0.6 watches the +// `CELLD_BIN` selects the binary (default: `celld` on PATH). celld 0.6 watches the // project by default; this proof owns its build, so it opts out when supported. -const celld = process.env.CELLD || "celld"; +const celld = process.env.CELLD_BIN || "celld"; const celldVersion = execFileSync(celld, ["--version"], { encoding: "utf8" }) .split("\n") .find(line => line.startsWith("celld ")); diff --git a/tests/celld/storage-conformance.mjs b/tests/celld/storage-conformance.mjs index 30e170be4..2ede52415 100644 --- a/tests/celld/storage-conformance.mjs +++ b/tests/celld/storage-conformance.mjs @@ -50,7 +50,7 @@ async function until(description, predicate, timeout = 70_000) { } async function start() { const log = createWriteStream(join(artifacts, `celld-${++generation}.log`)); - child = spawn("celld", ["dev", project, "--host", "127.0.0.1", "--port", String(port), "--logs"], { + child = spawn(process.env.CELLD_BIN || "celld", ["dev", project, "--host", "127.0.0.1", "--port", String(port), "--logs"], { detached: true, env: { ...process.env, RUST_LOG: "warn" }, stdio: ["ignore", "pipe", "pipe"], diff --git a/tests/e2e-celld/Makefile b/tests/e2e-celld/Makefile index a49d1564e..c5ae8768a 100644 --- a/tests/e2e-celld/Makefile +++ b/tests/e2e-celld/Makefile @@ -1,6 +1,6 @@ # Celld GraphQL example — sibling of tests/e2e-ui, not make run there. # -# make -C tests/e2e-ui up-celld-nats # celld 0.4 local store + Queue + NATS +# make -C tests/e2e-ui up-celld-nats # celld local store + Queue + NATS # make run # this GraphQL host + the e2e-ui Svelte app # # Distributed owns the GraphQL + sibling UI lifecycle. The independent celld @@ -16,7 +16,7 @@ PUBLIC_ORIGIN ?= http://localhost:8791 UI_INTERNAL_ORIGIN ?= http://localhost:$(UI_PORT) UI_URL ?= $(PUBLIC_ORIGIN) ENV_FILE ?= ../e2e-ui/e2e-ui.env -CELLD_HTTP_PORT ?= 18080 +CELLD_HTTP_PORT ?= 18880 CELLD_URL ?= http://127.0.0.1:$(CELLD_HTTP_PORT) NATS_PORT ?= 14222 NATS_URL ?= nats://127.0.0.1:$(NATS_PORT) @@ -46,13 +46,9 @@ run: $(if $(filter 1,$(WATCH_WORKER)),ensure-watch) _celld="$(CELLD_URL)"; \ code=$$(curl -s -o /dev/null -w '%{http_code}' "$${_celld}/health" 2>/dev/null || echo 000); \ if [ "$$code" != "200" ]; then \ - if curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:18880/health" 2>/dev/null | grep -q 200; then \ - _celld="http://127.0.0.1:18880"; \ - else \ - echo "celld not healthy at $$_celld — start: make -C ../e2e-ui up-celld-nats"; \ - echo "or: CELLD_HTTP_PORT=18880 make run"; \ - exit 1; \ - fi; \ + echo "celld not healthy at $$_celld — start: make -C ../e2e-ui up-celld-nats"; \ + echo "(a non-default port: CELLD_HTTP_PORT= make run)"; \ + exit 1; \ fi; \ export CELLD_URL="$$_celld"; \ _nats="$(NATS_URL)"; \ diff --git a/tests/e2e-celld/README.md b/tests/e2e-celld/README.md index e49637200..c796b48d5 100644 --- a/tests/e2e-celld/README.md +++ b/tests/e2e-celld/README.md @@ -51,7 +51,7 @@ make test # cargo test --workspace (CI) ```sh cd tests/e2e-ui make up # Zitadel + Postgres (read models + login) -make up-celld-nats # celld 0.4 local store + NATS +make up-celld-nats # celld local store + Queue relay + NATS cd ../e2e-celld make run # public gateway :8791 + internal UI :5180 (watches sources) @@ -82,4 +82,5 @@ Open `http://localhost:8791`. The navbar shows a **celld** badge. Sign in (`alice` / `Password1!` when Zitadel is up). Todos create/complete and lobby posts go to cells; open Chat in two tabs to see `@live` still fire. -Override a busy celld port: `CELLD_HTTP_PORT=18880 make run`. +celld listens on 18880 by default (Zitadel owns 18080). Override a busy port +with `CELLD_HTTP_PORT=` on both `make up-celld-nats` and `make run`. diff --git a/tests/e2e-ui/Makefile b/tests/e2e-ui/Makefile index b8f53fb4b..815b3654f 100644 --- a/tests/e2e-ui/Makefile +++ b/tests/e2e-ui/Makefile @@ -29,12 +29,14 @@ JS_DIR ?= ../../js DISTRIBUTED ?= cargo run --quiet --manifest-path ../../Cargo.toml -p distributed_cli --bin distributed -- CARGO_TEST_FLAGS ?= -- --nocapture -DATABASE_URL ?= sqlite:./e2e-ui.db?mode=rwc - # Optional celld+NATS profile (same UI, not the one-process playground). REPO_ROOT := $(abspath ../..) PROFILE_COMPOSE := celld-nats-profile/docker-compose.yml -CELLD_HTTP_PORT ?= 18080 +# 18080 is Zitadel's edge (docker/docker-compose.yml); celld uses 18880. +CELLD_HTTP_PORT ?= 18880 +# celld binary; scripts/celld.sh requires the version in .celld-version. +CELLD_BIN ?= celld +CELLD_CHECK := $(REPO_ROOT)/scripts/celld.sh check NATS_PORT ?= 14222 CELLD_URL ?= http://127.0.0.1:$(CELLD_HTTP_PORT) NATS_URL ?= nats://127.0.0.1:$(NATS_PORT) @@ -66,29 +68,17 @@ run-api: cargo run -p e2e-runner stop: - @stop_pidfile() { \ - [ -f "$$1" ] || return 0; \ - _pid=$$(cat "$$1"); \ - for _child in $$(pgrep -P "$$_pid" 2>/dev/null); do \ - pkill -P "$$_child" 2>/dev/null || true; \ - kill "$$_child" 2>/dev/null || true; \ - done; \ - kill "$$_pid" 2>/dev/null || true; \ - rm -f "$$1"; \ - }; \ - stop_pidfile .make-ui.pid; \ - stop_pidfile .make-runner.pid; \ - lsof -ti:$(API_PORT) 2>/dev/null | xargs kill -9 2>/dev/null || true; \ + @lsof -ti:$(API_PORT) 2>/dev/null | xargs kill -9 2>/dev/null || true; \ lsof -ti:$(UI_PORT) 2>/dev/null | xargs kill -9 2>/dev/null || true; \ echo stopped ## Optional celld + NATS profile of the same UI. Does NOT replace make run. -## celld 0.4 keeps its local object store under tests/celld/worker/.celld/dev. +## celld (version pinned in .celld-version) keeps its local object store under +## tests/celld/worker/.celld/dev. up-celld-nats: @set -e; \ command -v docker >/dev/null || { echo "docker required"; exit 1; }; \ - command -v celld >/dev/null || { echo "celld CLI required: curl -fsSL https://celld.dev/install.sh | sh"; exit 1; }; \ - celld dev --help >/dev/null 2>&1 || { echo "celld 0.4.0+ required (missing 'celld dev')"; exit 1; }; \ + celld=$$($(CELLD_CHECK) "$(CELLD_BIN)"); \ command -v worker-build >/dev/null || { echo "worker-build required: cargo install worker-build"; exit 1; }; \ $(MAKE) --no-print-directory down-celld; \ if lsof -tiTCP:$(CELLD_HTTP_PORT) -sTCP:LISTEN >/dev/null 2>&1; then \ @@ -114,7 +104,7 @@ up-celld-nats: mkdir -p $(CELLD_PROJECT)/build/relay-worker; \ cp -R $(CELLD_RELAY_PROJECT)/build/. $(CELLD_PROJECT)/build/relay-worker; \ echo "registering Queue consumer in the persistent celld dev store…"; \ - celld dev $(CELLD_PROJECT)/relay.wrangler.jsonc --host 127.0.0.1 --port $(CELLD_HTTP_PORT) --logs > $(CELLD_RELAY_LOG) 2>&1 & \ + "$$celld" dev $(CELLD_PROJECT)/relay.wrangler.jsonc --host 127.0.0.1 --port $(CELLD_HTTP_PORT) --logs > $(CELLD_RELAY_LOG) 2>&1 & \ relay_launcher_pid=$$!; \ relay_ok=0; \ for i in $$(seq 1 120); do \ @@ -150,7 +140,7 @@ up-celld-nats: echo "celld relay listener remained after TERM and KILL"; exit 1; \ fi; \ echo "starting aggregate Worker from the same celld dev store…"; \ - celld dev $(CELLD_PROJECT) --host 127.0.0.1 --port $(CELLD_HTTP_PORT) --logs > $(CELLD_LOG) 2>&1 & \ + "$$celld" dev $(CELLD_PROJECT) --host 127.0.0.1 --port $(CELLD_HTTP_PORT) --logs > $(CELLD_LOG) 2>&1 & \ launcher_pid=$$!; \ ok=0; \ for i in $$(seq 1 120); do \ @@ -186,12 +176,12 @@ down-celld: launcher_command=$$(ps -p "$$launcher_pid" -o command= 2>/dev/null || true); \ node_command=$$(ps -p "$$node_pid" -o command= 2>/dev/null || true); \ case "$$launcher_command" in \ - *"celld dev $(CELLD_PROJECT) --host 127.0.0.1 --port $(CELLD_HTTP_PORT)"*) kill "$$launcher_pid" 2>/dev/null || true ;; \ + *" dev $(CELLD_PROJECT) --host 127.0.0.1 --port $(CELLD_HTTP_PORT)"*) kill "$$launcher_pid" 2>/dev/null || true ;; \ "") ;; \ *) echo "refusing to stop unexpected tracked process $$launcher_pid: $$launcher_command"; exit 1 ;; \ esac; \ case "$$node_command" in \ - *"celld --no-control-plane --bucket celld-dev --listen 127.0.0.1:$(CELLD_HTTP_PORT)"*) kill "$$node_pid" 2>/dev/null || true ;; \ + *"--listen 127.0.0.1:$(CELLD_HTTP_PORT)"*) kill "$$node_pid" 2>/dev/null || true ;; \ "") ;; \ *) echo "refusing to stop unexpected celld node $$node_pid: $$node_command"; exit 1 ;; \ esac; \ @@ -256,7 +246,8 @@ test-domain: test-suite: cargo test -p e2e-suite --test behavioral $(CARGO_TEST_FLAGS) -## Browser e2e (Playwright). Requires make up + make run (UI :5180, API :8791). +## Browser e2e (Playwright). Requires make up + make run; the browser uses +## PUBLIC_ORIGIN (:8791), which serves the API and proxies the UI. test-browser-install: $(NPM) install npx playwright install chromium @@ -296,7 +287,7 @@ check: application-build clean: stop cargo clean - rm -f .make-runner.log e2e-ui.db + rm -f e2e-ui.db rm -rf ui/node_modules ui/build ui/.svelte-kit ui/src/lib/blob/pkg help: @@ -307,6 +298,7 @@ help: @echo " make ci-offline coherent build + offline suites with safe pipeline overlap" @echo " make down playground docker compose down (not celld/NATS)" @echo " make up-celld-nats optional celld+NATS profile (not make run)" + @echo " celld :18880, version in /.celld-version; CELLD_BIN=" @echo " rebuild/restart with make up-celld-nats after Worker source changes" @echo " make test-celld live --test celld + e2e_ui_celld_nats_profile (CI)" @echo " make test-celld-nats cargo test --test e2e_ui_celld_nats_profile" diff --git a/tests/e2e-ui/README.md b/tests/e2e-ui/README.md index a26891050..11b02b102 100644 --- a/tests/e2e-ui/README.md +++ b/tests/e2e-ui/README.md @@ -101,7 +101,7 @@ cd tests/e2e-ui && make up && make up-celld-nats cd ../e2e-celld && make run ``` -`make up-celld-nats` starts celld 0.4 with its persistent local store, the Queue +`make up-celld-nats` starts celld (version pinned in `/.celld-version`) with its persistent local store, the Queue relay, and NATS (`celld-nats-profile/`). `make test-celld-nats` only validates an already-running profile. Neither command is `make run`. GraphQL wait-dispatches Todo create/complete and `chat.post` to cells (one SQLite shard per todo or diff --git a/tests/e2e-ui/celld-nats-profile/README.md b/tests/e2e-ui/celld-nats-profile/README.md index 166e9f5cf..3ddfc8fcb 100644 --- a/tests/e2e-ui/celld-nats-profile/README.md +++ b/tests/e2e-ui/celld-nats-profile/README.md @@ -16,7 +16,7 @@ Optional profile: ```sh cd tests/e2e-ui -make up-celld-nats # celld 0.4 local store + Queue relay + NATS +make up-celld-nats # celld local store + Queue relay + NATS make test-celld # live --test celld + GraphQL wait-path smoke (CI) make test-celld-nats # GraphQL wait-path smoke + SQL list only make down-celld-nats # NATS only @@ -45,7 +45,7 @@ GraphQL, `@live`, and Eventual projectors are **not** cell class methods ## Bring-up (local only) -celld 0.4 runs through its CLI and keeps the object store under +celld runs through its CLI and keeps the object store under `tests/celld/worker/.celld/dev`. The Make target first registers the separate Queue consumer in that store, then serves the aggregate Worker. NATS is the only Docker service in this optional profile. @@ -56,7 +56,13 @@ make up-celld-nats make test-celld-nats ``` -Override ports if busy: `CELLD_HTTP_PORT=18880 NATS_PORT=14223 make up-celld-nats`. +celld listens on 18880 by default; 18080 belongs to Zitadel's edge from +`make up`, so both stacks run side by side. Override ports if busy: +`CELLD_HTTP_PORT=18881 NATS_PORT=14223 make up-celld-nats`. + +The celld version is pinned in `/.celld-version` (currently 0.6.2) and +checked by `scripts/celld.sh`. To keep an older `celld` on PATH, point the +profile at the pinned binary: `CELLD_BIN=/path/to/celld make up-celld-nats`. If `14222` is already taken by a leftover `docker run` NATS, `make down-celld-nats` removes that container too. The relevant manual commands are: @@ -70,13 +76,13 @@ The relevant manual commands are: # Register the Queue-only deployment, stop that dev listener, then serve the # aggregate project from the same local store. Use make up-celld-nats for the # complete, guarded lifecycle. -celld dev tests/celld/worker/relay.wrangler.jsonc --port 18080 --logs -celld dev tests/celld/worker --port 18080 --logs +celld dev tests/celld/worker/relay.wrangler.jsonc --port 18880 --logs +celld dev tests/celld/worker --port 18880 --logs # NATS for this optional profile only docker compose -f tests/e2e-ui/celld-nats-profile/docker-compose.yml up -d -export CELLD_URL=http://127.0.0.1:${CELLD_HTTP_PORT:-18080} +export CELLD_URL=http://127.0.0.1:${CELLD_HTTP_PORT:-18880} export NATS_URL=nats://127.0.0.1:${NATS_PORT:-14222} cargo test --test e2e_ui_celld_nats_profile --features graphql,http,sqlite diff --git a/tests/e2e-ui/celld-nats-profile/docker-compose.yml b/tests/e2e-ui/celld-nats-profile/docker-compose.yml index cede306f2..86f434b4f 100644 --- a/tests/e2e-ui/celld-nats-profile/docker-compose.yml +++ b/tests/e2e-ui/celld-nats-profile/docker-compose.yml @@ -2,7 +2,7 @@ # Not the default playground (`tests/e2e-ui/docker/docker-compose.yml` + make run). # Not a three-process GraphQL/commands/projectors matrix. # -# NATS only. celld 0.4 runs through its CLI with a persistent local dev store. +# NATS only. celld runs through its CLI with a persistent local dev store. # # docker compose -f tests/e2e-ui/celld-nats-profile/docker-compose.yml up -d # NATS_URL=nats://127.0.0.1:14222 diff --git a/tests/e2e-ui/docs/zitadel-ingestor.md b/tests/e2e-ui/docs/zitadel-ingestor.md index aec760fd7..bca4864ee 100644 --- a/tests/e2e-ui/docs/zitadel-ingestor.md +++ b/tests/e2e-ui/docs/zitadel-ingestor.md @@ -95,18 +95,22 @@ query DirectoryJoins { owner_id owner { display_name email } } - auth_users { - user_id - display_name - chat_messages { body } - blob_games { score } + todos { + title + owner { display_name } } } ``` -`auth_users.user_id` is the Zitadel subject. Chat `author_id` and Blob Game -`owner_id` are the OIDC `sub`, so the relationships resolve after that subject -has been ingested. +`auth_users.user_id` is the Zitadel subject. Chat `author_id`, Blob Game +`owner_id` and Todo `owner_id` are the OIDC `sub`, so the joins resolve after +that subject has been ingested. + +The relationships are one way: `ChatMessages.author`, `BlobGames.owner` and +`Todos.owner` point at `AuthUsers`, and `AuthUsers` declares none back (see +`crates/readmodels/src/lib.rs`), so `auth_users { chat_messages { … } }` is not +part of the schema. Query the referencing model and filter by its foreign key +instead. ## Tests diff --git a/tests/e2e-ui/scripts/dual-worktree-suite.sh b/tests/e2e-ui/scripts/dual-worktree-suite.sh index 170ef252b..c93b55b3e 100755 --- a/tests/e2e-ui/scripts/dual-worktree-suite.sh +++ b/tests/e2e-ui/scripts/dual-worktree-suite.sh @@ -122,7 +122,7 @@ wait_http() { } if [[ "$SKIP_GITOPS" != "1" ]]; then - if ! kubectl get ns >/dev/null 2>&1; then + if ! kubectl --context "$CONTEXT" get ns >/dev/null 2>&1; then die "kubectl cannot talk to a cluster (set KUBECONFIG); dual gitops requires local CP" fi log "reconciling workspace $ALICE_NAME from $ALICE_E2E" @@ -145,9 +145,9 @@ wait_http "$ALICE_URL" alice 120 wait_http "$BOB_URL" bob 120 # Assert gitops isolation: distinct OIDC names if present -if kubectl get oidc.application.zitadel.m.crossplane.io -A >/dev/null 2>&1; then - kubectl get oidc.application.zitadel.m.crossplane.io -A -o wide 2>/dev/null | tee "$SCRATCH/oidc-apps.log" || true - kubectl get project.project.zitadel.m.crossplane.io -A 2>/dev/null | tee "$SCRATCH/projects.log" || true +if kubectl --context "$CONTEXT" get oidc.application.zitadel.m.crossplane.io -A >/dev/null 2>&1; then + kubectl --context "$CONTEXT" get oidc.application.zitadel.m.crossplane.io -A -o wide 2>/dev/null | tee "$SCRATCH/oidc-apps.log" || true + kubectl --context "$CONTEXT" get project.project.zitadel.m.crossplane.io -A 2>/dev/null | tee "$SCRATCH/projects.log" || true fi # Playwright HMR — fail the suite if this fails (do not swallow via bare pipe). From bbae0562a0e5cf7713b6d179e960ce7c59e1280f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:31:38 -0500 Subject: [PATCH 23/40] docs: fix dangling paths to e2e-ui code and test dirs README pointed at a removed blob_move.rs handler (now blob-domain's move_dir.rs) and a tests/read_models/ dir that does not exist. application-composition.md and the e2e-ui README still named crates/service/src/service.rs, which became application.rs, host.rs and modules/. The gateway metrics doc now says where the load report is written versus the checked-in recorded run. [[tasks/gr-p40-examples-e2e-coverage]] --- README.md | 4 ++-- docs/application-composition.md | 4 ++-- docs/gateway/metrics.md | 9 ++++++--- tests/e2e-ui/README.md | 8 +++++--- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 256e4806b..f056e77d1 100644 --- a/README.md +++ b/README.md @@ -624,7 +624,7 @@ Start here in the code: | [`ui/src/lib/components/blob/SelectedBlobGame.graphql`](tests/e2e-ui/ui/src/lib/components/blob/SelectedBlobGame.graphql) | Reusable component island promoted to the nearest static route boundary; one route-param binding drives SSR and component reads. | | [`ui/src/routes/blob/[[gameId]]/+page.svelte`](tests/e2e-ui/ui/src/routes/blob/[[gameId]]/+page.svelte) | Arrow keys → `commands.blob.move`; board from `BlobGames.use()`. | | [`crates/service/src/modules/compose.rs`](tests/e2e-ui/crates/service/src/modules/compose.rs) | One `Service` lists modules. No `Runtime::role`. | -| [`crates/service/src/handlers/commands/blob_move.rs`](tests/e2e-ui/crates/service/src/handlers/commands/blob_move.rs) | `PreparedCommand>` — map/score written with the event. | +| [`crates/blob-domain/src/commands/move_dir.rs`](tests/e2e-ui/crates/blob-domain/src/commands/move_dir.rs) | `portable_command!` returning `Atomic`: the map/score row commits with the event, and a WASM pure previews the move. | | [`crates/todo-domain/src/models/todo.rs`](tests/e2e-ui/crates/todo-domain/src/models/todo.rs) | Plain aggregate — no GraphQL in the domain. | | [`ui/src/auth.ts`](tests/e2e-ui/ui/src/auth.ts) | Auth.js + Zitadel scopes/groups → engine roles. | @@ -2868,7 +2868,7 @@ e2e-celld (`tests/e2e-celld`), Blob game, live chat, GraphiQL. | `tests/graphql_*` | Engine, HTTP/WS, harden, identity, multi-IdP OIDC | | `tests/typed_commands/` | Eventual / `Atomic` / `Succeeded` command registration | | `tests/microsvc/` | Handlers on HTTP, gRPC, bus, session | -| `tests/read_models/`, `tests/distributed_read_model/` | Atomic vs eventual projections | +| `tests/read_model_*/`, `tests/distributed_read_model/` | Read models: commit bridge, keyset scans, relationships, sessions, Atomic vs eventual projections | | `tests/sourced*` / `tests/snapshots/` / `tests/upcasting/` | Macros, snapshots, event versioning | | `tests/sagas/` | Orchestration + choreography with the outbox | | `tests/*_transport/`, `tests/knative_cloudevents/` | Broker adapters + conformance | diff --git a/docs/application-composition.md b/docs/application-composition.md index 884d1b5e6..28f47ce4c 100644 --- a/docs/application-composition.md +++ b/docs/application-composition.md @@ -4,7 +4,7 @@ **Goal:** Make the same packages easy to run as a monolith or as microservices, with persistence/locks/transports as a separate, boring plane. **Non-goal:** Back-compat aliases or dual public APIs for the same concept. -Related pain today: `tests/e2e-ui/crates/service/src/service.rs` (~projection catalog, surfaces, command walls) and `tests/e2e-ui/crates/runner/src/main.rs` (SQLite/Postgres × API/outbox/consumer wiring). +Motivating pain (now resolved): e2e-ui once had a single `service.rs` holding the projection catalog, surfaces and command walls, and a runner that hand-wired SQLite/Postgres × API/outbox/consumer. Today the composition root is `tests/e2e-ui/crates/service/src/application.rs` (surfaces, module inventory), infrastructure is `host.rs`, each bounded context is a module under `modules/` (listed by `modules/compose.rs`), and `crates/runner/src/main.rs` is a short entrypoint. --- @@ -187,7 +187,7 @@ Rough size: **~100–150 lines of product wiring** for logical app + a tiny depl 3. **Runtime::{sqlite, postgres, in_memory}** — pair repo + locks + bus. 4. **ProcessRole** — Full / CommandWriter / EventualProjector / QueryApi spawn policy. 5. **Atomic mount checks** — fail closed if Atomic is mounted without write path. -6. **Collapse e2e-ui `service.rs` + `runner`** onto the new APIs (no dual/legacy surface). +6. **Collapse e2e-ui `service.rs` + `runner`** onto the new APIs (no dual/legacy surface). Done: see `application.rs`, `host.rs` and `modules/` in `tests/e2e-ui/crates/service/src/`. No second application-owned optimism map: projection mutation IR is the source of cache effects. diff --git a/docs/gateway/metrics.md b/docs/gateway/metrics.md index 6fac87025..e7e5796a8 100644 --- a/docs/gateway/metrics.md +++ b/docs/gateway/metrics.md @@ -18,11 +18,14 @@ worker-build as described in [worker.md](worker.md): cargo test --no-default-features --features gateway-graphql-native,gateway-delivery,sqlite --test graphql_query_protocol separate_origin_and_client_savings -- --ignored --nocapture ``` -`tests/gateway-worker/artifacts/load-report.json` records the source revision and +`tests/gateway-worker/load-runtime.mjs` writes its report to +`tests/gateway-worker/artifacts/load-report.json` (uploaded by CI, not +checked in); [`load-report.json`](load-report.json) beside this page is a +recorded run. The report records the source revision and whether it was dirty, runtime/platform, repetitions, payload size and resource limits. Native and workerd each run disabled, coalescing-only, snapshot-only and -live-only modes. A barrier holds actual SQL until100 coalesced consumers join. -Snapshots report warmup separately before100 hits. Every mode holds100 live +live-only modes. A barrier holds actual SQL until 100 coalesced consumers join. +Snapshots report warmup separately before 100 hits. Every mode holds 100 live consumers during a real projection commit; result SQL, origin validation, steady producers and all ongoing/temporary origin WebSocket connections are measured. Different-subject controls prove private isolation. External writes, explicit diff --git a/tests/e2e-ui/README.md b/tests/e2e-ui/README.md index 11b02b102..91e134b3e 100644 --- a/tests/e2e-ui/README.md +++ b/tests/e2e-ui/README.md @@ -320,7 +320,7 @@ collections for every model that references it. Read RBAC is attached to those same query models through `Todos::permissions()`, `BlobGames::permissions()`, and their peers. The runtime GraphQL engine and generated application surfaces reuse those -declarations; `service.rs` does not recreate the grants. +declarations; the service crate does not recreate the grants. ## Outcomes @@ -341,8 +341,10 @@ the normal GraphQL command result. | `crates/readmodels/` | Query shapes, relationships, and model-owned read RBAC. | | `crates/projections/` | Todo, Chat, and Blob event-to-read-model programs. | | `crates/service/src/handlers/events/` | Explicit projector handlers that apply modeled programs. | -| `crates/service/src/handlers/commands/` | Repository → get/create → domain operation → fluent commit. | -| `crates/service/src/service.rs` | Deployment catalog, placement, routes, and typed commands. | +| `crates/*-domain/src/commands/` | `portable_command!` handlers: repository → get/create → domain operation → fluent commit. | +| `crates/service/src/application.rs` | Composition root: client surfaces and the module inventory. | +| `crates/service/src/modules/` | One module per bounded context (`todo`, `chat`, `blob`) mounting its typed commands and projectors; `compose.rs` lists them. | +| `crates/service/src/host.rs` | Infrastructure: SQL dialect, outbox, OIDC, serving. | | `ui/src/routes/*/+page.graphql` | Co-located SSR/live reads. | | `ui/src/routes/*/+page.svelte` | `*.use()` and ordinary typed command calls. | | `.distributed/lifecycle/generations//ui/src/lib/generated/` | Immutable lifecycle-owned user/admin/public clients. | From 617a452ed502c1b34a50bca7f0abc109fdcdc802 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:32:21 -0500 Subject: [PATCH 24/40] docs(e2e-ui): match README samples and claims to the code The handler samples used .causal(...) and .projected(), which do not exist; they now show the real portable_command! for todo.complete and the blob.move handler ending in .atomic(). The tag line listed Kafka, RabbitMQ, Keycloak and Authentik, which this app does not use. The optimism section said all three journeys hold the request before the wire; only chat and blob do, todo holds the response. [[tasks/gr-p40-examples-e2e-coverage]] --- tests/e2e-ui/README.md | 74 +++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/tests/e2e-ui/README.md b/tests/e2e-ui/README.md index 91e134b3e..26c238adf 100644 --- a/tests/e2e-ui/README.md +++ b/tests/e2e-ui/README.md @@ -4,8 +4,8 @@ A copyable Distributed service and SvelteKit UI demonstrating one modeled projection from aggregate transition to server read model, generated GraphQL client, optimistic replica update, and causal confirmation. -Rust · TypeScript · CQRS / ES · SvelteKit · celld · Kafka · NATS · RabbitMQ · -PSQL · SQLite · OIDC · Keycloak · Authentik +Rust · TypeScript · CQRS / ES · SvelteKit · GraphQL `@live` · Postgres · +SQLite · celld + NATS (optional profile) · OIDC via Zitadel Default is **one command** (`make run`). The same UI can wait-dispatch Todo create/complete and `chat.post` to celld from [`../e2e-celld`](../e2e-celld). @@ -202,27 +202,31 @@ recovery or revalidation. Actual emitted occurrences—not the declaration—min exact obligations for the active projector binding. A no-op command therefore emits zero occurrences, mints zero obligations, and completes as `Succeeded`. -Handlers use the fluent unit of work (eventual — projector applies the mutation): +Todo commands are `portable_command!` declarations +(`crates/todo-domain/src/commands/complete.rs`). The macro loads the +aggregate, calls the domain method with the session principal, commits, and +returns the payload; the projector applies the mutation later (`Eventual`): ```rust -let repo = ctx.repo(); -let mut todo = repo - .get(&input.todo_id) - .await? - .ok_or_else(|| HandlerError::NotFound(input.todo_id.clone()))?; -todo.complete(&owner).map_err(rejected)?; - -let state = TodoState::from(&*todo); -repo.publish_events() - .commit(todo)? - .causal(TodoStatusPayload { - todo_id: state.todo_id, - status: state.status, - }) +portable_command! { + name: "todo.complete", + transition: domain_commands::Complete, + aggregate: Todo, + input: TodoCompleteInput, + outcome: Eventual, + shard: |input| input.todo_id.clone(), + load: required, + roles: ["user", "admin"], + field: "todos_complete", + invoke: |todo, _input, principal| todo.complete(principal), + payload: |todo| TodoStatusPayload::from_todo(&**todo), +} ``` -Blob uses a **handler-owned projected** commit: materialize the row from the -same mutation used for event bindings, stage it, and seal `Atomic`: +Blob uses a **handler-owned Atomic** commit +(`crates/blob-domain/src/commands/move_dir.rs`): the handler materializes the +row from the same mutation the event bindings use, stages it, and seals +`Atomic`: ```rust let repo = ctx.repo(); @@ -231,15 +235,13 @@ let mut game = repo .await? .ok_or_else(|| HandlerError::NotFound(input.game_id.clone()))?; game.move_dir(&owner, direction).map_err(rejected)?; - -let row = SaveBlobGame() - .from_state(&BlobGameState::from(&*game))?; -repo.readmodel(row) - .publish_events() - .commit(game)? - .projected() +let row = sealed_row(&*game)?; // SaveBlobGame().from_state(&BlobGameState::from(game)) +repo.readmodel(row).publish_events().commit(game)?.atomic() ``` +A hand-written handler returning `PreparedCommand>` ends with +`repo.publish_events().commit(aggregate)?.eventual(payload)` instead. + `Atomic` means aggregate history, command ledger, read-model row, and response evidence commit atomically. Its deliberately narrow eligibility is one complete row upsert; patches, deletes, multi-row programs, and stateful @@ -259,8 +261,8 @@ differs on purpose — do **not** collapse them into “always send a causal del Handler for Atomic — this *is* returning atomic read-model updates: ```rust -let row = SaveBlobGame().from_state(&BlobGameState::from(&*game))?; -repo.readmodel(row).publish_events().commit(game)?.projected() +let row = sealed_row(&*game)?; +repo.readmodel(row).publish_events().commit(game)?.atomic() // GraphQL returns BlobGames; extensions.records from same-tx evidence. ``` @@ -307,7 +309,7 @@ domain (`domain_commands::*`); client cache previews are auto-derived from input, defaults, and row-policy claims. Both Eventual and Direct surfaces may export those previews from the portable program. Eventual commands do not stage rows in the handler (the event handler applies the mutation later). Blob stages -the mutation-derived row with `readmodel(row).commit()?.projected()` so the +the mutation-derived row with `readmodel(row)…commit(game)?.atomic()` so the GraphQL response carries the authoritative row — possible only because apply happens in the command handler, not in a later event handler. @@ -390,10 +392,16 @@ Client optimism is proven in two layers: 1. **Offline artifacts** (`ui/tests/optimism-artifacts.test.mjs`, run via `make ui-test` / UI `npm test`): every demo write command must export non-empty `projection.preview.operations`; Atomic also needs `directProjection`. -2. **Browser paint-before-wire** (the Chat, Todo, and Blob product journeys plus - `e2e/helpers/optimism.ts`): hold the GraphQL mutation response longer than the - assert deadline and require the UI to update first. The same journey then - proves authoritative convergence and continuity. +2. **Browser optimism** (Playwright product journeys): + - Chat (`e2e/helpers/optimism.ts`, `expectOptimisticPaint`) holds the + mutation **before** it reaches the server and requires the UI to paint + first: paint before the wire. + - Blob holds the first move before `route.fetch()` (inline in + `blob.user.spec.ts`), so the board must move before the server sees it. + - Todo holds only the **response** (`route.fetch()` first, then a delay), + so it proves paint before the response, not before the wire. + + Each journey then proves authoritative convergence and continuity. ```bash # offline From 358a9e7717edbcbb26cd55face0ff9245c415b70 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:36:34 -0500 Subject: [PATCH 25/40] docs(e2e-ui): make walkthrough samples match the code File labels named */src/commands.rs and service.rs, which became commands/*.rs and modules/graphql.rs. The host sample called build_graphql_engine and serve_with_oidc; the runner sample lacked three HostOptions fields; the blob tab told readers to `make wasm`, which does not exist (distributed build owns the pure). The celld tabs described a CelldTodoCommandHost that dual-writes locally and routes only create/complete; the real path is the framework CelldCommandHost with per-crate CelldRoutes for every Todo transition, and projections fed by the cell outbox through the Queue relay and NATS. Removes the unused allWalkthroughs/walkthroughById exports. [[tasks/gr-p40-examples-e2e-coverage]] --- tests/e2e-ui/ui/src/lib/walkthrough/demos.ts | 163 +++++++++---------- tests/e2e-ui/ui/src/lib/walkthrough/index.ts | 4 +- 2 files changed, 80 insertions(+), 87 deletions(-) diff --git a/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts b/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts index 614acfa4c..c675daabb 100644 --- a/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts +++ b/tests/e2e-ui/ui/src/lib/walkthrough/demos.ts @@ -38,18 +38,22 @@ const hostAndRunnerSamples: WalkthroughSample[] = [ code: `// host.rs — library API used by the runner binary pub async fn run( database_url: &str, - options: HostOptions, // bind + IdentityConfig + options: HostOptions, // bind, identity, origins, gateway delivery ) -> Result<(), Box> { - // Connect SQLite or Postgres. Create tables. Open the bus. - let service = build_service(repo, locks, read_models).with_bus(bus); - let gql = build_graphql_engine(&repo, &service, identity, Some(change_rx))?; + // postgres:// → run_postgres, else run_sqlite. Each one: + // connect, create tables, open the bus. + let service = build_service(repo.clone(), locks.clone(), repo.clone()) + .with_bus(bus); + let gql = build_graphql_engine_with_delivery( + &repo, &service, options.identity.clone(), Some(change_rx), versions, + )?; let service = Arc::new(service.try_with_graphql(gql)?); - spawn_outbox_publish_loop(/* … */); - spawn_service_consumer_loop(|| build_service(/* … */).with_bus(bus)); - // Optional Zitadel scrape loop. + spawn_outbox_publish_loop(/* outbox → bus */); + spawn_service_consumer_loop(move || build_service(/* … */).with_bus(bus)); + spawn_zitadel_scrape(repo.clone()); // only when Zitadel env is set - serve_with_oidc(service, identity, &options.bind).await + serve(service, &options).await }` }, { @@ -66,6 +70,11 @@ async fn main() -> Result<(), Box> { HostOptions { bind, identity: identity_from_env(), // OidcBearer or DevHeaders + public_origin: env::var("PUBLIC_ORIGIN") + .unwrap_or_else(|_| "http://localhost:8791".into()), + ui_origin: env::var("UI_INTERNAL_ORIGIN") + .unwrap_or_else(|_| "http://localhost:5180".into()), + delivery, // GATEWAY_DELIVERY=none|all }, ) .await @@ -79,7 +88,7 @@ export const todosWalkthrough: DemoWalkthrough = { title: 'Todos', kicker: 'Query, then command, then domain, then projection', summary: - 'This page loads one GraphQL query into the client replica. Todo commands are Eventual. The client shows a safe preview from the command input and the domain transition. The event handler applies the same mutation program later. The client waits for projection obligations. The client does not wait for a response row. The same portable_command! declarations mount on a local Service (tests/e2e-ui) or wait-dispatch create and complete to a cell (tests/e2e-celld).', + 'This page loads one GraphQL query into the client replica. Todo commands are Eventual. The client shows a safe preview from the command input and the domain transition. The event handler applies the same mutation program later. The client waits for projection obligations. The client does not wait for a response row. The same portable_command! declarations mount on a local Service (tests/e2e-ui) or wait-dispatch every Todo transition to a cell (tests/e2e-celld).', tabs: [ { id: 'query', @@ -157,7 +166,7 @@ await commands.todo.create({ title: text }); await commands.todo.complete({ todo_id });` }, { - file: 'todo-domain/src/commands.rs · complete', + file: 'todo-domain/src/commands/complete.rs', caption: 'A thin portable command is shard + invoke + Eventual. The host does not write a handler body.', code: `portable_command! { name: "todo.complete", @@ -190,7 +199,7 @@ await commands.todo.complete({ todo_id });` principle: 'A command changes the write model. A table is only for reads.', samples: [ { - file: 'todo-domain/src/commands.rs · complete (thin)', + file: 'todo-domain/src/commands/complete.rs · thin', caption: 'No CausalCommandContext body. The mount loads by shard, invokes the domain, and commits Eventual.', code: `portable_command! { name: "todo.complete", @@ -201,7 +210,7 @@ await commands.todo.complete({ todo_id });` }` }, { - file: 'todo-domain/src/commands.rs · handle_create', + file: 'todo-domain/src/commands/create.rs · handle_create', caption: 'Escape hatch: get the principal, create the aggregate, call the domain, commit Eventual.', code: `portable_command! { name: "todo.create", @@ -237,8 +246,10 @@ pub async fn handle_create( { file: 'todo-domain · Todo', caption: 'This aggregate is the consistency boundary for one todo.', - code: `#[derive(Debug, Clone, Default, Serialize, Deserialize)] + code: `#[derive(Debug, Clone, Default, Serialize, Deserialize, Snapshot)] +#[snapshot(id = "todo_id")] pub struct Todo { + #[serde(skip, default)] pub entity: Entity, pub todo_id: String, pub owner_id: String, @@ -345,7 +356,7 @@ mutation DeleteTodo { { id: 'service', label: '6 · Service + host', - lede: 'The domain crate is the same on both hosts. `tests/e2e-ui` is one process: SQLite or Postgres, a bus, GraphQL, and Eventual projectors. `tests/e2e-celld` is the same UI and GraphQL process, but Todo create and complete wait-dispatch to a cell (one private SQLite per todo id). GraphQL, `@live`, and projectors are not cell methods. Chat posts also wait-dispatch to a Chat cell on that path; Blob stays in-process.', + lede: 'The domain crate is the same on both hosts. `tests/e2e-ui` is one process: SQLite or Postgres, a bus, GraphQL, and Eventual projectors. `tests/e2e-celld` is the same UI and GraphQL process, but every Todo transition wait-dispatches to a cell (one private SQLite per todo id). GraphQL, `@live`, and projectors are not cell methods. Chat posts also wait-dispatch to a Chat cell on that path; Blob stays in-process.', principle: 'Compose modules into one Service. Let the host start the process. Keep the runner small.', samples: [ { @@ -384,26 +395,24 @@ pub const E2E_UI_MODULE_IDS: &[&str] = compose::MODULE_IDS; }, ...hostAndRunnerSamples, { - file: 'e2e-celld · CelldTodoCommandHost', - caption: 'The celld example GraphQL process wait-dispatches todo.create and todo.complete to POST {CELLD_URL}/todo/{id}/{command}. Other commands stay local. SQL lists dual-write after the cell wait-path so the page can render.', - code: `// tests/e2e-celld — not make run -impl CommandHost for CelldTodoCommandHost { - async fn invoke(&self, command, command_id, input, …) { - if command == "todo.create" || command == "todo.complete" { - let id = input["todo_id"]; - HttpCommandHost::new(format!("{celld}/todo/{id}")) - .invoke(command, command_id, input, …) - .await?; - // then local dual-write so Eventual SQL lists fill - } - self.local.invoke(command, command_id, input, …).await - } + file: 'e2e-celld · graphql-service/host.rs · celld_command_host', + caption: 'The celld example GraphQL process uses the framework `CelldCommandHost`. Each aggregate crate supplies a `CelldRoute`: its commands, the cell kind, the shard, and the GraphQL payload. Every Todo transition goes to POST {CELLD_URL}/todo/{todo_id}/{command}. Commands with no route run on the local Service. The cell outbox reaches the projectors through the celld Queue relay and NATS. There is no dual write.', + code: `// tests/e2e-celld/crates/graphql-service/src/host.rs +CelldCommandHost::new(celld_url, service, internal_secret)? + .route(e2e_celld_todo::celld_route()) + .route(e2e_celld_chat::celld_route()) + +// tests/e2e-celld/crates/todo-service/src/host.rs +pub fn celld_route() -> CelldRoute { + CelldRoute::new(TODO_CELL_COMMANDS, "todo", todo_shard, graphql_todo_payload) } +// TODO_CELL_COMMANDS: create, rename, complete, reopen, +// archive, force_archive, purge // cell class (tests/celld/worker): command HTTP + sealed GET only POST /todo/:id/todo.create { commandId, input } -POST /todo/:id/todo.complete -GET /todo/:id // sealed row` +GET /todo/:id // sealed row +// cell outbox → celld Queue → /internal/celld-queue/relay → NATS → projectors` } ] } @@ -524,9 +533,9 @@ export type GeneratedCommands = Readonly>;` principle: 'Use the signed-in principal. Do not trust the author field in the request body.', samples: [ { - file: 'chat-domain/src/commands.rs · handle_post', + file: 'chat-domain/src/commands/post.rs · handle_post', caption: 'Get the principal. Reject a duplicate id. Create the aggregate. Call `post`. Commit Eventual.', - code: `pub async fn handle( + code: `pub async fn handle_post( ctx: &CausalCommandContext<'_, ChatMessage>, input: ChatPostInput, ) -> Result>, HandlerError> { @@ -676,7 +685,7 @@ mutation SaveChatMessage { { id: 'service', label: '6 · Service + host', - lede: 'The chat module mounts lobby posts. On the one-process playground it also mounts identity ingress. The module mounts the chat projector. `build_service` adds these routes to one Service. On `tests/e2e-celld`, `chat.post` wait-dispatches to a Chat cell; the local host dual-writes so this projector and `@live` still run. The host starts the process. The runner only reads the environment.', + lede: 'The chat module mounts lobby posts. On the one-process playground it also mounts identity ingress. The module mounts the chat projector. `build_service` adds these routes to one Service. On `tests/e2e-celld`, `chat.post` wait-dispatches to a Chat cell; the cell outbox reaches this projector through the celld Queue relay and NATS, so `@live` still runs. The host starts the process. The runner only reads the environment.', principle: 'Compose modules into one Service. Let the host start the process. Keep the runner small.', samples: [ { @@ -711,25 +720,17 @@ Service::new() }, ...hostAndRunnerSamples, { - file: 'e2e-celld · CelldChatCommandHost', - caption: 'The celld example wait-dispatches chat.post to POST {CELLD_URL}/chat/{message_id}/chat.post. Then it dual-writes the local Chat service so Eventual projectors update SQL and GraphQL @live still fires. GraphQL is not a cell method.', - code: `// tests/e2e-celld — not make run -impl CommandHost for CelldChatCommandHost { - async fn invoke(&self, command, command_id, input, …) { - if command == "chat.post" { - let id = input["message_id"]; - HttpCommandHost::new(format!("{celld}/chat/{id}")) - .invoke(command, command_id, input, …) - .await?; - // then local dual-write so the projector + @live fill - } - self.local.invoke(command, command_id, input, …).await - } + file: 'e2e-celld · chat-service/host.rs · celld_route', + caption: 'The celld example routes chat.post to POST {CELLD_URL}/chat/{message_id}/chat.post through the same `CelldCommandHost`. The cell outbox reaches the chat projector through the celld Queue relay and NATS, so SQL updates and GraphQL @live fires. GraphQL is not a cell method.', + code: `// tests/e2e-celld/crates/chat-service/src/host.rs +pub fn celld_route() -> CelldRoute { + CelldRoute::new(&["chat.post"], "chat", chat_shard, graphql_chat_payload) } // cell class (tests/celld/worker): command HTTP + sealed GET only POST /chat/:id/chat.post { commandId, input } -GET /chat/:id // sealed row` +GET /chat/:id // sealed row +// cell outbox → celld Queue → relay → NATS → chat projector → @live` } ] } @@ -852,7 +853,7 @@ await commands.blob.move({ game_id, direction }); // Atomic seal confirms or corrects` }, { - file: 'blob-domain/src/commands.rs · move_dir', + file: 'blob-domain/src/commands/move_dir.rs · mount', caption: 'The domain mount names the pure function, the WASM package, the keys, the args, and the assign fields.', code: `.mount(blob_domain::commands::move_dir()) // preview: blob.simulate_move → blobSimulateMove WASM` @@ -891,9 +892,9 @@ export async function ensurePureFunctionsReady() { principle: 'A command changes the write model. A table is only for reads.', samples: [ { - file: 'blob-domain/src/commands.rs · handle_move', + file: 'blob-domain/src/commands/move_dir.rs · handle_move', caption: 'Load the game. Call `move_dir`. Stage `SaveBlobGame`. Commit Atomic. The Atomic row is replica authority.', - code: `pub async fn handle( + code: `pub async fn handle_move( ctx: &CausalCommandContext<'_, BlobGame>, input: BlobMoveInput, ) -> Result>, HandlerError> { @@ -913,13 +914,10 @@ export async function ensurePureFunctionsReady() { // move_dir → core::simulate_move (same pure as client WASM) game.move_dir(&owner, dir).map_err(rejected)?; - let row = SaveBlobGame() - .from_state(&BlobGameState::from(&*game)) - .map_err(|error| HandlerError::Other(Box::new(error)))?; - repo.readmodel(row) - .publish_events() - .commit(game)? - .atomic() // sealed row → replica authority + // sealed_row = SaveBlobGame().from_state(&BlobGameState::from(game)) + let row = sealed_row(&*game)?; + repo.readmodel(row).publish_events().commit(game)?.atomic() + // sealed row → replica authority }` } ] @@ -974,11 +972,14 @@ pub fn blob_simulate_move(record_json: &str, args_json: &str) }` }, { - file: 'blob-domain · features + make wasm', - caption: 'The server builds the default `domain` feature. The client pure function builds with `--features wasm`.', + file: 'blob-domain · features + distributed build', + caption: 'The server builds the default `domain` feature. `distributed build` (and `distributed dev`) compiles the declared pure with wasm-pack and `--features wasm` into `$lib/blob/pkg`. There is no separate make target.', code: `// Cargo.toml // default = ["domain"] → aggregate + levels + distributed -// wasm → blobSimulateMove (make wasm → $lib/blob/pkg)` +// wasm → blobSimulateMove +// +// move_dir.rs declares the package path: +// CommandProjectionPureReduce::wasm("blob.simulate_move", "blob/pkg/blob_wasm", …)` } ] }, @@ -1124,15 +1125,19 @@ pub struct Todos { }` }, { - file: 'service.rs · dual surfaces', - caption: 'One engine opens three surfaces: user, admin, and public.', + file: 'modules/graphql.rs · build_engine surfaces', + caption: 'One engine opens three surfaces: user, admin, and public. Each surface names its eligible roles and its privilege pack.', code: `.client_application_surface_with_schema_roles( - "e2e-ui", - ["admin", "user"], // eligible - ["user"], // portable privilege pack + DISTRIBUTED_CLIENT_SURFACE, // "e2e-ui" + ["admin", "user"], // eligible + ["user"], // portable privilege pack ) -.client_application_surface("e2e-ui-admin", ["admin"]) -.client_application_surface("e2e-ui-public", ["anonymous"])` +.client_application_surface(DISTRIBUTED_ADMIN_CLIENT_SURFACE, ["admin"], ["admin"]) +.client_application_surface( + DISTRIBUTED_PUBLIC_CLIENT_SURFACE, // "e2e-ui-public" + ["anonymous"], + ["anonymous"], +)` } ] }, @@ -1179,9 +1184,9 @@ pub struct Todos { principle: 'Use the signed-in principal. Do not trust the request body for identity.', samples: [ { - file: 'todo-domain/src/commands.rs · handle_force_archive', + file: 'todo-domain/src/commands/force_archive.rs · handle_force_archive', caption: 'Get the admin principal. Load the todo. Call the domain. Commit Eventual.', - code: `pub async fn handle( + code: `pub async fn handle_force_archive( ctx: &CausalCommandContext<'_, Todo>, input: TodoForceArchiveInput, ) -> Result>, HandlerError> { @@ -1215,8 +1220,10 @@ pub struct Todos { { file: 'todo-domain · Todo', caption: 'This is the same aggregate shape as the user surface.', - code: `#[derive(Debug, Clone, Default, Serialize, Deserialize)] + code: `#[derive(Debug, Clone, Default, Serialize, Deserialize, Snapshot)] +#[snapshot(id = "todo_id")] pub struct Todo { + #[serde(skip, default)] pub entity: Entity, pub todo_id: String, pub owner_id: String, @@ -1409,7 +1416,7 @@ const client = provideDistributed({ });` }, { - file: 'service.rs · oidc_bearer_config', + file: 'modules/graphql.rs · oidc_bearer_config', caption: 'This claim map sets the allowed engine roles on the session.', code: `oidc.claim_map.engine_roles = vec![ "user".into(), @@ -1588,15 +1595,3 @@ pub async fn handle( } ] }; - -export const allWalkthroughs: DemoWalkthrough[] = [ - todosWalkthrough, - chatWalkthrough, - blobWalkthrough, - adminWalkthrough, - sessionWalkthrough -]; - -export function walkthroughById(id: string): DemoWalkthrough | undefined { - return allWalkthroughs.find((d) => d.id === id); -} diff --git a/tests/e2e-ui/ui/src/lib/walkthrough/index.ts b/tests/e2e-ui/ui/src/lib/walkthrough/index.ts index 7025f3ed6..3961e15c7 100644 --- a/tests/e2e-ui/ui/src/lib/walkthrough/index.ts +++ b/tests/e2e-ui/ui/src/lib/walkthrough/index.ts @@ -1,10 +1,8 @@ export type { DemoWalkthrough, WalkthroughSample, WalkthroughTab } from './types'; export { adminWalkthrough, - allWalkthroughs, blobWalkthrough, chatWalkthrough, sessionWalkthrough, - todosWalkthrough, - walkthroughById + todosWalkthrough } from './demos'; From fa6c5b5194e831ff32a7575da62ae5ed5625601a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:39:47 -0500 Subject: [PATCH 26/40] docs(game)!: drop pre-celld wording, rename the shared cells worker The cells worker and its Queue carry every demo's cells (Mob Pit, The Freeze, zone leases and state), so mobpit-cells / mobpit-outbox become game-demo-cells / game-demo-outbox (and -dlq). Docs no longer say grid zones hibernate into SQLite, that the zone server drains the outbox, or that only mobpit|leaderboard|sveltekit need celld. The README Layout lists cells/, devcelld.rs, common/{celld,relay, zone_status}.rs and helpers/celld.mjs; dev routes list /dev/mobpit/presence; the demo count is eight. run.sh help prints the whole header instead of stopping at line 39. BREAKING CHANGE: the cells worker is game-demo-cells and its Queue game-demo-outbox; existing local celld dev state under .data/ keyed by the old worker name is not reused. [[tasks/gr-p40-examples-e2e-coverage]] --- README.md | 2 +- distributed_game/README.md | 2 +- examples/game/README.md | 38 +++++++++++++------- examples/game/cells/Cargo.toml | 2 +- examples/game/cells/src/lib.rs | 2 +- examples/game/cells/wrangler.jsonc | 8 ++--- examples/game/client/index.html | 2 +- examples/game/client/tests/helpers/celld.mjs | 8 ++--- examples/game/run.sh | 10 +++--- examples/game/server/service/Cargo.toml | 5 +-- examples/game/server/service/src/lib.rs | 8 +++-- examples/game/server/src/devcelld.rs | 15 ++++---- examples/game/server/src/grid/mod.rs | 3 +- examples/game/server/src/mobpit/store.rs | 2 +- examples/game/server/tests/mobpit_smoke.rs | 2 +- 15 files changed, 65 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index f056e77d1..bbd336ac0 100644 --- a/README.md +++ b/README.md @@ -571,7 +571,7 @@ with interest management, client prediction (`@hops-ops/distributed/game`), outcomes saved as idempotent commands on cells without ever pausing the game, and zones spread across servers with handoff, hibernation and live content updates. Start with **[`distributed_game/README.md`](distributed_game/README.md)**; -seven runnable demos are in [`examples/game`](examples/game/README.md). +eight runnable demos are in [`examples/game`](examples/game/README.md). --- diff --git a/distributed_game/README.md b/distributed_game/README.md index 764528e93..616f1da4b 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -94,7 +94,7 @@ The words used throughout: ## Try it -Seven runnable demos live in [`examples/game`](../examples/game/README.md), +Eight runnable demos live in [`examples/game`](../examples/game/README.md), each built to show one part of the engine: | Demo | Run | Shows | diff --git a/examples/game/README.md b/examples/game/README.md index 299bc45ba..c33bde557 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -369,6 +369,10 @@ What the TypeScript still restates, and why it stays: ``` examples/game/ run.sh one-command dev + smoke runner + cells/ crate `game-demo-cells` (own workspace, wasm32, worker-build): the + demos' celld cells — Crawler + presence (Mob Pit), gem pouch + + content log (The Freeze), zone lease + zone state (multi-host + demos), and the `game-demo-outbox` Queue consumer (relay) server/ Rust crate `game_demos`, its own cargo workspace (with sim/, crawler/, service/); also the `distributed build|dev` project crawler/ crate `mobpit_crawler`: Crawler aggregate, RecordKill (XP rules), @@ -382,11 +386,15 @@ examples/game/ server and to wasm32 for the browser src/main.rs `game-demos ` subcommands (+ hidden `predict-trace`) src/predict_trace.rs native traces of every step (recorded inputs) and pure function (arguments) - src/common/ shared: Demo trait, run/spawn, dev routes, guests + src/common/ shared: Demo trait, run/spawn, dev routes, guests; + celld.rs (CelldOptions: celld URL + internal secret), + relay.rs (authenticated /internal/celld-queue/relay → SqliteBus), + zone_status.rs (zone lease/snapshot status read model) + src/devcelld.rs `celld dev` harness for tests (own project, port, process tree) src/cluster.rs multi-host demos: --host-id/--peer, directory, ownership, snapshots src/arena/ Arena zone (`movement` = `game_sim::arena`) - src/mobpit/ Mob Pit zone, Crawler cells, fact sink + projection-bus drain, - leaderboards, dev routes + src/mobpit/ Mob Pit zone, fact sink to the Crawler/presence cells, crawler + directory projection, leaderboards, dev routes src/gallery/ Shooting Gallery zone (targets, rewind hit tests) src/freeze/ The Freeze zone, content packs, two-host world + admin routes src/grid/ Endless Grid zone (`world` = `game_sim::grid`), dev map routes @@ -425,7 +433,8 @@ examples/game/ arena-headless, mobpit-headless, leaderboard-headless, gallery-headless, freeze-headless, grid-headless, ripple-headless (+ helpers/demo-server.mjs, helpers/mobpit.mjs, - helpers/cluster.mjs for multi-host demos) + helpers/cluster.mjs for multi-host demos, helpers/celld.mjs for + a per-test `celld dev`) ``` The server crate is excluded from the root workspace (`Cargo.toml` @@ -481,7 +490,7 @@ after dying. Level `n` starts at `40·(n−1)²` XP. 5. The read model (`crawler/`, `service/`): `crawler.mob_killed` is a **domain event** — the same transaction writes the post-kill `CrawlerState` (principal, name, XP, kills, level) to the cell's outbox - (`repo.publish_events()`). celld drains the outbox to the `mobpit-outbox` + (`repo.publish_events()`). celld drains the outbox to the `game-demo-outbox` Queue; the cells worker's Queue consumer relays each message to the zone server's `/internal/celld-queue/relay` (secret-authenticated), which publishes it onto the `SqliteBus` at `--projection-bus` (default @@ -547,12 +556,15 @@ kills ≤ settled + in flight`, the board (rebuilt at boot) equals the cells, and re-committing a pre-crash fact (same key and input) returns the original receipt with `replayed: true` and unchanged kills. -**celld.** `run.sh mobpit|leaderboard|sveltekit|smoke` need celld 0.6.2+ -(`CELLD_BIN`, default `celld` on PATH; install with -`curl -fsSL https://celld.dev/install.sh | CELLD_VERSION=v0.6.2 sh`) and -`worker-build` (`cargo install worker-build --locked`) for `cells/`. The -tests (`tests/mobpit_smoke.rs`, `service/tests/projection.rs`, the headless -and Playwright tests) each start their own `celld dev` on a temp project +**celld.** Every demo with durable state runs its cells on celld: +`run.sh mobpit|leaderboard|freeze|grid|ripple|sveltekit|smoke` need the celld +version pinned in the repository's `.celld-version` (`CELLD_BIN`, default +`celld` on PATH; `scripts/celld.sh check` verifies it and prints the install +command) and `worker-build` (`cargo install worker-build --locked`) for +`cells/`. Only `arena` and `gallery` run without celld. The tests +(`tests/mobpit_smoke.rs`, `tests/freeze_smoke.rs`, `tests/grid_smoke.rs`, +`tests/ripple_smoke.rs`, `service/tests/projection.rs`, the headless and +Playwright tests) each start their own `celld dev` on a temp project (`game_demos::devcelld`, `client/tests/helpers/celld.mjs`). ## Leaderboard @@ -584,7 +596,9 @@ and Playwright tests) each start their own `celld dev` on a temp project Dev routes (`--dev` only, none takes an identity or a reward): `GET|POST /dev/mobpit/storage`, `GET /dev/mobpit/stats` (sink counters, -pending, recent commits), `GET /dev/mobpit/crawlers` (every cell on disk), +pending, recent commits), `GET /dev/mobpit/crawlers` (every crawler in the +directory: the `player_stats` projection of the cells' events, eventually +consistent), `GET /dev/mobpit/presence` (the presence cells' view), `GET /dev/mobpit/leaderboard[?member=]`, `POST /dev/mobpit/leaderboard/rebuild`. ## Adding a demo diff --git a/examples/game/cells/Cargo.toml b/examples/game/cells/Cargo.toml index 47f109f76..65545b171 100644 --- a/examples/game/cells/Cargo.toml +++ b/examples/game/cells/Cargo.toml @@ -10,7 +10,7 @@ members = ["."] resolver = "2" [package] -name = "mobpit-cells" +name = "game-demo-cells" version = "0.1.0" edition = "2021" publish = false diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs index d2a4cf4af..b7cfc59e7 100644 --- a/examples/game/cells/src/lib.rs +++ b/examples/game/cells/src/lib.rs @@ -7,7 +7,7 @@ //! aggregate with its `#[command]` `crawler.record_kill` (XP decided here //! from the mob tier; the principal comes from the trusted fact session). //! Each commit writes the `crawler.mob_killed` domain event to the cell -//! outbox, which drains to the `mobpit-outbox` Queue. +//! outbox, which drains to the `game-demo-outbox` Queue. //! - `presence/{principal}` (`PresenceCell`): the runtime's player lifecycle //! facts (`zone.*`). A fact emitted before the one that set the current //! entry (`FactStamp::emission_order`) is ignored, so a late "left pit-1" diff --git a/examples/game/cells/wrangler.jsonc b/examples/game/cells/wrangler.jsonc index b4b42e049..c78fe9da8 100644 --- a/examples/game/cells/wrangler.jsonc +++ b/examples/game/cells/wrangler.jsonc @@ -3,7 +3,7 @@ // copy of this file with DISTRIBUTED_INTERNAL_SECRET and // CELLD_QUEUE_RELAY_URL (the zone server's `/internal/celld-queue/relay`). { - "name": "mobpit-cells", + "name": "game-demo-cells", "main": "build/worker/shim.mjs", "compatibility_date": "2026-01-01", "durable_objects": { @@ -27,15 +27,15 @@ ], "queues": { "producers": [ - { "binding": "OUTBOX", "queue": "mobpit-outbox" } + { "binding": "OUTBOX", "queue": "game-demo-outbox" } ], "consumers": [ { - "queue": "mobpit-outbox", + "queue": "game-demo-outbox", "max_batch_size": 10, "max_batch_timeout": 1, "max_retries": 100, - "dead_letter_queue": "mobpit-outbox-dlq", + "dead_letter_queue": "game-demo-outbox-dlq", "retry_delay": 2 } ] diff --git a/examples/game/client/index.html b/examples/game/client/index.html index dda69fcad..4615fc991 100644 --- a/examples/game/client/index.html +++ b/examples/game/client/index.html @@ -12,7 +12,7 @@

distributed_game demos

Each demo runs its own server: examples/game/run.sh <demo>. The page for the running demo works; the others need their server.

  • Arena — fixed tick, binary deltas, interest management, prediction + reconciliation, interpolation.
    • -
  • Mob Pit — authoritative combat, non-blocking durable facts into SQLite cells, slow storage, crash recovery.
    • +
  • Mob Pit — authoritative combat, non-blocking durable facts into per-player celld cells, slow storage, crash recovery.
  • Leaderboard — kills/XP sorted-set boards projected from the cells, pushed live, rebuilt from events (runs with Mob Pit).
  • Shooting Gallery — lag compensation: hitscan shots rewound to the shooter's view time, max-rewind clamp, compensation on/off.
  • The Freeze — world-version barrier across two server processes as a story beat: hard reload, safe-room gating of old clients, aborted rollout.
    • diff --git a/examples/game/client/tests/helpers/celld.mjs b/examples/game/client/tests/helpers/celld.mjs index a138872c8..e4de7bfcc 100644 --- a/examples/game/client/tests/helpers/celld.mjs +++ b/examples/game/client/tests/helpers/celld.mjs @@ -1,5 +1,5 @@ /** - * `celld dev` serving Mob Pit's cells (`examples/game/cells`) for one Node + * `celld dev` serving the demos' cells (`examples/game/cells`) for one Node * test: its own temp project, port and internal secret; `stop()` kills the * process tree this helper started (nothing else). * @@ -38,11 +38,11 @@ export function freePort() { */ export async function startCelld({ relayPort } = {}) { const bin = process.env.CELLD_BIN; - if (!bin) throw new Error('CELLD_BIN is not set: Mob Pit\'s cells run on celld (0.6.2+); point CELLD_BIN at the celld binary'); + if (!bin) throw new Error('CELLD_BIN is not set: the demos\' cells run on celld (the version in .celld-version); point CELLD_BIN at the celld binary'); if (!existsSync(join(CELLS, 'build/worker/shim.mjs'))) { - throw new Error(`the Mob Pit cells worker is not built: run \`worker-build --release\` in ${CELLS}`); + throw new Error(`the demos' cells worker is not built: run \`worker-build --release\` in ${CELLS}`); } - const project = mkdtempSync(join(tmpdir(), 'mobpit-celld-js-')); + const project = mkdtempSync(join(tmpdir(), 'game-demo-celld-js-')); cpSync(join(CELLS, 'build'), join(project, 'build'), { recursive: true }); cpSync(join(CELLS, 'wrangler.jsonc'), join(project, 'wrangler.jsonc')); const secret = `test-only-${randomBytes(24).toString('hex')}`; diff --git a/examples/game/run.sh b/examples/game/run.sh index b04ff4270..71db0acb6 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -106,7 +106,8 @@ ensure_celld() { export CELLD_BIN CELLD_ESBUILD } -# ensure_cells: build the Mob Pit cells worker when its sources changed. +# ensure_cells: build the demos' cells worker (examples/game/cells) when its +# sources changed. ensure_cells() { local marker="$CELLS/build/worker/shim.mjs" if [[ ! -f "$marker" ]] || [[ -n "$(find "$CELLS/src" "$CELLS/Cargo.toml" "$SERVER/crawler" "$REPO/src" "$REPO/distributed_game/src" -newer "$marker" -name '*.rs' -print -quit 2>/dev/null)" ]] || @@ -115,7 +116,7 @@ ensure_cells() { echo "worker-build not found: cargo install worker-build --locked" >&2 exit 1 } - log "building the Mob Pit cells worker (worker-build)" + log "building the demos' cells worker (worker-build)" (cd "$CELLS" && worker-build --release >/dev/null) fi } @@ -379,7 +380,8 @@ run_sveltekit() { smoke() { ensure_sdk ensure_client - # Mob Pit's tests start their own `celld dev` (CELLD_BIN) on the worker. + # The celld-backed tests (Mob Pit, The Freeze, grid, ripple) start their + # own `celld dev` (CELLD_BIN) on the cells worker. ensure_celld ensure_cells log "generated clients are current; predicted steps and pure functions compiled to WASM" @@ -411,7 +413,7 @@ case "${1:-}" in gen) gen "${2:-}" ;; sveltekit) run_sveltekit ;; list) printf '%s\n' "${DEMOS[@]}" sveltekit ;; - "" | -h | --help | help) sed -n '2,39p' "$0" ;; + "" | -h | --help | help) awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0" ;; *) if [[ " ${CLUSTER_DEMOS[*]} " == *" $1 "* ]]; then run_cluster "$1" diff --git a/examples/game/server/service/Cargo.toml b/examples/game/server/service/Cargo.toml index e684ea58d..300ce37fa 100644 --- a/examples/game/server/service/Cargo.toml +++ b/examples/game/server/service/Cargo.toml @@ -1,6 +1,7 @@ # Mob Pit's business-side read model: a Distributed service that projects -# the Crawler cells' published `crawler.mob_killed` domain events (drained -# by the zone server onto a SQLite bus) into a `player_stats` SQL table and +# the Crawler cells' published `crawler.mob_killed` domain events (celld +# drains each cell outbox to a Queue; its consumer relays them through the +# zone server onto a SQLite bus) into a `player_stats` SQL table and # serves it over GraphQL with `@live`. Read-only: it has no commands, so # nothing here can credit XP. [package] diff --git a/examples/game/server/service/src/lib.rs b/examples/game/server/service/src/lib.rs index a17ff3566..0d8a84d9b 100644 --- a/examples/game/server/service/src/lib.rs +++ b/examples/game/server/service/src/lib.rs @@ -1,9 +1,11 @@ //! Mob Pit's business-side read model (dev demo). //! //! ```text -//! zone ──fact──▶ Crawler cell (RecordKill: XP decided there; event + outbox -//! row in one SQLite transaction) -//! └─ zone server drains the cell outbox ──▶ SQLite bus (`bus_log`) +//! zone ──fact──▶ Crawler cell on celld (RecordKill: XP decided there; event +//! + outbox row in one cell SQLite transaction) +//! └─ celld drains the outbox ──▶ `game-demo-outbox` Queue ──▶ consumer +//! relays to the zone server's /internal/celld-queue/relay +//! ──▶ SQLite bus (`bus_log`) //! │ ordered, at-least-once //! mobpit-service: modeled projector `project_player_stats` ◀──┘ //! └─ upsert `player_stats` (checkpointed, deduplicated) ──▶ GraphQL diff --git a/examples/game/server/src/devcelld.rs b/examples/game/server/src/devcelld.rs index 75682e971..566904446 100644 --- a/examples/game/server/src/devcelld.rs +++ b/examples/game/server/src/devcelld.rs @@ -1,4 +1,5 @@ -//! `celld dev` serving Mob Pit's cells (`examples/game/cells`) for tests: +//! `celld dev` serving the demos' cells (`examples/game/cells`: Mob Pit, +//! The Freeze, zone leases and hibernation state) for tests: //! its own project directory, port and process tree (killed on drop; only //! the processes this harness started). //! @@ -52,11 +53,11 @@ pub struct DevCelld { impl DevCelld { /// Start celld on a fresh temp project. `relay_url` is where the outbox - /// Queue consumer relays domain events (a Mob Pit server's + /// Queue consumer relays domain events (a demo server's /// `/internal/celld-queue/relay`); `None` keeps them queued. pub fn start(relay_url: Option) -> Result { let project = std::env::temp_dir().join(format!( - "mobpit-celld-{}-{}", + "game-demo-celld-{}-{}", std::process::id(), crate::common::random_hex(8) )); @@ -70,7 +71,7 @@ impl DevCelld { let build = cells_dir().join("build"); if !build.join("worker/shim.mjs").exists() { return Err(format!( - "the Mob Pit cells worker is not built: run `worker-build --release` in {} \ + "the demos' cells worker is not built: run `worker-build --release` in {} \ (examples/game/run.sh does)", cells_dir().display() )); @@ -100,7 +101,7 @@ impl DevCelld { format!("http://127.0.0.1:{}", self.port) } - /// Options for a Mob Pit world on this celld. + /// Options for a demo world on this celld. pub fn options(&self) -> CelldOptions { CelldOptions::new(self.url(), self.secret.clone()).expect("celld options") } @@ -119,8 +120,8 @@ impl DevCelld { self.generation += 1; let binary = std::env::var(CELLD_BIN_ENV).map_err(|_| { format!( - "{CELLD_BIN_ENV} is not set: Mob Pit's cells run on celld (0.6.2+); \ - point {CELLD_BIN_ENV} at the celld binary" + "{CELLD_BIN_ENV} is not set: the demos' cells run on celld (the version in \ + .celld-version); point {CELLD_BIN_ENV} at the celld binary" ) })?; let mut vars = format!("DISTRIBUTED_INTERNAL_SECRET={}\n", self.secret); diff --git a/examples/game/server/src/grid/mod.rs b/examples/game/server/src/grid/mod.rs index 362889dc6..55b774604 100644 --- a/examples/game/server/src/grid/mod.rs +++ b/examples/game/server/src/grid/mod.rs @@ -18,7 +18,8 @@ //! ring ([`HashRing`]); a Hello for a foreign zone gets `Redirect`, a //! handoff into it reserves the transfer ticket with the issuing host. //! - **Hibernation**: a zone nobody is in or observing turns Dormant, then -//! hibernates into SQLite (critter positions and dropped markers) and +//! hibernates into its `zone-state/{zone}` celld cell (critter positions +//! and dropped markers; large snapshots in the cell's R2 bucket) and //! restores on the next activation; its hello says `restored`. //! //! Zone ids: `grid..` with canonical integers, |x|, |y| ≤ 1 000 000. diff --git a/examples/game/server/src/mobpit/store.rs b/examples/game/server/src/mobpit/store.rs index bc8b50195..156bcfc8e 100644 --- a/examples/game/server/src/mobpit/store.rs +++ b/examples/game/server/src/mobpit/store.rs @@ -14,7 +14,7 @@ //! zone's queue (`RetryPolicy::until_settled`), never in a local store. //! - Each kill commit also writes the `crawler.mob_killed` domain event to //! the cell's outbox in the same cell transaction. celld drains it to the -//! `mobpit-outbox` Queue; the worker's consumer relays every message to +//! `game-demo-outbox` Queue; the worker's consumer relays every message to //! this server's authenticated `/internal/celld-queue/relay` //! (`crate::common::relay`), which publishes it onto the projection bus. //! `mobpit-service` and this server's [`super::directory`] project it diff --git a/examples/game/server/tests/mobpit_smoke.rs b/examples/game/server/tests/mobpit_smoke.rs index b356b202f..aa07bd36f 100644 --- a/examples/game/server/tests/mobpit_smoke.rs +++ b/examples/game/server/tests/mobpit_smoke.rs @@ -436,7 +436,7 @@ async fn bots_fight_slow_storage_keeps_the_tick_flat_and_boards_match_cells() { }) .await; let total = assert_board_matches_cells(&world).await; - eprintln!("bots: {stats:?}, kills on disk {total}"); + eprintln!("bots: {stats:?}, durable kills in the cells {total}"); assert_eq!(total, stats.committed, "one kill per committed fact"); let (_, rebuild) = http(demo.addr, "POST", "/dev/mobpit/leaderboard/rebuild", None).await; assert_eq!(rebuild["matchesCells"], true, "{rebuild}"); From 875dc2c295dacf07766bc0b17ebb14b8a8daf708 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:43:49 -0500 Subject: [PATCH 27/40] feat(cli)!: contracts under hops service, real catalog by default `distributed contracts check|accept` was missing from ServiceCommands, so `hops service contracts` did not exist; a test now asserts the hosted command set equals the standalone one. Its --catalog default was contracts/catalog.json, a file that does not exist: make contracts-check failed and the PR gate silently skipped. Both now use distributed.contracts.json (the catalog `distributed build` reads), and the gate always runs. Root README: feature table adds metrics, otel, workers-rs, application-runtime/runtime and the gateway features; the Locking row no longer claims Redis; Project Structure lists every src module and the distributed_game*, distributed_cli, examples and e2e-celld trees. distributed_cli README documents contracts; js README lists ./replica/lazy; distributed_game's Cargo comment names celld_worker. BREAKING CHANGE: `distributed contracts check` defaults to --catalog distributed.contracts.json. [[tasks/gr-p40-examples-e2e-coverage]] --- .github/workflows/on-pr-quality.yaml | 7 +-- Makefile | 2 +- README.md | 89 ++++++++++++++++++++-------- distributed_cli/README.md | 23 +++++++ distributed_cli/src/cli.rs | 27 ++++++++- distributed_game/Cargo.toml | 2 +- js/README.md | 3 + tests/e2e-ui/Makefile | 8 +-- 8 files changed, 121 insertions(+), 40 deletions(-) diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index ecc421b3f..3a67d2e62 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -25,12 +25,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable - name: Run contracts check - run: | - if [ -f contracts/catalog.json ]; then - cargo run -p distributed_cli --quiet -- contracts check --root . --catalog contracts/catalog.json --output json - else - echo "No contracts/catalog.json yet; gate is present and ready for producers." - fi + run: cargo run -p distributed_cli --quiet -- contracts check --root . --catalog distributed.contracts.json --output json # Framework monorepo keeps its existing quality provider. The reusable # quality.yaml in this directory is the contract for *consumers* (domain diff --git a/Makefile b/Makefile index 947bd294d..1bf424707 100644 --- a/Makefile +++ b/Makefile @@ -55,4 +55,4 @@ compose-down: ## Read-only aggregate contract lifecycle check (never writes tracked files). contracts-check: - $(CARGO) run -p distributed_cli --quiet -- contracts check --root . --catalog contracts/catalog.json --output human + $(CARGO) run -p distributed_cli --quiet -- contracts check --root . --catalog distributed.contracts.json --output human diff --git a/README.md b/README.md index bbd336ac0..713fa5cd2 100644 --- a/README.md +++ b/README.md @@ -737,6 +737,16 @@ network servers. | `kafka` | No | `KafkaBus` (Kafka source/publisher). | | `redis` | No | `leaderboard::RedisLeaderboard` (sorted-set leaderboards) and `bus::RedisStreamPublisher` (Redis Streams `XADD`, id dedupe); implies `leaderboard-background`. The `leaderboard` module and `InMemoryLeaderboard` are always available. | | `leaderboard-background` | No | `leaderboard::BackgroundLeaderboard` (fire-and-forget updates on a Tokio task) over any `LeaderboardStore`, e.g. `InMemoryLeaderboard`, without pulling in Redis. | +| `metrics` | No | `distributed::metrics`: framework counters and gauges with Prometheus text exposition (no SDK dependency); with `http`, a scrape handler. Bus receive loops record into it. | +| `otel` | No | OpenTelemetry tracing spans for bus receive loops and `microsvc` dispatch/handlers (`opentelemetry`, `tracing-opentelemetry`). | +| `workers-rs` | No | Cloudflare `worker` crate bindings for celld cells: the aggregate cell host and the celld Queue publisher/relay (`bus::CelldQueue*`) on `wasm32`. | +| `application-runtime` (alias `runtime`) | No | Checked by `#[command]`/`portable_command!` output in the **declaring** crate: enable it in that crate's features to emit the runtime mounts (`*_mount`, `register_*`, `CommandDefinition`). Contract-only packages leave it off. | +| `gateway` | No | `distributed::gateway`: portable gateway declarations (bindings, routes, dispatch). Starts no listeners. | +| `gateway-native` | No | Native HTTP gateway adapter (hyper/tower), independent of GraphQL and SQL. | +| `gateway-graphql` | No | Whole-operation GraphQL selection for the gateway (parser only). | +| `gateway-delivery` | No | Portable identity, freshness and delivery contracts (snapshots, coalescing, live sharing). | +| `gateway-graphql-native` | No | Native GraphQL execution behind the gateway, including WebSocket `@live` (implies `gateway-native`, `gateway-graphql`, `graphql`). | +| `gateway-worker` | No | workers-rs gateway ingress and delivery contracts, without native executors. | > The `InMemoryBus`, `PostgresBus`, and `SqliteBus` need no separate broker > feature. SQL-backed bus support comes from the matching `postgres` or `sqlite` @@ -816,7 +826,7 @@ Every infrastructure concern in `distributed` follows the same pattern: a **trai | Read model rows | `ReadModelWritePlanStore` + `RelationalReadModelQueryStore` | `InMemoryReadModelStore` | Postgres, SQLite | | Snapshot store | `SnapshotStore` | `InMemorySnapshotStore` | Postgres, SQLite, … | | Outbox publishing | `OutboxStore` + async `MessagePublisher` | `InMemoryRepository` outbox store (dev/test) | Any `MessagePublisher` (e.g. `BusPublisher` over a real `Bus`) | -| Locking | `Lock` + `LockManager` | `InMemoryLockManager` | `PostgresLockManager`, `SqliteLockManager` (durable leases), Redis, … | +| Locking | `Lock` + `LockManager` | `InMemoryLockManager` | `PostgresLockManager`, `SqliteLockManager` (durable leases); implement the traits for anything else | All in-memory defaults are `Clone` and `Send + Sync`, so they work in single-task tests and multi-task servers alike. When you're ready for production, implement the trait for your infrastructure and plug it in — handler code does not change. @@ -2780,30 +2790,59 @@ reference: [`distributed_cli/README.md`](distributed_cli/README.md). ```text src/ - aggregate/ # Aggregate trait, hydration, async aggregate repository helpers - bus/ # Bus facade + adapters (in-memory, sqlite, postgres, nats, rabbitmq, kafka, knative) - commit_builder/ # Transactional batches for aggregates, outbox, and read models - emitter/ # In-process event emitter helpers (feature = "emitter") - entity/ # Entity, event records, metadata, upcasting codecs - graphql/ # Query service: engine, permissions, identity, SDL, HTTP/WS (feature = "graphql") - in_memory_repo/ # In-memory repository (implements every async trait) - lock/ # Lock + lock manager traits, in-memory locks - microsvc/ # Command/event handler framework: service, context, session - outbox/ # Durable outbox message + commit extension - outbox_worker/ # Outbox claiming, publishing, workers - postgres_repo/ # Postgres async SQL repository (feature = "postgres") - queued_repo/ # Queue-based locking repository wrapper - read_model/ # Read model store traits, in-memory store, schema metadata - snapshot/ # Snapshot store traits, in-memory store, snapshot repository - sqlite_repo/ # SQLite async SQL repository (feature = "sqlite") - table/ # Neutral table/row primitives shared by read models and ops tables - lib.rs # Public exports -distributed_macros/ - src/ # Proc macros: sourced, digest, aggregate, enqueue, ReadModel, Snapshot -js/ # @hops-ops/distributed JS/TS client, command runtime, and SvelteKit adapter -tests/e2e-ui/ # Full-stack CQRS + GraphQL + SvelteKit template (nested workspace) -migrations/ # Explicit SQLite and Postgres migrations -compose.yaml # Local postgres / rabbitmq / kafka / nats for integration tests + aggregate/ # Aggregate trait, hydration, async aggregate repository helpers + application/ # Placement-independent application composition (modules, mounts, plans) + bus/ # Bus facade + adapters (in-memory, sqlite, postgres, nats, rabbitmq, + # kafka, knative, redis streams, celld Queue relay) + clock.rs # Injectable wall clock for coordination primitives + command/ # Typed commands: Eventual/Atomic/Succeeded, prepared completions, client effects + command_dispatch/ # CommandHost: local, HTTP and remote dispatch boundary + command_ledger/ # Durable command identity and fenced commit primitives (crate-private) + commit_builder/ # Transactional batches for aggregates, outbox, and read models + domain_event/ # Typed outward domain events, separate from replay bytes + emitter/ # In-process event emitter helpers (feature = "emitter") + entity/ # Entity, event records, metadata, upcasting codecs + gateway/ # Portable application gateway contracts (feature = "gateway*") + graphql/ # Query service: engine, permissions, identity, SDL, HTTP/WS (feature = "graphql") + in_memory_repo/ # In-memory repository (implements every async trait) + leaderboard/ # Sorted-set leaderboards as a rebuildable projection target (memory, Redis) + lease/ # TTL leases with monotonic fencing tokens + lock/ # Lock + lock manager traits, in-memory locks + metrics/ # Prometheus-compatible framework metrics (feature = "metrics") + microsvc/ # Command/event handler framework: service, context, session, cell_host (celld) + mutation/ # Event-independent read-model mutation IR and interpreters + outbox/ # Durable outbox message + commit extension + outbox_worker/ # Outbox claiming, publishing, workers + postgres_repo/ # Postgres async SQL repository (feature = "postgres") + projection/ # Deterministic domain-event projection programs and runtime + projection_protocol/ # Projection identities, ordering vocabulary, codecs (+ projection_protocol.rs) + queued_repo/ # Queue-based locking repository wrapper + read_model/ # Read model store traits, in-memory store, schema metadata, keyset scans + repository/ # Repository traits (GetStream, TransactionalCommit, …) + saga/ # Event-sourced process managers for cross-cell workflows (incl. fan-out) + schedule.rs # Portable aggregate schedules, host clock, backend capabilities + snapshot/ # Snapshot store traits, in-memory store, snapshot repository + sqlite_repo/ # SQLite async SQL repository (feature = "sqlite") + sqlx_repo/ # SQL code shared by the Postgres and SQLite repositories + table/ # Neutral table/row primitives shared by read models and ops tables + telemetry.rs # Telemetry vocabulary shared by metrics and tracing + time.rs # Wall clock that also works on wasm32 + trace_context.rs # W3C trace-context metadata helpers + lib.rs # Public exports +distributed_macros/ # Proc macros: sourced, digest, aggregate, enqueue, command, portable_command, + # module, application, mutation!, ReadModel, Snapshot, DomainState, + # DomainEvent, CommandInput, CommandOutput (projection! is in src/lib.rs) +distributed_cli/ # The `distributed` CLI (build/dev lifecycle, client codegen, contracts + # check/accept, game-manifest/game-client/game-wasm); also a library +distributed_game/ # Authoritative multiplayer zone runtime: fixed tick, facts, replication, + # WebSocket net, celld cells (zone lease/state, fact sink) +distributed_game_macros/ # Derives for distributed_game: Replicated, GameInput, GameEvent, … +js/ # @hops-ops/distributed JS/TS client, command runtime, SvelteKit adapter, game SDK +examples/ # graphiql.rs, and examples/game (game demos + SvelteKit Mob Pit; own workspace) +tests/e2e-ui/ # Full-stack CQRS + GraphQL + SvelteKit template (nested workspace) +tests/e2e-celld/ # The same UI with commands on celld cells and NATS (nested workspace) +migrations/ # Explicit SQLite and Postgres migrations +compose.yaml # Local postgres / rabbitmq / kafka / nats / redis for integration tests ``` ## Running Tests diff --git a/distributed_cli/README.md b/distributed_cli/README.md index f53a96995..9b75010b0 100644 --- a/distributed_cli/README.md +++ b/distributed_cli/README.md @@ -333,6 +333,29 @@ keep `wasm/` out of version control and run `game-wasm` before bundling. Unlike browser WASM pures it needs no wasm-pack, wasm-bindgen glue or JSON bridge; the SDK instantiates the module directly over a binary ABI. +## `distributed contracts check` / `accept` — the contract catalog + +```bash +distributed contracts check # read-only; never writes tracked files +distributed contracts check --output json +distributed contracts accept --scope --staged staged.json +``` + +`check` loads the catalog (default `distributed.contracts.json` under +`--root`, the current directory by default) and verifies that every entry's +recorded identity still matches its sources (for this repository, the +migration inventory). It exits non-zero on drift and +prints what changed. `make contracts-check` and the PR quality workflow run it +on this repository. + +`accept` takes one exact scope (no wildcards: `catalog`, +`migration_inventory`, `surface_client_manifest`, `generated_client_tree`, +`application_manifest`, `deployment_plan`, or `program:`) and a staged JSON object mapping portable relative paths to UTF-8 contents. +It writes those files through a staging directory with an atomic replace and +rolls back on failure. An unchanged payload is a no-op. + +`distributed build` reads the same catalog. + ## `distributed describe` — manifest as JSON ```bash diff --git a/distributed_cli/src/cli.rs b/distributed_cli/src/cli.rs index fff086802..33d8ee402 100644 --- a/distributed_cli/src/cli.rs +++ b/distributed_cli/src/cli.rs @@ -119,6 +119,8 @@ pub enum ServiceCommands { Build(BuildArgs), /// Run one coherent local application supervisor Dev(DevArgs), + /// Aggregate contract lifecycle check and accept + Contracts(ContractsArgs), /// Scaffold a new Distributed microservice crate #[command(alias = "create")] Scaffold(ScaffoldArgs), @@ -222,7 +224,7 @@ pub struct ContractsCheckArgs { #[arg(long, default_value = ".")] pub root: PathBuf, /// Path to the contract catalog JSON (relative to root or absolute) - #[arg(long, default_value = "contracts/catalog.json")] + #[arg(long, default_value = "distributed.contracts.json")] pub catalog: PathBuf, /// Output format #[arg(long, value_enum, default_value = "human")] @@ -762,6 +764,7 @@ pub fn run(args: &ServiceArgs) -> Result<(), Box> { match &args.command { ServiceCommands::Build(build) => run_build(build, &["service"]), ServiceCommands::Dev(dev) => run_dev(dev, &["service"]), + ServiceCommands::Contracts(contracts) => run_contracts(contracts), ServiceCommands::Scaffold(scaffold) => run_scaffold(scaffold), ServiceCommands::Describe(describe) => run_describe(describe), ServiceCommands::Client(client) => run_client(client), @@ -3226,4 +3229,26 @@ mod tests { both.db_url = Some("postgres://localhost/orders".to_string()); assert!(atlas_spec_from_flags(&both, "sql".into()).is_err()); } + + #[test] + fn hosted_service_commands_cover_every_public_distributed_command() { + use clap::Subcommand as _; + let names = |command: clap::Command| -> Vec { + command + .get_subcommands() + .filter(|sub| !sub.is_hide_set()) + .map(|sub| sub.get_name().to_string()) + .collect() + }; + let standalone = names(DistributedCommands::augment_subcommands( + clap::Command::new("distributed"), + )); + let hosted = names(ServiceCommands::augment_subcommands(clap::Command::new( + "service", + ))); + assert_eq!( + standalone, hosted, + "`hops service` (ServiceArgs) must expose every public `distributed` command" + ); + } } diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index d71f92a8b..9116199d7 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -23,7 +23,7 @@ command-host = ["cell", "distributed/graphql"] # celld clients: `CelldFactSink` (the command sink over a remote-only # `CelldCommandHost`), `CellZoneOwnership`, `CellZoneStateStore`. celld = ["cell", "command-host", "dep:reqwest", "dep:uuid"] -# celld worker glue (wasm32 Durable Objects): `world::zone_state::worker`, +# celld worker glue (wasm32 Durable Objects): `world::zone_state::celld_worker`, # the zone-state HTTP surface over a cell and an R2 bucket binding. workers-rs = ["cell", "dep:worker", "distributed/workers-rs"] # Per-zone Prometheus metrics (dependency-free text exposition). diff --git a/js/README.md b/js/README.md index 96938d282..371f9d8bb 100644 --- a/js/README.md +++ b/js/README.md @@ -887,6 +887,9 @@ duplicated as decision documents in this package. primitives. - `@hops-ops/distributed/replica` — replica, GraphQL transport, generated command runtime, query-plan helpers, and optional persistence. +- `@hops-ops/distributed/replica/lazy` — `createLazyReplicaCommandRuntime` + only, so a client that loads commands on demand (the generated lazy + `commands.ts`) keeps eager command exports out of its first import graph. - `@hops-ops/distributed/diagnostics` — redacted support snapshots and artifact inspection. - `@hops-ops/distributed/sveltekit` — Svelte stores, island SSR composition, diff --git a/tests/e2e-ui/Makefile b/tests/e2e-ui/Makefile index 815b3654f..f7ef0ce18 100644 --- a/tests/e2e-ui/Makefile +++ b/tests/e2e-ui/Makefile @@ -272,13 +272,9 @@ ui-install: js-build ui-test: cd ui && $(NPM) test -## Aggregate read-only contract check (repo root catalog when present). +## Aggregate read-only contract check (the repository's distributed.contracts.json). contracts-check: - @if [ -f ../../contracts/catalog.json ] || [ -f contracts/catalog.json ]; then \ - $(MAKE) -C ../.. contracts-check; \ - else \ - echo "contracts-check: no catalog yet; skipping aggregate gate"; \ - fi + $(MAKE) -C ../.. contracts-check check: application-build cargo check --workspace From 2597265d0a8503ad63c223bac54e76d24d9c0d60 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 16:44:16 -0500 Subject: [PATCH 28/40] chore(e2e-ui): remove the orphaned gitops tree and unused scripts tests/e2e-ui/gitops/ (cluster providers, Auth/PSQL stacks, env apps) was superseded by .gitops/local and referenced nowhere. The contracts/deployment-plans/*.json fixtures and the dual-worktree suite (plus its HMR helper) were also unreferenced and never run. [[tasks/gr-p40-examples-e2e-coverage]] --- tests/e2e-ui/.gitignore | 1 - .../contracts/deployment-plans/api-only.json | 11 - .../contracts/deployment-plans/full.json | 11 - .../contracts/deployment-plans/mixed.json | 19 -- .../deployment-plans/projector-todo-chat.json | 19 -- .../contracts/deployment-plans/split.json | 29 --- .../deployment-plans/writer-todo.json | 15 -- tests/e2e-ui/gitops/README.md | 26 -- tests/e2e-ui/gitops/cluster/README.md | 80 ------ tests/e2e-ui/gitops/cluster/SECRETS.md | 161 ------------ tests/e2e-ui/gitops/cluster/auth/README.md | 57 ----- .../auth/external-secret-masterkey.yaml | 27 -- .../cluster/configurations/auth-stack.yaml | 13 - .../cluster/configurations/psql-stack.yaml | 12 - .../cluster/configurations/secret-stack.yaml | 11 - .../gitops/cluster/providerconfigs/helm.yaml | 8 - .../cluster/providerconfigs/kubernetes.yaml | 8 - .../cluster/providerconfigs/zitadel.yaml | 26 -- .../gitops/cluster/providers/helm-drc.yaml | 28 --- .../e2e-ui/gitops/cluster/providers/helm.yaml | 9 - .../cluster/providers/kubernetes-drc.yaml | 28 --- .../gitops/cluster/providers/kubernetes.yaml | 9 - .../gitops/cluster/providers/zitadel.yaml | 6 - .../e2e-ui/gitops/cluster/secrets/stack.yaml | 148 ----------- .../cluster/secrets/vault-auth-delegator.yaml | 17 -- tests/e2e-ui/gitops/cluster/stacks/auth.yaml | 50 ---- tests/e2e-ui/gitops/cluster/stacks/psql.yaml | 25 -- tests/e2e-ui/gitops/envs/local/api.yaml | 26 -- tests/e2e-ui/gitops/envs/local/ui.yaml | 31 --- tests/e2e-ui/scripts/dual-worktree-hmr.mjs | 238 ------------------ tests/e2e-ui/scripts/dual-worktree-suite.sh | 184 -------------- 31 files changed, 1333 deletions(-) delete mode 100644 tests/e2e-ui/contracts/deployment-plans/api-only.json delete mode 100644 tests/e2e-ui/contracts/deployment-plans/full.json delete mode 100644 tests/e2e-ui/contracts/deployment-plans/mixed.json delete mode 100644 tests/e2e-ui/contracts/deployment-plans/projector-todo-chat.json delete mode 100644 tests/e2e-ui/contracts/deployment-plans/split.json delete mode 100644 tests/e2e-ui/contracts/deployment-plans/writer-todo.json delete mode 100644 tests/e2e-ui/gitops/README.md delete mode 100644 tests/e2e-ui/gitops/cluster/README.md delete mode 100644 tests/e2e-ui/gitops/cluster/SECRETS.md delete mode 100644 tests/e2e-ui/gitops/cluster/auth/README.md delete mode 100644 tests/e2e-ui/gitops/cluster/auth/external-secret-masterkey.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/configurations/auth-stack.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/configurations/psql-stack.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/configurations/secret-stack.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providerconfigs/helm.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providerconfigs/kubernetes.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providerconfigs/zitadel.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providers/helm-drc.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providers/helm.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providers/kubernetes-drc.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providers/kubernetes.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/providers/zitadel.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/secrets/stack.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/secrets/vault-auth-delegator.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/stacks/auth.yaml delete mode 100644 tests/e2e-ui/gitops/cluster/stacks/psql.yaml delete mode 100644 tests/e2e-ui/gitops/envs/local/api.yaml delete mode 100644 tests/e2e-ui/gitops/envs/local/ui.yaml delete mode 100644 tests/e2e-ui/scripts/dual-worktree-hmr.mjs delete mode 100755 tests/e2e-ui/scripts/dual-worktree-suite.sh diff --git a/tests/e2e-ui/.gitignore b/tests/e2e-ui/.gitignore index c5789bfbe..04cf40f93 100644 --- a/tests/e2e-ui/.gitignore +++ b/tests/e2e-ui/.gitignore @@ -16,4 +16,3 @@ e2e/.auth playwright-report test-results blob-report -gitops/cluster/e2e-identity/grants.yaml diff --git a/tests/e2e-ui/contracts/deployment-plans/api-only.json b/tests/e2e-ui/contracts/deployment-plans/api-only.json deleted file mode 100644 index 7aadc5d1c..000000000 --- a/tests/e2e-ui/contracts/deployment-plans/api-only.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "e2e-ui-api", - "application": "e2e-ui", - "processes": [ - { - "id": "api", - "preset": "query_api", - "remote_commands": true - } - ] -} diff --git a/tests/e2e-ui/contracts/deployment-plans/full.json b/tests/e2e-ui/contracts/deployment-plans/full.json deleted file mode 100644 index cb3134d01..000000000 --- a/tests/e2e-ui/contracts/deployment-plans/full.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "e2e-ui-full", - "application": "e2e-ui", - "processes": [ - { - "id": "full", - "preset": "full", - "remote_commands": false - } - ] -} diff --git a/tests/e2e-ui/contracts/deployment-plans/mixed.json b/tests/e2e-ui/contracts/deployment-plans/mixed.json deleted file mode 100644 index 325ac87ab..000000000 --- a/tests/e2e-ui/contracts/deployment-plans/mixed.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "e2e-ui-mixed", - "application": "e2e-ui", - "processes": [ - { - "id": "mixed", - "mounts": [ - { - "kind": "command", - "id": "todo.create" - }, - { - "kind": "surface", - "id": "e2e-ui" - } - ] - } - ] -} diff --git a/tests/e2e-ui/contracts/deployment-plans/projector-todo-chat.json b/tests/e2e-ui/contracts/deployment-plans/projector-todo-chat.json deleted file mode 100644 index 0f6b1e0e2..000000000 --- a/tests/e2e-ui/contracts/deployment-plans/projector-todo-chat.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "e2e-ui-projectors", - "application": "e2e-ui", - "processes": [ - { - "id": "projectors", - "mounts": [ - { - "kind": "projector", - "id": "project_todos" - }, - { - "kind": "projector", - "id": "project_chat" - } - ] - } - ] -} diff --git a/tests/e2e-ui/contracts/deployment-plans/split.json b/tests/e2e-ui/contracts/deployment-plans/split.json deleted file mode 100644 index e6f6cdb1a..000000000 --- a/tests/e2e-ui/contracts/deployment-plans/split.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "e2e-ui-split", - "application": "e2e-ui", - "processes": [ - { - "id": "api", - "preset": "query_api", - "remote_commands": true - }, - { - "id": "todo-writer", - "mounts": [ - { - "kind": "command", - "id": "todo.create" - } - ] - }, - { - "id": "projectors", - "mounts": [ - { - "kind": "projector", - "id": "project_todos" - } - ] - } - ] -} diff --git a/tests/e2e-ui/contracts/deployment-plans/writer-todo.json b/tests/e2e-ui/contracts/deployment-plans/writer-todo.json deleted file mode 100644 index b7bd2bde4..000000000 --- a/tests/e2e-ui/contracts/deployment-plans/writer-todo.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "e2e-ui-writer-todo", - "application": "e2e-ui", - "processes": [ - { - "id": "todo-writer", - "mounts": [ - { - "kind": "command", - "id": "todo.create" - } - ] - } - ] -} diff --git a/tests/e2e-ui/gitops/README.md b/tests/e2e-ui/gitops/README.md deleted file mode 100644 index 45cf822d5..000000000 --- a/tests/e2e-ui/gitops/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# e2e-ui local gitops - -```text -gitops/ - cluster/ # fixture CP tree (real meta repos: put this at meta root) - envs/ - local/ # app Applications → worktree namespaces -``` - -```bash -# Terminal 1: cluster apply/watch -hops local gitops cluster ./gitops/cluster \ - --cluster-provider kind --docker-provider dory \ - --cluster-name hops --context kind-hops - -# Terminal 2: per-workspace apps apply/watch -hops local gitops worktree ./gitops/envs/local --name alice \ - --cluster-provider kind --docker-provider dory \ - --cluster-name hops --context kind-hops - -# One-shot (CI / scripts): add --once -# hops local gitops cluster ./gitops/cluster --once -``` - -On multi-project metas: put `gitops/cluster` at the **meta root**; projects keep -`.gitops/deploy` charts; env Application YAMLs point at those charts. diff --git a/tests/e2e-ui/gitops/cluster/README.md b/tests/e2e-ui/gitops/cluster/README.md deleted file mode 100644 index 26df8c24c..000000000 --- a/tests/e2e-ui/gitops/cluster/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# `gitops/cluster` — local control-plane configuration - -> **Real meta repos:** put this tree at the **meta root** (`gitops/cluster/`), -> not inside a single project. One local CP (dory) serves every worktree; apps -> only get different **namespaces**. This e2e-ui copy is a self-contained -> fixture — same files, nested under the test app. Point hops with -> `hops local gitops cluster ./gitops/cluster` or `HOPS_LOCAL_CLUSTER` when -> applying a meta-root cluster tree. - -Platform / CP resources for the **shared** local control plane: - -```text -/ - gitops/cluster/ # ← this tree (AuthStack, PSQLStack, configurations) - gitops/envs/local/ # Application YAMLs → namespace = --name - clients/foo/.gitops/deploy/ - platform/api/.gitops/deploy/ -``` - -| Path | Contents | -|------|----------| -| `configurations/` | Crossplane `Configuration` installs (psql-stack, auth-stack, secret-stack) | -| `stacks/psql.yaml` | **`PSQLStack` XR** — CNPG + SC `psql` (local-path) | -| `stacks/auth.yaml` | **`AuthStack` XR** — Zitadel + embedded Postgres | -| `secrets/` | **`SecretStack`** + vault-auth-delegator CRB (ESO + Vault; k8s auth via Helm postStart) | -| `auth/` | Auth residuals (e.g. masterkey ExternalSecret), not the XR claim | -| `providers/` | Provider installs + per-provider DRCs (`helm.yaml` / `helm-drc.yaml`, …) | -| `providerconfigs/` | ProviderConfig shapes (`secretRef` only) | -| `e2e-identity/` | **Moved** — worktree UI chart `identity.*` values (pointer README only) | - -## Apply + watch (local gitops vibes) - -This fixture pins the auth stack's content-derived local dev image because its -Grant reference support is newer than the published package. Build and push -that package once before the cluster watcher. Re-run the source install and -update `configurations/auth-stack.yaml` when the auth package content changes. - -```bash -# One-time package build/push for the pin in configurations/auth-stack.yaml -hops config install --path /xrs/stacks/k8s/auth \ - --cluster-provider kind --docker-provider dory \ - --cluster-name hops --context kind-hops - -# Terminal 1: shared control-plane tree (apply + watch) -hops local gitops cluster ./gitops/cluster \ - --cluster-provider kind --docker-provider dory \ - --cluster-name hops --context kind-hops - -# Terminal 2: per-workspace Applications + charts (apply + watch) -hops local gitops worktree ./gitops/envs/local --name alice \ - --cluster-provider kind --docker-provider dory \ - --cluster-name hops --context kind-hops -``` - -Crossplane reconciles XRs (`PSQLStack`, `AuthStack`, `SecretStack`, …) after each apply. -Edit `stacks/auth.yaml`, `stacks/psql.yaml`, or `secrets/stack.yaml` → saved → applied → CP converges. - -**SecretStack (local):** install the Configuration from source first -(`hops config install --path …/xrs/stacks/aws/secret`), then apply this cluster tree -so `secrets/stack.yaml` is not stuck Unpacking. - -## vs `gitops/envs/` - -| Tree | Purpose | -|------|---------| -| **cluster/** | CP packages + platform XRs (PSQLStack, AuthStack) | -| **envs/\-web` — redirects + Login V2 baseUri) | - -| Secret | Purpose | -|--------|---------| -| `e2e-human-passwords` | Keys `alice`, `bob`, `admin` (cluster-shared; local chart seeds when `local: true`) | -| `zitadel-credentials` | ProviderConfig / ClusterProviderConfig (`hops local zitadel`) | -| Oidc connection Secret | Oidc MR `writeConnectionSecretToRef` → `attribute.client_id` / `attribute.client_secret` (UI mounts these) | -| `e2e-ui-oidc` | `AUTH_SECRET` (+ optional `ZITADEL_SERVICE_USER_TOKEN` from AuthStack `auth/login-client`) | - -```bash -# Humans (if not using chart local seed): -kubectl -n default create secret generic e2e-human-passwords \ - --from-literal=alice=Password1! \ - --from-literal=bob=Password1! \ - --from-literal=admin=Password1! - -# Login V2 service user (after AuthStack Ready): -kubectl -n create secret generic e2e-ui-oidc \ - --from-literal=AUTH_SECRET='local-workbench-dev-auth-secret-not-for-prod' \ - --from-literal=ZITADEL_SERVICE_USER_TOKEN="$(kubectl -n auth get secret login-client -o jsonpath='{.data.pat}' | base64 -d)" \ - --dry-run=client -o yaml | kubectl apply -f - -# OIDC_CLIENT_ID/SECRET: do not hand-copy — Deployment reads Oidc connection Secret. -``` - -Demo login (every env): **alice / bob / admin · Password1!** - -Do **not** commit live `orgId`, Project external ids, or OIDC client ids into Application values. MRs use `projectIdRef` and ProviderConfig org default. -## e2e-ui app (`ns: --name`) - -| Secret | Keys | Used by | -|--------|------|---------| -| `e2e-ui-oidc` | `AUTH_SECRET`, `ZITADEL_SERVICE_USER_TOKEN` | UI session/login residuals | -| `e2e-ui--oidc-conn` | `attribute.client_id`, `attribute.client_secret` | Oidc MR connection secret; UI client env | - -```bash -# Login-client PAT: AuthStack residual secret auth/login-client key pat -kubectl -n dogfood create secret generic e2e-ui-oidc \ - --from-literal=AUTH_SECRET='local-workbench-dogfood-auth-secret-not-for-prod' \ - --from-literal=ZITADEL_SERVICE_USER_TOKEN="$(kubectl -n auth get secret login-client -o jsonpath='{.data.pat}' | base64 -d)" -``` - -Do not copy OIDC client IDs or secrets into this Secret. The Oidc managed -resource writes them to `e2e-ui--oidc-conn`, and the Deployment -references `attribute.client_id` / `attribute.client_secret` there. - -Non-secret OIDC config (issuer, AUTH_URL) stays in `ui/.gitops/deploy/values.yaml`. - -### Login V2 (required for browser sign-in) - -Gitops (UI chart when `identity.enabled`): - -1. **Oidc** app MR — redirects + `loginVersion.loginV2.baseUri` from release ns -2. **Features** MR (`instance.zitadel…/Features`) — instance `loginV2.required` - + `baseUri` so authorize redirects to this UI’s `/login` - -Both use `http://e2e-ui-ui..svc.cluster.local:5180`. Instance Features -is global per AuthStack (last applied worktree wins). Do not hand-`PUT` -`/v2/features/instance` — re-reconcile the UI chart instead. - -Prefer app-level baseUri over instance-wide -defaults so multiple worktrees can coexist. - -## SecretStack / ESO + Vault (`secrets/`) - -Local fixture uses **`backend: vault`** with **`vault.install: true`** (in-cluster -Vault Helm + ESO). No AWS PodIdentity on dory. - -| Resource | Purpose | -|----------|---------| -| `secrets/stack.yaml` | SecretStack XR — ESO + Vault Helm + ClusterSecretStore | -| `secrets/vault-auth-delegator.yaml` | CRB: Vault SA → `system:auth-delegator` (TokenReview) | -| Helm `vault` `server.postStart` | Init/unseal (once), KV mount, k8s auth + ESO role | -| PVC (`dataStorage` 1Gi) | File storage under `/vault/data` (survives pod restarts) | -| ClusterSecretStore `vault` | ESO → Vault (Ready after postStart + CRB) | - -### Persistence - -`server.dev` is **off**. Vault runs standalone with `storage "file"` on a PVC -(default StorageClass, dory: `local-path`). First start initializes and writes: - -```text -/vault/data/.hops-init # Unseal Key 1 + Initial Root Token (chmod 600) -``` - -Later starts: unseal from that file, re-apply k8s auth (idempotent). **KV data -survives pod restarts**; deleting the PVC is a full reset. - -Root token is **not** the fixed string `root` (that was dev-only). For CLI writes: - -```bash -export VAULT_TOKEN="$( - kubectl -n vault exec vault-0 -- \ - awk '/^Initial Root Token:/{print $NF}' /vault/data/.hops-init -)" -export VAULT_ADDR=http://127.0.0.1:8200 -# kubectl port-forward -n vault svc/vault 8200:8200 # if needed -hops secrets sync vault -y -``` - -### Declarative trust path - -1. Apply cluster gitops (SecretStack + CRB). -2. postStart: init (once) → unseal → `secret/` KV v2 → k8s auth for ESO. -3. ClusterSecretStore `vault` Ready. -4. **Write** with `hops secrets sync vault`. -5. **Read** via ExternalSecrets (auth masterkey, UI oidc/humans). - -Install package from source before first apply: - -```bash -hops config install --path /xrs/stacks/aws/secret --context dory -hops local gitops cluster ./gitops/cluster -``` - -Switching an existing **dev/inmem** Vault release to file storage may need a -one-time recreate of the Helm release / PVC (data was ephemeral anyway). - -Do **not** put the SecretStack claim under meta `local/` — that folder is for -ad-hoc colima-style one-off claims, not the workbench cluster gitops path. - -## Rules - -1. **Never** put passwords, PATs, client secrets, or masterkeys in committed YAML. -2. Gitops may name Secrets and keys (`secretRef` / `secretEnv`). -3. Create Secrets before the XRs/Deployments that need them (or accept CrashLoop until present). -4. Cloud: prefer ESO / SOPS; local: `kubectl create` or `hops local zitadel|aws|github`. -5. Platform secrets engine for local CP lives in **`gitops/cluster/secrets/`**, not `local/`. diff --git a/tests/e2e-ui/gitops/cluster/auth/README.md b/tests/e2e-ui/gitops/cluster/auth/README.md deleted file mode 100644 index 7da694c52..000000000 --- a/tests/e2e-ui/gitops/cluster/auth/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Auth (local) - -| File | Kind | -|------|------| -| [`../stacks/auth.yaml`](../stacks/auth.yaml) | `AuthStack` XR — Zitadel + embedded `PSQLCluster` | -| `external-secret-masterkey.yaml` | Optional ESO materialize of `zitadel-masterkey` from Vault | - -## Local addressing (no ingress) - -| | Value | -|--|--------| -| Install namespace | **`auth`** | -| AuthStack XR name | **`zitadel`** | -| Helm Service (default fullname) | **`zitadel-zitadel.auth.svc.cluster.local:8080`** | -| OIDC issuer / `domain` | `http://zitadel-zitadel.auth.svc.cluster.local:8080` | -| Gateway / ingress | disabled — ClusterIP only for now | - -## Apply order - -```bash -hops local start --cluster-provider kind --docker-provider dory \ - --cluster-name hops --gitops ./gitops/cluster -# or after configurations: -hops local gitops cluster ./gitops/cluster - -kubectl get authstack zitadel -n default -kubectl get svc -n auth -# expect: zitadel-zitadel (API) — not a separate short-name alias -``` - -## e2e-ui identity (worktree) - -App Project / humans / OIDC app render from the **UI** chart when -`gitops/envs/local/ui.yaml` sets `identity.enabled: true` (not cluster gitops). - -ProviderConfig residual (cluster): - -```bash -hops local zitadel --source-context dory --source-namespace auth \ - --domain zitadel-zitadel.auth.svc.cluster.local --port 8080 --insecure -``` - -## Secrets - -Masterkey is **not** inline — `secretRef` to `zitadel-masterkey` in ns `auth`: - -```bash -kubectl -n auth create secret generic zitadel-masterkey \ - --from-literal=masterkey='hops-like-basketball-but-for-ops' # exactly 32 bytes -``` - -Create **before** AuthStack becomes Ready (or re-apply after). Full table: -[../SECRETS.md](../SECRETS.md). - -## Residual - -- Host browser access without ingress is later (gateway / map / promote) diff --git a/tests/e2e-ui/gitops/cluster/auth/external-secret-masterkey.yaml b/tests/e2e-ui/gitops/cluster/auth/external-secret-masterkey.yaml deleted file mode 100644 index 3ed7a6996..000000000 --- a/tests/e2e-ui/gitops/cluster/auth/external-secret-masterkey.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Materialize AuthStack masterkey from Vault so it survives ns/worktree churn. -# Requires ClusterSecretStore vault Ready (Vault Helm postStart + vault-auth-delegator) -# and path auth/zitadel-masterkey written via: hops secrets sync vault -# See SECRETS.md. -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: zitadel-masterkey - namespace: auth - labels: - hops.ops.com.ai/fixture: e2e-ui - hops.ops.com.ai/secret: zitadel-masterkey -spec: - refreshInterval: 1m - secretStoreRef: - name: vault - kind: ClusterSecretStore - target: - name: zitadel-masterkey - creationPolicy: Owner - template: - type: Opaque - data: - - secretKey: masterkey - remoteRef: - key: auth/zitadel-masterkey - property: masterkey diff --git a/tests/e2e-ui/gitops/cluster/configurations/auth-stack.yaml b/tests/e2e-ui/gitops/cluster/configurations/auth-stack.yaml deleted file mode 100644 index 81732ea27..000000000 --- a/tests/e2e-ui/gitops/cluster/configurations/auth-stack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Crossplane Configuration for AuthStack (Zitadel + auth XRDs). -# This fixture exercises Grant references that are newer than the published -# package. Build/push this content-derived dev tag before starting cluster -# GitOps (see ../README.md). The watched pin then matches the source install -# instead of replacing it with an older package. -# Depends on: provider-helm, provider-kubernetes, (optional) provider-upjet-zitadel. -apiVersion: pkg.crossplane.io/v1 -kind: Configuration -metadata: - name: auth-stack -spec: - package: registry.crossplane-system.svc.cluster.local:5000/hops-ops/auth-stack:dev-15db94d94e7b - packagePullPolicy: Always diff --git a/tests/e2e-ui/gitops/cluster/configurations/psql-stack.yaml b/tests/e2e-ui/gitops/cluster/configurations/psql-stack.yaml deleted file mode 100644 index 2427526f0..000000000 --- a/tests/e2e-ui/gitops/cluster/configurations/psql-stack.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Crossplane Configuration for the PostgreSQL platform stack. -# Prefer building/pushing from source into the local registry: -# hops config install /xrs/stacks/k8s/psql -# Or set package to a published tag when available. -apiVersion: pkg.crossplane.io/v1 -kind: Configuration -metadata: - name: psql-stack -spec: - package: ghcr.io/hops-ops/psql-stack:v0.9.1 - # Local registry after hops local start + config install: - # package: registry.crossplane-system.svc.cluster.local:5000/hops-ops/psql-stack: diff --git a/tests/e2e-ui/gitops/cluster/configurations/secret-stack.yaml b/tests/e2e-ui/gitops/cluster/configurations/secret-stack.yaml deleted file mode 100644 index a69929cee..000000000 --- a/tests/e2e-ui/gitops/cluster/configurations/secret-stack.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Crossplane Configuration for platform SecretStack (ESO + AWS|Vault backends). -# Build and publish the source package to the control plane's local registry first: -# hops config install --path /xrs/stacks/aws/secret \ -# --cluster-provider kind --docker-provider dory \ -# --cluster-name hops --context kind-hops -apiVersion: pkg.crossplane.io/v1 -kind: Configuration -metadata: - name: secret-stack -spec: - package: registry.crossplane-system.svc.cluster.local:5000/hops-ops/secret-stack:dev-29d20a9a9aa7 diff --git a/tests/e2e-ui/gitops/cluster/providerconfigs/helm.yaml b/tests/e2e-ui/gitops/cluster/providerconfigs/helm.yaml deleted file mode 100644 index 148cb4fc9..000000000 --- a/tests/e2e-ui/gitops/cluster/providerconfigs/helm.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: helm.m.crossplane.io/v1beta1 -kind: ProviderConfig -metadata: - name: default - namespace: default -spec: - credentials: - source: InjectedIdentity diff --git a/tests/e2e-ui/gitops/cluster/providerconfigs/kubernetes.yaml b/tests/e2e-ui/gitops/cluster/providerconfigs/kubernetes.yaml deleted file mode 100644 index 95029bccc..000000000 --- a/tests/e2e-ui/gitops/cluster/providerconfigs/kubernetes.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kubernetes.m.crossplane.io/v1alpha1 -kind: ProviderConfig -metadata: - name: default - namespace: default -spec: - credentials: - source: InjectedIdentity diff --git a/tests/e2e-ui/gitops/cluster/providerconfigs/zitadel.yaml b/tests/e2e-ui/gitops/cluster/providerconfigs/zitadel.yaml deleted file mode 100644 index b601e2b93..000000000 --- a/tests/e2e-ui/gitops/cluster/providerconfigs/zitadel.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: zitadel.m.crossplane.io/v1beta1 -kind: ProviderConfig -metadata: - name: default - namespace: default -spec: - credentials: - source: Secret - secretRef: - namespace: default - name: zitadel-credentials - key: credentials ---- -# Namespaced identity MRs (project/humans) and worktree Oidc/Features use -# ClusterProviderConfig so all namespaces share one credential secret. -apiVersion: zitadel.m.crossplane.io/v1beta1 -kind: ClusterProviderConfig -metadata: - name: default -spec: - credentials: - source: Secret - secretRef: - namespace: default - name: zitadel-credentials - key: credentials diff --git a/tests/e2e-ui/gitops/cluster/providers/helm-drc.yaml b/tests/e2e-ui/gitops/cluster/providers/helm-drc.yaml deleted file mode 100644 index c9953f169..000000000 --- a/tests/e2e-ui/gitops/cluster/providers/helm-drc.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Per-provider DeploymentRuntimeConfig for provider-helm (local dev). -# Colocated with providers/helm.yaml (runtimeConfigRef: local-dev-helm). -# -# Each provider gets its OWN uniquely-named DRC + cluster-admin ServiceAccount. -# Providers must NOT share a DRC: a shared runtime config lets one provider's -# image/SA silently clobber the other's pod (and makes drift impossible to -# attribute). -apiVersion: pkg.crossplane.io/v1beta1 -kind: DeploymentRuntimeConfig -metadata: - name: local-dev-helm -spec: - serviceAccountTemplate: - metadata: - name: local-dev-helm ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: local-dev-helm-cluster-admin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: local-dev-helm - namespace: crossplane-system diff --git a/tests/e2e-ui/gitops/cluster/providers/helm.yaml b/tests/e2e-ui/gitops/cluster/providers/helm.yaml deleted file mode 100644 index 44d33b640..000000000 --- a/tests/e2e-ui/gitops/cluster/providers/helm.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: pkg.crossplane.io/v1 -kind: Provider -metadata: - name: crossplane-contrib-provider-helm -spec: - # Renovate: github-releases crossplane-contrib/provider-helm (cli/renovate.json) - package: xpkg.crossplane.io/crossplane-contrib/provider-helm:v1.3.0 - runtimeConfigRef: - name: local-dev-helm diff --git a/tests/e2e-ui/gitops/cluster/providers/kubernetes-drc.yaml b/tests/e2e-ui/gitops/cluster/providers/kubernetes-drc.yaml deleted file mode 100644 index 76f6f5a6e..000000000 --- a/tests/e2e-ui/gitops/cluster/providers/kubernetes-drc.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Per-provider DeploymentRuntimeConfig for provider-kubernetes (local dev). -# Colocated with providers/kubernetes.yaml (runtimeConfigRef: local-dev-kubernetes). -# -# Each provider gets its OWN uniquely-named DRC + cluster-admin ServiceAccount. -# Providers must NOT share a DRC: a shared runtime config lets one provider's -# image/SA silently clobber the other's pod (and makes drift impossible to -# attribute). -apiVersion: pkg.crossplane.io/v1beta1 -kind: DeploymentRuntimeConfig -metadata: - name: local-dev-kubernetes -spec: - serviceAccountTemplate: - metadata: - name: local-dev-kubernetes ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: local-dev-kubernetes-cluster-admin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: local-dev-kubernetes - namespace: crossplane-system diff --git a/tests/e2e-ui/gitops/cluster/providers/kubernetes.yaml b/tests/e2e-ui/gitops/cluster/providers/kubernetes.yaml deleted file mode 100644 index c400c3878..000000000 --- a/tests/e2e-ui/gitops/cluster/providers/kubernetes.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: pkg.crossplane.io/v1 -kind: Provider -metadata: - name: crossplane-contrib-provider-kubernetes -spec: - # Renovate: github-releases crossplane-contrib/provider-kubernetes (cli/renovate.json) - package: xpkg.crossplane.io/crossplane-contrib/provider-kubernetes:v1.2.1 - runtimeConfigRef: - name: local-dev-kubernetes diff --git a/tests/e2e-ui/gitops/cluster/providers/zitadel.yaml b/tests/e2e-ui/gitops/cluster/providers/zitadel.yaml deleted file mode 100644 index 41d45afdb..000000000 --- a/tests/e2e-ui/gitops/cluster/providers/zitadel.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: pkg.crossplane.io/v1 -kind: Provider -metadata: - name: crossplane-contrib-provider-upjet-zitadel -spec: - package: xpkg.crossplane.io/crossplane-contrib/provider-upjet-zitadel:v0.1.1 diff --git a/tests/e2e-ui/gitops/cluster/secrets/stack.yaml b/tests/e2e-ui/gitops/cluster/secrets/stack.yaml deleted file mode 100644 index 33870beab..000000000 --- a/tests/e2e-ui/gitops/cluster/secrets/stack.yaml +++ /dev/null @@ -1,148 +0,0 @@ -# Platform SecretStack for local CP (dory workbench). -# -# Install Configuration from source first (breaking API: hops.ops.com.ai): -# hops config install --path /xrs/stacks/aws/secret --context dory -# -# Then: hops local gitops cluster ./gitops/cluster -# -# backend=vault + vault.install for local (no AWS PodIdentity required). -# -# Persistence: file storage on a PVC (not server.dev / inmem). postStart -# init+unseal once, stores root token + unseal key on the volume: -# /vault/data/.hops-init -# K8s auth for ESO is re-applied every start. Seed KV with: -# export VAULT_TOKEN=$(kubectl -n vault exec vault-0 -- \ -# awk '/^Initial Root Token:/{print $NF}' /vault/data/.hops-init) -# hops secrets sync vault -apiVersion: hops.ops.com.ai/v1alpha1 -kind: SecretStack -metadata: - name: external-secrets - namespace: default -spec: - clusterName: dory - backend: vault - namespace: external-secrets - helmProviderConfigRef: - name: default - kubernetesProviderConfigRef: - name: default - labels: - team: platform - hops.ops.com.ai/fixture: e2e-ui - secretStore: - enabled: true - scope: Cluster - name: vault - vault: - install: true - namespace: vault - path: secret - version: v2 - auth: - method: kubernetes - mountPath: kubernetes - role: external-secrets - # Merged into official Vault Helm chart (SecretStack composition defaults - # use server.dev + no PVC; we override for durable local dogfood). - values: - server: - # Leave dev mode — inmem cannot use dataStorage. - dev: - enabled: false - standalone: - enabled: true - config: | - ui = true - disable_mlock = true - listener "tcp" { - tls_disable = 1 - address = "[::]:8200" - cluster_address = "[::]:8201" - } - storage "file" { - path = "/vault/data" - } - dataStorage: - enabled: true - size: 1Gi - # omit storageClass → cluster default (dory: local-path) - postStart: - - /bin/sh - - -c - - | - # NOTE: do not use bare `set -e` around `vault status` — sealed/uninit - # returns exit 2 and would abort the hook (killing the container). - set -u - export VAULT_ADDR=http://127.0.0.1:8200 - INIT_FILE=/vault/data/.hops-init - - vault_status_rc() { - vault status >/dev/null 2>&1 - echo $? - } - - # API is up when status returns 0 (unsealed) or 2 (sealed / not init). - i=0 - while [ "$i" -lt 90 ]; do - rc=$(vault_status_rc) - if [ "$rc" -eq 0 ] || [ "$rc" -eq 2 ]; then - break - fi - i=$((i + 1)) - sleep 1 - done - rc=$(vault_status_rc) - if [ "$rc" -ne 0 ] && [ "$rc" -ne 2 ]; then - echo "vault postStart: API not ready after wait (rc=$rc)" >&2 - exit 1 - fi - - if [ ! -f "$INIT_FILE" ]; then - echo "vault postStart: initializing (1 share / threshold 1)" - vault operator init -key-shares=1 -key-threshold=1 >"$INIT_FILE" - chmod 600 "$INIT_FILE" || true - fi - - UNSEAL_KEY=$(awk '/Unseal Key 1:/{print $NF}' "$INIT_FILE") - ROOT_TOKEN=$(awk '/Initial Root Token:/{print $NF}' "$INIT_FILE") - if [ -z "${UNSEAL_KEY}" ] || [ -z "${ROOT_TOKEN}" ]; then - echo "vault postStart: failed to parse $INIT_FILE" >&2 - cat "$INIT_FILE" >&2 || true - exit 1 - fi - - # Unseal when sealed (status exit 2). - rc=$(vault_status_rc) - if [ "$rc" -ne 0 ]; then - vault operator unseal "$UNSEAL_KEY" >/dev/null - fi - export VAULT_TOKEN="$ROOT_TOKEN" - - # Non-dev installs do not auto-mount secret/ — enable KV v2 for ESO. - if ! vault secrets list -format=json 2>/dev/null | grep -q '"secret/"'; then - vault secrets enable -path=secret kv-v2 - fi - - if ! vault auth list -format=json 2>/dev/null | grep -q '"kubernetes/"'; then - vault auth enable kubernetes - fi - - vault write auth/kubernetes/config \ - kubernetes_host="https://kubernetes.default.svc" \ - token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ - kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ - disable_iss_validation=true - - printf '%s\n' \ - 'path "secret/data/*" { capabilities = ["read", "list"] }' \ - 'path "secret/metadata/*" { capabilities = ["read", "list"] }' \ - | vault policy write external-secrets - - - vault write auth/kubernetes/role/external-secrets \ - bound_service_account_names=external-secrets \ - bound_service_account_namespaces=external-secrets \ - policies=external-secrets \ - ttl=24h - - echo "vault postStart: unsealed + kubernetes auth ready (init file $INIT_FILE)" diff --git a/tests/e2e-ui/gitops/cluster/secrets/vault-auth-delegator.yaml b/tests/e2e-ui/gitops/cluster/secrets/vault-auth-delegator.yaml deleted file mode 100644 index 7b9e772a6..000000000 --- a/tests/e2e-ui/gitops/cluster/secrets/vault-auth-delegator.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Allow the Vault server SA to call TokenReview (required for kubernetes auth). -# Independent of the Vault pod lifecycle — apply with cluster gitops. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: vault-auth-delegator - labels: - hops.ops.com.ai/fixture: e2e-ui - hops.ops.com.ai/component: vault -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: - - kind: ServiceAccount - name: vault - namespace: vault diff --git a/tests/e2e-ui/gitops/cluster/stacks/auth.yaml b/tests/e2e-ui/gitops/cluster/stacks/auth.yaml deleted file mode 100644 index ca70689db..000000000 --- a/tests/e2e-ui/gitops/cluster/stacks/auth.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Local AuthStack: Zitadel + embedded PSQLCluster (owned by this XR). -# -# No ingress/gateway for local first step — Service is ClusterIP only. -# Helm chart fullname is release+chart → Service **zitadel-zitadel** (default). -# Issuer / ExternalDomain must match that FQDN (Option A): -# http://zitadel-zitadel.auth.svc.cluster.local:8080 -# (Later: chartValues.fullnameOverride: zitadel if we want a shorter name.) -# -# Shared cluster GitOps owns this XR after Configuration bootstrap. -apiVersion: hops.ops.com.ai/v1alpha1 -kind: AuthStack -metadata: - name: zitadel - namespace: default -spec: - clusterName: dory - helmProviderConfigRef: - name: default - kubernetesProviderConfigRef: - name: default - - # Install namespace - namespace: auth - - # Must match Helm Service FQDN (default fullname = zitadel-zitadel) - domain: zitadel-zitadel.auth.svc.cluster.local - externalSecure: false - - gateway: - enabled: false - - externalSecrets: - enabled: false - - firstInstance: - org: hops-ops - masterkey: - # Live Secret in install ns (auth) — not inline in gitops. - # kubectl -n auth create secret generic zitadel-masterkey \ - # --from-literal=masterkey='hops-like-basketball-but-for-ops' - # Chart expects the key material under the secret name; see AuthStack XRD. - secretRef: - name: zitadel-masterkey - - # Embedded PG lands in the same install namespace (auth). - database: - embedded: - storage: - size: 2Gi - class: psql diff --git a/tests/e2e-ui/gitops/cluster/stacks/psql.yaml b/tests/e2e-ui/gitops/cluster/stacks/psql.yaml deleted file mode 100644 index dc1a13789..000000000 --- a/tests/e2e-ui/gitops/cluster/stacks/psql.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Platform layer: CNPG + Atlas + StorageClass named `psql`. -# Install package first: `hops config install ` (or apply configurations/psql-stack.yaml). -# -# Dory/k3s uses rancher.io/local-path (default SC). Snapshot class off (no CSI snapshotter). -apiVersion: hops.ops.com.ai/v1alpha1 -kind: PSQLStack -metadata: - name: psql - namespace: default -spec: - clusterName: dory - helmProviderConfigRef: - name: default - kubernetesProviderConfigRef: - name: default - storageClass: - enabled: true - name: psql - provisioner: rancher.io/local-path - allowVolumeExpansion: false - parameters: {} - snapshotClass: - enabled: false - scaleToZeroPlugin: - enabled: false diff --git a/tests/e2e-ui/gitops/envs/local/api.yaml b/tests/e2e-ui/gitops/envs/local/api.yaml deleted file mode 100644 index 358c46a35..000000000 --- a/tests/e2e-ui/gitops/envs/local/api.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: hops.local/v1alpha1 -kind: Application -metadata: - name: e2e-ui-api -spec: - source: - path: ../../../api/.gitops/deploy - helm: - values: - local: true - appRuntime: cluster-dev - clusterDev: - sourceGeneration: 1 - database: - enabled: false - # No live project/client UUIDs. The API chart derives the matching - # workspace Secret and reads the generated client id as its audience. - identity: - enabled: true - oidcGeneration: 1 - secretEnv: - ZITADEL_SERVICE_USER_TOKEN: - secretName: e2e-ui-oidc - key: ZITADEL_SERVICE_USER_TOKEN - syncPolicy: - prune: true diff --git a/tests/e2e-ui/gitops/envs/local/ui.yaml b/tests/e2e-ui/gitops/envs/local/ui.yaml deleted file mode 100644 index 204baf992..000000000 --- a/tests/e2e-ui/gitops/envs/local/ui.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: hops.local/v1alpha1 -kind: Application -metadata: - name: e2e-ui-ui -spec: - source: - path: ../../../ui/.gitops/deploy - helm: - values: - local: true - appRuntime: cluster-dev - clusterDev: - sourceGeneration: 1 - # Chart identity MRs: Project/roles/humans + Oidc. - # - projectIdRef / labels (no live UUIDs) - # - writeConnectionSecretToRef for client id/secret - # - orgId omitted (ProviderConfig PAT org) - identity: - enabled: true - # Declarative client-secret rotation; bump and let prune replace OIDC. - oidcGeneration: 1 - projectName: e2e-ui - projectNamespace: default - humansNamespace: default - instanceLoginV2: true - demoUsers: true - providerConfigRef: - name: default - kind: ClusterProviderConfig - syncPolicy: - prune: true diff --git a/tests/e2e-ui/scripts/dual-worktree-hmr.mjs b/tests/e2e-ui/scripts/dual-worktree-hmr.mjs deleted file mode 100644 index 764a967da..000000000 --- a/tests/e2e-ui/scripts/dual-worktree-hmr.mjs +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Dual-workspace HMR check for cluster-dev UIs. - * - * Expects both UIs already serving (suite shell brings them up). Opens each - * FQDN, edits `HmrBeacon.svelte` in that worktree, copies **only that file** - * into the live pod (full-tree tar causes Vite SSR reload / full navigation), - * and asserts the beacon updates with **zero** main-frame navigations. - * - * Usage: - * node scripts/dual-worktree-hmr.mjs \ - * --alice-url http://e2e-ui-ui.alice.svc.cluster.local:5180 \ - * --bob-url http://e2e-ui-ui.bob.svc.cluster.local:5180 \ - * --alice-root /path/to/wt-alice/tests/e2e-ui \ - * --bob-root /path/to/wt-bob/tests/e2e-ui - */ -import { chromium } from 'playwright'; -import { readFileSync, writeFileSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { spawnSync } from 'node:child_process'; - -function arg(name, fallback) { - const i = process.argv.indexOf(`--${name}`); - if (i === -1) return fallback; - return process.argv[i + 1]; -} - -const aliceUrl = arg('alice-url'); -const bobUrl = arg('bob-url'); -const aliceRoot = arg('alice-root'); -const bobRoot = arg('bob-root'); -const timeoutMs = Number(arg('timeout-ms', '120000')); -const kubeconfig = process.env.KUBECONFIG || `${process.env.HOME}/.kube/dory-config`; - -if (!aliceUrl || !bobUrl || !aliceRoot || !bobRoot) { - console.error('missing required args'); - process.exit(2); -} - -const beaconRel = 'ui/src/lib/components/HmrBeacon.svelte'; -const remoteBeacon = '/workspace/tests/e2e-ui/ui/src/lib/components/HmrBeacon.svelte'; - -const beaconFile = (root) => resolve(root, beaconRel); - -function setBeacon(root, marker) { - const path = beaconFile(root); - let src = readFileSync(path, 'utf8'); - // Match with or without existing data-hmr attribute. - const re = - /(([^<]*)(<\/span>)/; - if (!re.test(src)) { - throw new Error(`no data-hmr-beacon span in ${path}`); - } - src = src.replace(re, `$1 data-hmr="${marker}">${marker}$3`); - writeFileSync(path, src); - return path; -} - -function kubectlBase() { - const ctx = process.env.HOPS_KUBE_CONTEXT || ''; - return [ - 'kubectl', - ...(ctx ? ['--context', ctx] : []), - ...(kubeconfig ? ['--kubeconfig', kubeconfig] : []), - ]; -} - -function runningUiPod(namespace) { - const base = kubectlBase(); - const get = spawnSync( - base[0], - [ - ...base.slice(1), - 'get', - 'pod', - '-n', - namespace, - '-l', - 'hops.ops.com.ai/local-app=e2e-ui-api', - '--field-selector=status.phase=Running', - '-o', - 'jsonpath={.items[0].metadata.name}', - ], - { encoding: 'utf8' } - ); - const pod = (get.stdout || '').trim(); - if (!pod) { - throw new Error(`no Running e2e-ui application pod in ${namespace}: ${get.stderr || get.status}`); - } - return pod; -} - -/** - * Pause hops continuous full-tree tar watchers. Full monorepo re-tar after an - * edit rewrites many files and forces Vite SSR page reload (full navigation), - * which is not the HMR path under test. - */ -function pauseDeliveryWatchers() { - // Match the multi-app tar watch script body hops spawns (sync_one / needs_marker). - const out = spawnSync( - 'bash', - [ - '-c', - `pgrep -f 'sync_one\\(\\)|needs_marker\\(\\)' 2>/dev/null || true`, - ], - { encoding: 'utf8' } - ); - const pids = (out.stdout || '') - .split(/\s+/) - .map((s) => s.trim()) - .filter((s) => /^\d+$/.test(s)); - for (const pid of pids) { - spawnSync('kill', [pid], { encoding: 'utf8' }); - console.log(`paused delivery watcher pid=${pid}`); - } -} - -/** - * Single-file delivery: host worktree beacon → pod path Vite watches. - * Avoid full-tree tar (causes SSR reload / full navigation). - */ -function forceBeaconSync(namespace, worktreeRoot) { - const pod = runningUiPod(namespace); - const hostPath = beaconFile(worktreeRoot); - const base = kubectlBase(); - // kubectl cp does not support -c for all versions; use tar of one file. - const script = ` -set -euo pipefail -export COPYFILE_DISABLE=1 -tar cf - -C ${JSON.stringify(resolve(worktreeRoot, 'ui/src/lib/components'))} HmrBeacon.svelte \ - | ${base.join(' ')} exec -i -n ${namespace} ${pod} -c application -- \ - tar xf - -C /workspace/tests/e2e-ui/ui/src/lib/components -${base.join(' ')} exec -n ${namespace} ${pod} -c application -- touch ${remoteBeacon} -`; - const tar = spawnSync('bash', ['-c', script], { - encoding: 'utf8', - timeout: 30000, - }); - if (tar.status !== 0) { - throw new Error( - `beacon sync ${namespace}/${pod} failed: ${tar.stderr || tar.stdout || tar.status}` - ); - } - console.log(`[${namespace}] single-file sync ${hostPath} → ${pod}:${remoteBeacon}`); - return pod; -} - -async function assertHmrBeacon(page, url, root, namespace, marker) { - const navigations = []; - page.on('framenavigated', (frame) => { - if (frame === page.mainFrame()) { - navigations.push({ url: frame.url(), t: Date.now() }); - } - }); - - await page.goto(url, { waitUntil: 'networkidle', timeout: timeoutMs }); - await page.waitForSelector('[data-hmr-beacon]', { timeout: timeoutMs }); - // Give Vite HMR websocket time to connect before we edit. - await page.waitForTimeout(1500); - const baselineNav = navigations.length; - - const stamp = `HMR-${marker}-${Date.now()}`; - const path = setBeacon(root, stamp); - console.log(`[${marker}] wrote beacon ${stamp} → ${path}`); - forceBeaconSync(namespace, root); - - const deadline = Date.now() + timeoutMs; - let seen = false; - while (Date.now() < deadline) { - // Fail immediately if a full navigation already happened after the edit. - const extraSoFar = navigations.length - baselineNav; - if (extraSoFar > 0) { - throw new Error( - `[${marker}] full navigation during beacon update (extra=${extraSoFar}); HMR required (hmr-no-nav)` - ); - } - try { - // CSS may text-transform: uppercase the kicker; compare case-insensitively. - seen = await page.evaluate((expected) => { - const el = document.querySelector('[data-hmr-beacon]'); - if (!el) return false; - const t = (el.textContent || el.innerText || '').toUpperCase(); - return t.includes(String(expected).toUpperCase()); - }, stamp); - } catch (e) { - const msg = e && e.message ? e.message : String(e); - // Context destroyed = navigation/reload — fail (do not swallow). - if (/Execution context was destroyed|navigation/i.test(msg)) { - throw new Error( - `[${marker}] page reloaded during HMR wait (context destroyed): ${msg}` - ); - } - throw e; - } - if (seen) break; - forceBeaconSync(namespace, root); - await page.waitForTimeout(1000); - } - if (!seen) { - throw new Error(`[${marker}] beacon never showed stamp ${stamp} within ${timeoutMs}ms`); - } - - const after = await page.locator('[data-hmr-beacon]').innerText(); - const extraNav = navigations.length - baselineNav; - if (extraNav > 0) { - throw new Error( - `[${marker}] full navigation during beacon update (extra=${extraNav}); text=${JSON.stringify(after)}` - ); - } - if (!after.toUpperCase().includes(stamp.toUpperCase())) { - throw new Error(`[${marker}] beacon missing stamp: ${after}`); - } - console.log(`[${marker}] OK beacon=${JSON.stringify(after)} (hmr-no-nav)`); - return stamp; -} - -// Continuous full-tree delivery watchers fight single-file HMR (mass rewrite → SSR reload). -pauseDeliveryWatchers(); - -const browser = await chromium.launch({ headless: true }); -const context = await browser.newContext(); -try { - const alice = await context.newPage(); - const bob = await context.newPage(); - const aliceStamp = await assertHmrBeacon(alice, aliceUrl, aliceRoot, 'alice', 'alice'); - const bobStamp = await assertHmrBeacon(bob, bobUrl, bobRoot, 'bob', 'bob'); - const aliceText = await alice.locator('[data-hmr-beacon]').innerText(); - const bobText = await bob.locator('[data-hmr-beacon]').innerText(); - const aU = aliceText.toUpperCase(); - const bU = bobText.toUpperCase(); - if (aU.includes(bobStamp.toUpperCase()) || bU.includes(aliceStamp.toUpperCase())) { - throw new Error( - `workspace isolation broken: alice=${JSON.stringify(aliceText)} bob=${JSON.stringify(bobText)}` - ); - } - console.log('dual-worktree-hmr: OK (hmr-no-nav + isolation)'); -} finally { - await browser.close(); -} diff --git a/tests/e2e-ui/scripts/dual-worktree-suite.sh b/tests/e2e-ui/scripts/dual-worktree-suite.sh deleted file mode 100755 index c93b55b3e..000000000 --- a/tests/e2e-ui/scripts/dual-worktree-suite.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env bash -# Local dual-workspace suite: two git worktrees, hops gitops for each, HMR title check. -# -# Prerequisites: -# - local CP with hops (dory/colima/kind) and kubectl context -# - AuthStack optional for identity; HMR does not require login -# - hops binary on PATH or HOPS=path -# - helm, git, node, npm -# -# cd tests/e2e-ui && ./scripts/dual-worktree-suite.sh -# -# Policy: alice owns instance Login V2 Features; bob sets instanceLoginV2=false. -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -DIST_ROOT="$(cd "$ROOT/../.." && pwd)" -SCRATCH="${DUAL_SCRATCH:-${SCRATCH_DIR:-$(pwd)/.dual-worktree-scratch}}" -HOPS="${HOPS:-hops}" -KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/config}" -export KUBECONFIG - -ALICE_NAME="${ALICE_NAME:-alice}" -BOB_NAME="${BOB_NAME:-bob}" -SKIP_GITOPS="${SKIP_GITOPS:-0}" -CLUSTER_PROVIDER="${CLUSTER_PROVIDER:-kind}" -DOCKER_PROVIDER="${DOCKER_PROVIDER:-dory}" -CLUSTER_NAME="${CLUSTER_NAME:-hops}" -CONTEXT="${CONTEXT:-kind-hops}" - -log() { printf '%s\n' "$*"; } -die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } - -need() { command -v "$1" >/dev/null 2>&1 || die "missing $1"; } -need git -need helm -need node -need npm -need kubectl - -if ! command -v "$HOPS" >/dev/null 2>&1; then - # Prefer worktree-built hops-cli - if [[ -x "$DIST_ROOT/../cli/target/release/hops-cli" ]]; then - HOPS="$DIST_ROOT/../cli/target/release/hops-cli" - elif [[ -x "$DIST_ROOT/../../cli/target/release/hops-cli" ]]; then - HOPS="$DIST_ROOT/../../cli/target/release/hops-cli" - else - die "hops binary not found (set HOPS=)" - fi -fi - -mkdir -p "$SCRATCH" -log "scratch=$SCRATCH hops=$HOPS" - -# --- helm contract (always) --- -chmod +x "$ROOT/scripts/helm-contract-test.sh" -"$ROOT/scripts/helm-contract-test.sh" | tee "$SCRATCH/helm-contract.log" - -# --- two worktrees of distributed --- -ALICE_WT="$SCRATCH/wt-${ALICE_NAME}" -BOB_WT="$SCRATCH/wt-${BOB_NAME}" -BRANCH_ALICE="suite/dual-${ALICE_NAME}-$$" -BRANCH_BOB="suite/dual-${BOB_NAME}-$$" - -cleanup_worktrees() { - git -C "$DIST_ROOT" worktree remove --force "$ALICE_WT" 2>/dev/null || true - git -C "$DIST_ROOT" worktree remove --force "$BOB_WT" 2>/dev/null || true - git -C "$DIST_ROOT" branch -D "$BRANCH_ALICE" "$BRANCH_BOB" 2>/dev/null || true -} -trap cleanup_worktrees EXIT - -cleanup_worktrees -git -C "$DIST_ROOT" worktree add -b "$BRANCH_ALICE" "$ALICE_WT" HEAD -git -C "$DIST_ROOT" worktree add -b "$BRANCH_BOB" "$BOB_WT" HEAD - -# Overlay working-tree e2e-ui (includes uncommitted chart/script fixes) so the -# suite exercises the same tree under development, not only last commit. -overlay_e2e() { - local dest="$1" - rsync -a --delete \ - --exclude node_modules --exclude .svelte-kit --exclude dist \ - --exclude playwright-report --exclude test-results --exclude target \ - "$ROOT/" "$dest/tests/e2e-ui/" -} -overlay_e2e "$ALICE_WT" -overlay_e2e "$BOB_WT" - -# Bob: do not own instance Features -BOB_ENV="$BOB_WT/tests/e2e-ui/.gitops/local/environment.yaml" -if [[ -f "$BOB_ENV" ]]; then - if rg -q 'instanceLoginV2:' "$BOB_ENV"; then - sed -i.bak 's/instanceLoginV2: true/instanceLoginV2: false/' "$BOB_ENV" || true - else - # inject under identity: - python3 - </dev/null || echo 000) - if [[ "$code" == "200" ]]; then - log "ready $label $url ($i)" - return 0 - fi - sleep 3 - done - die "timeout waiting for $label at $url (last=$code)" -} - -if [[ "$SKIP_GITOPS" != "1" ]]; then - if ! kubectl --context "$CONTEXT" get ns >/dev/null 2>&1; then - die "kubectl cannot talk to a cluster (set KUBECONFIG); dual gitops requires local CP" - fi - log "reconciling workspace $ALICE_NAME from $ALICE_E2E" - (cd "$ALICE_E2E" && "$HOPS" local gitops environment ./.gitops/local/environment.yaml \ - --name "$ALICE_NAME" --cluster-provider "$CLUSTER_PROVIDER" \ - --docker-provider "$DOCKER_PROVIDER" --cluster-name "$CLUSTER_NAME" \ - --context "$CONTEXT" --once) \ - | tee "$SCRATCH/gitops-alice.log" \ - || die "workspace gitops alice failed — see $SCRATCH/gitops-alice.log" - log "reconciling workspace $BOB_NAME from $BOB_E2E" - (cd "$BOB_E2E" && "$HOPS" local gitops environment ./.gitops/local/environment.yaml \ - --name "$BOB_NAME" --cluster-provider "$CLUSTER_PROVIDER" \ - --docker-provider "$DOCKER_PROVIDER" --cluster-name "$CLUSTER_NAME" \ - --context "$CONTEXT" --once) \ - | tee "$SCRATCH/gitops-bob.log" \ - || die "workspace gitops bob failed — see $SCRATCH/gitops-bob.log" -fi - -wait_http "$ALICE_URL" alice 120 -wait_http "$BOB_URL" bob 120 - -# Assert gitops isolation: distinct OIDC names if present -if kubectl --context "$CONTEXT" get oidc.application.zitadel.m.crossplane.io -A >/dev/null 2>&1; then - kubectl --context "$CONTEXT" get oidc.application.zitadel.m.crossplane.io -A -o wide 2>/dev/null | tee "$SCRATCH/oidc-apps.log" || true - kubectl --context "$CONTEXT" get project.project.zitadel.m.crossplane.io -A 2>/dev/null | tee "$SCRATCH/projects.log" || true -fi - -# Playwright HMR — fail the suite if this fails (do not swallow via bare pipe). -( - cd "$ROOT" - if [[ ! -d node_modules/playwright ]]; then - npm install --no-fund --no-audit 2>&1 | tail -5 - fi - npx playwright install chromium 2>&1 | tail -5 - node scripts/dual-worktree-hmr.mjs \ - --alice-url "$ALICE_URL" \ - --bob-url "$BOB_URL" \ - --alice-root "$ALICE_E2E" \ - --bob-root "$BOB_E2E" \ - --timeout-ms 180000 -) 2>&1 | tee "$SCRATCH/dual-worktree-hmr.log" -hmr_rc=${PIPESTATUS[0]} -if [[ "$hmr_rc" -ne 0 ]]; then - die "HMR check failed (exit $hmr_rc) — see $SCRATCH/dual-worktree-hmr.log" -fi - -# Prove dory desktop name was not rewritten by workspace --name -dory_name_file="${HOME}/.hops/local/dory-name" -if [[ -f "$dory_name_file" ]]; then - dn=$(tr -d '[:space:]' <"$dory_name_file") - if [[ "$dn" == "alice" || "$dn" == "bob" ]]; then - die "dory-name was corrupted to workspace name ($dn) — --dory-name / --name isolation broken" - fi - log "dory-name ok: $dn" -fi - -log "dual-worktree-suite: OK" -log " alice: $ALICE_URL" -log " bob: $BOB_URL" From 67e3362509d2b9ad7d20a0052b92c1757418e824 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 17:43:36 -0500 Subject: [PATCH 29/40] test(e2e-celld): run the full Playwright suite on celld + NATS The live celld job ran only todos and chat; blob, admin, anon and the unique-key spec never ran on celld. `make -C tests/e2e-celld test-browser` and CI now run every Playwright project against the celld-backed app. Locally all 21 tests pass on celld 0.6.2. Found while running it: - up-celld-nats needs esbuild for celld's bundler; it now falls back to js/node_modules/.bin/esbuild and says how to get one otherwise. - Zitadel's first boot took over three minutes here, past up.sh's 180 s wait; the wait is ZITADEL_WAIT_TRIES (default 180 x 2 s). - The app Postgres host port is E2E_APP_DB_PORT (default 5433). [[tasks/gr-p40-examples-e2e-coverage]] --- .github/workflows/integration-celld.yaml | 9 +++------ tests/e2e-celld/Makefile | 12 +++++++++++- tests/e2e-celld/README.md | 7 +++++++ tests/e2e-ui/Makefile | 9 +++++++++ tests/e2e-ui/docker/docker-compose.yml | 4 ++-- tests/e2e-ui/scripts/up.sh | 11 +++++++---- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index b939e220f..5f196a208 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -7,8 +7,7 @@ name: celld (live + e2e-celld) # make -C tests/e2e-celld test # make -C tests/e2e-ui up && make -C tests/e2e-ui up-celld-nats # WATCH=0 WATCH_WORKER=0 make -C tests/e2e-celld run -# E2E_UI_ORIGIN=http://localhost:8791 npx --prefix tests/e2e-ui playwright test \ -# todos.user.spec.ts chat.user.spec.ts --project chromium-user +# make -C tests/e2e-celld test-browser # the full Playwright suite # # Default `cargo test` (quality) still runs fixture-only celld checks and # skips live HTTP unless CELLD_URL is set. This job sets CELLD_URL / NATS_URL. @@ -295,11 +294,9 @@ jobs: -H "x-distributed-internal-secret: $DISTRIBUTED_INTERNAL_SECRET" \ -d '{"ids":["forged"]}' - - name: Todo + Chat browser lifecycle through celld + - name: Full Playwright suite through celld (user, admin, anon) working-directory: tests/e2e-ui - run: >- - npx playwright test todos.user.spec.ts chat.user.spec.ts - --project chromium-user + run: npx playwright test env: E2E_UI_ORIGIN: http://localhost:8791 E2E_API_ORIGIN: http://127.0.0.1:8791 diff --git a/tests/e2e-celld/Makefile b/tests/e2e-celld/Makefile index c5ae8768a..0ff95512d 100644 --- a/tests/e2e-celld/Makefile +++ b/tests/e2e-celld/Makefile @@ -6,7 +6,7 @@ # Distributed owns the GraphQL + sibling UI lifecycle. The independent celld # worker watcher remains infrastructure until celld exposes a lifecycle probe. -.PHONY: run stop test help ensure-watch +.PHONY: run stop test test-browser help ensure-watch BIND ?= 127.0.0.1:8791 API_PORT ?= 8791 @@ -93,6 +93,15 @@ run: $(if $(filter 1,$(WATCH_WORKER)),ensure-watch) test: cargo test --workspace -- --nocapture +## The full e2e-ui Playwright suite (every project: setup, user, admin, anon) +## against this celld-backed app. Needs make -C ../e2e-ui up, +## make -C ../e2e-ui up-celld-nats and make run (this directory). +test-browser: + @code=$$(curl -s -o /dev/null -w '%{http_code}' "$(UI_URL)/" 2>/dev/null || echo 000); \ + if [ "$$code" = "000" ]; then echo "e2e-celld not reachable at $(UI_URL) — start: make run"; exit 1; fi + cd ../e2e-ui && npm install --no-audit --no-fund && npx playwright install chromium + cd ../e2e-ui && E2E_UI_ORIGIN="$(UI_URL)" E2E_API_ORIGIN="http://$(BIND)" npx playwright test + stop: @stop_pidfile() { \ [ -f "$$1" ] || return 0; \ @@ -114,6 +123,7 @@ help: @echo " make test cargo test --workspace (CI)" @echo " make run distributed dev owns GraphQL + sibling UI; worker reload runs alongside" @echo " WATCH_WORKER=0 skip worker-build + celld deploy watch" + @echo " make test-browser full e2e-ui Playwright suite against this app (after make run)" @echo " make stop stop API + UI + worker watch" @echo " infra: make -C ../e2e-ui up && make -C ../e2e-ui up-celld-nats" @echo " Postgres read models from e2e-ui.env DATABASE_URL (not sqlite)" diff --git a/tests/e2e-celld/README.md b/tests/e2e-celld/README.md index c796b48d5..8b779ab52 100644 --- a/tests/e2e-celld/README.md +++ b/tests/e2e-celld/README.md @@ -55,8 +55,15 @@ make up-celld-nats # celld local store + Queue relay + NATS cd ../e2e-celld make run # public gateway :8791 + internal UI :5180 (watches sources) +make test-browser # the full e2e-ui Playwright suite against this app ``` +`make test-browser` runs every Playwright project (setup, user, admin, anon) +of `tests/e2e-ui` against this app; CI's live celld job does the same. On a +cold checkout, build once first (`cargo run -p distributed_cli -- build +tests/e2e-celld` from the repository root): `distributed dev` gives the API +five minutes to become ready, and a cold compile can take longer. + Eventual projectors and `@live` use **Postgres** from `e2e-ui.env` (`DATABASE_URL`). There is no SQLite read-model path. Cells still keep private SQLite per Durable Object. Override with `E2E_CELLD_DATABASE_URL`. diff --git a/tests/e2e-ui/Makefile b/tests/e2e-ui/Makefile index f7ef0ce18..79f418be1 100644 --- a/tests/e2e-ui/Makefile +++ b/tests/e2e-ui/Makefile @@ -79,6 +79,15 @@ up-celld-nats: @set -e; \ command -v docker >/dev/null || { echo "docker required"; exit 1; }; \ celld=$$($(CELLD_CHECK) "$(CELLD_BIN)"); \ + if [ -z "$${CELLD_ESBUILD:-}" ] && ! command -v esbuild >/dev/null; then \ + if [ -x "$(REPO_ROOT)/js/node_modules/.bin/esbuild" ]; then \ + export CELLD_ESBUILD="$(REPO_ROOT)/js/node_modules/.bin/esbuild"; \ + else \ + echo "esbuild required (celld bundles Workers with it): npm install -g esbuild,"; \ + echo "make js-install (uses js/node_modules), or CELLD_ESBUILD="; \ + exit 1; \ + fi; \ + fi; \ command -v worker-build >/dev/null || { echo "worker-build required: cargo install worker-build"; exit 1; }; \ $(MAKE) --no-print-directory down-celld; \ if lsof -tiTCP:$(CELLD_HTTP_PORT) -sTCP:LISTEN >/dev/null 2>&1; then \ diff --git a/tests/e2e-ui/docker/docker-compose.yml b/tests/e2e-ui/docker/docker-compose.yml index aae9e6f85..669467db7 100644 --- a/tests/e2e-ui/docker/docker-compose.yml +++ b/tests/e2e-ui/docker/docker-compose.yml @@ -5,7 +5,7 @@ # cd tests/e2e-ui && make up # # Ports: -# 5433 app Postgres (event store, read models, bus) +# 5433 app Postgres (event store, read models, bus; E2E_APP_DB_PORT) # 18080 Zitadel via proxy (OIDC + Login V2 /ui/v2/login + console) services: app-db: @@ -15,7 +15,7 @@ services: POSTGRES_PASSWORD: e2e POSTGRES_DB: e2e_ui ports: - - "5433:5432" + - "${E2E_APP_DB_PORT:-5433}:5432" healthcheck: test: ["CMD-SHELL", "pg_isready -U e2e -d e2e_ui"] interval: 3s diff --git a/tests/e2e-ui/scripts/up.sh b/tests/e2e-ui/scripts/up.sh index b3edaebcc..fcbed9af0 100755 --- a/tests/e2e-ui/scripts/up.sh +++ b/tests/e2e-ui/scripts/up.sh @@ -59,8 +59,11 @@ for i in $(seq 1 60); do fi done -echo "==> Wait for Zitadel (API via proxy; first boot can take ~30–90s)" -for i in $(seq 1 90); do +# First boot runs every Zitadel migration; on a cold Docker VM that can take +# well over three minutes. ZITADEL_WAIT_TRIES × 2s bounds the wait. +ZITADEL_WAIT_TRIES="${ZITADEL_WAIT_TRIES:-180}" +echo "==> Wait for Zitadel (API via proxy; first boot can take minutes)" +for i in $(seq 1 "$ZITADEL_WAIT_TRIES"); do if curl -fsS --max-time 2 "$ZITADEL_HOST/debug/healthz" >/dev/null 2>&1 \ || curl -fsS --max-time 2 "$ZITADEL_HOST/debug/ready" >/dev/null 2>&1; then echo " zitadel probe ok (${i} tries)" @@ -73,7 +76,7 @@ for i in $(seq 1 90); do fi wait_tick "zitadel health" "$i" 5 sleep 2 - if [[ $i -eq 90 ]]; then + if [[ $i -eq $ZITADEL_WAIT_TRIES ]]; then echo "ERROR: Zitadel unreachable" docker compose -f "$COMPOSE" logs --tail=100 zitadel || true exit 1 @@ -553,7 +556,7 @@ ADMIN_M_UID=$(create_machine "e2e-ui-admin-m" "admin" "$ADMIN_M_KEY") echo " e2e-ui-admin-m → $ADMIN_M_UID" # OIDC audience for JWT validation: project id (Zitadel project-scoped aud) -DATABASE_URL="postgres://e2e:e2e@127.0.0.1:5433/e2e_ui" +DATABASE_URL="postgres://e2e:e2e@127.0.0.1:${E2E_APP_DB_PORT:-5433}/e2e_ui" # Keep Auth.js session cookie decryption stable across re-bootstrap. # Rotating AUTH_SECRET → JWTSessionError: no matching decryption secret. if [[ -z "${AUTH_SECRET:-}" && -f "$OUT" ]]; then From 866a7265f06c53a8aba2b47abb7a5a3216020d67 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 18:40:17 -0500 Subject: [PATCH 30/40] feat(schedule): schedule worker for SQL and in-memory repositories Shared repositories stored ctx.schedule rows but nothing fired them, so scheduling routes could not register there. A ScheduleWorker now claims due rows in one statement (UPDATE ... IN (SELECT ... FOR UPDATE SKIP LOCKED) RETURNING on Postgres; SQLite serializes the write), dispatches each through a CommandHost with the trusted schedule session, and settles only the generation that fired. The command id is derived from (owner, name, generation): one ledger serves every aggregate there, so the owner is part of it. A worker that crashes after the command committed leaves its claim (claimed_until_ms, migration 0013) to lapse; the next worker re-dispatches the same id and the ledger replays it, so each arm fires once. `with_schedule_worker(config)` on SqliteRepository, PostgresRepository and InMemoryRepository returns the repository, which now declares Capability::Scheduling, plus the worker to spawn with the service's host. Poll interval, claim TTL and batch size are configurable (250 ms / 30 s / 32). Latency vs celld is documented. Tests (tests/schedule_worker): capability gate, fire-once and periodic re-arm, crash between commit and settle with a restarted process, two concurrent workers on one database, retry backoff; on SQLite and, with DATABASE_URL, Postgres (added to the Postgres CI job); plus the in-memory repository on a ManualClock. [[tasks/gr-p39-postgres-game-adapters]] --- .github/workflows/integration-postgres.yaml | 3 + docs/coordination-primitives.md | 40 +- migrations/inventory.json | 12 + migrations/postgres/0013_schedule_claims.sql | 7 + migrations/sqlite/0013_schedule_claims.sql | 7 + src/in_memory_repo/repository.rs | 106 +++- src/postgres_repo/mod.rs | 1 + src/repository/sql.rs | 3 + src/repository/sql/schedules.rs | 120 +++- src/schedule.rs | 84 ++- src/schedule/worker.rs | 355 +++++++++++ src/sqlite_repo/mod.rs | 1 + src/sqlx_repo/repo/backend.rs | 20 +- src/sqlx_repo/repo/commit.rs | 94 ++- src/sqlx_repo/repo/executor.rs | 1 + src/sqlx_repo/repo/types.rs | 6 + tests/aggregate_schedules.rs | 4 +- tests/schedule_worker/main.rs | 608 +++++++++++++++++++ 18 files changed, 1428 insertions(+), 44 deletions(-) create mode 100644 migrations/postgres/0013_schedule_claims.sql create mode 100644 migrations/sqlite/0013_schedule_claims.sql create mode 100644 src/schedule/worker.rs create mode 100644 tests/schedule_worker/main.rs diff --git a/.github/workflows/integration-postgres.yaml b/.github/workflows/integration-postgres.yaml index 465eac066..43dae8fe4 100644 --- a/.github/workflows/integration-postgres.yaml +++ b/.github/workflows/integration-postgres.yaml @@ -45,6 +45,9 @@ jobs: cargo test --test distributed_read_model --no-default-features --features postgres --verbose cargo test --test sql_lock_manager --no-default-features --features postgres --verbose cargo test --test graphql_literal_text --no-default-features --features graphql,postgres --verbose + # Schedule worker: claim with FOR UPDATE SKIP LOCKED, crash/restart and + # concurrent workers fire each (owner, name, generation) once. + cargo test --test schedule_worker --no-default-features --features graphql,postgres --verbose - name: Verify source fencing and snapshot rebuild against PostgreSQL env: DISTRIBUTED_SNAPSHOT_TEST_POSTGRES_URL: postgres://postgres:postgres@localhost:5432/distributed_snapshots diff --git a/docs/coordination-primitives.md b/docs/coordination-primitives.md index 2a6fbf87b..fcb4ce0f9 100644 --- a/docs/coordination-primitives.md +++ b/docs/coordination-primitives.md @@ -94,8 +94,8 @@ ctx.schedule(Schedule::at("tick", due, "pool.tick", &input)?)?; | Backend | Stores schedules | Dispatches them (`Scheduling`) | Clock (`Clock`) | |---|---|---|---| | Aggregate cell (celld; reference host) | `cell_alarms` | yes: the Durable Object alarm; `fire_due_schedules` on the reference host | DO clock / store clock | -| In-memory repository | per-stream table | not yet | injected `Clock` (`with_clock`) | -| SQL (Postgres, SQLite) | `aggregate_schedules` | not yet (schedule worker planned) | database clock | +| In-memory repository | per-stream table | with a schedule worker attached (`with_schedule_worker`) | injected `Clock` (`with_clock`) | +| SQL (Postgres, SQLite) | `aggregate_schedules` | with a schedule worker attached (`with_schedule_worker`) | database clock | A route declares what it needs with `.requires(..)`; registering it on a backend without it panics with the missing capabilities while the service is @@ -104,6 +104,42 @@ declared `Clock` (the dispatcher stamps the host clock into the session and removes any caller value); `ctx.schedule` refuses unless the route declared `Scheduling`. +**Shared stores: the schedule worker** (`distributed::schedule::worker`, +`graphql` feature). SQL and in-memory repositories store schedules for +every aggregate in one table, so a worker process fires them: + +```rust +let (repo, schedules) = PostgresRepository::connect_and_migrate(url) + .await? + .with_schedule_worker(ScheduleWorkerConfig::default()); // repo now declares Scheduling +let service = Arc::new(build_service(repo.clone())); +let host: Arc = Arc::new(LocalCommandHost::new(Arc::clone(&service))); +let worker = schedules.spawn(host); // worker.stop().await on shutdown +``` + +- Each pass claims up to 32 due, unclaimed rows in one statement + (`UPDATE … WHERE … IN (SELECT … FOR UPDATE SKIP LOCKED) RETURNING`; + SQLite serializes writers instead), marking them `claimed_until_ms` + (default 30 s ahead). Concurrent workers in several processes get + disjoint rows. +- Each claimed arm is dispatched through the `CommandHost` with + `schedule_session` and a command id derived from the owning stream, the + name and the generation (one ledger serves every aggregate here, so the + owner is part of the id). A worker that crashes after the command + committed leaves the claim to lapse; the next worker re-dispatches the + same id and the ledger replays it, so the arm fires once. +- Settlement touches only the generation that fired, like on celld. +- The scheduled command finds its aggregate through its input; the handler + that called `ctx.schedule` supplies that input. + +**Latency.** On celld the Durable Object alarm wakes the cell at the due +time (a few milliseconds late) without any extra process. The schedule +worker fires a due row within one poll interval (`poll_interval`, default +250 ms) plus the dispatch, and only while a worker runs; in the test suite +with a 20 ms poll the observed delay past due was about 20–45 ms on SQLite. +Lower the interval for tighter timing at the cost of one claim query per +interval per worker. + **celld.** A Durable Object has one alarm. The outbox watchdog (`CelldOutbox`) and due schedules share it: every `setAlarm` targets the earliest of the next schedule and the next drain, so a watchdog rearm never diff --git a/migrations/inventory.json b/migrations/inventory.json index ef1998c3b..f96f184c4 100644 --- a/migrations/inventory.json +++ b/migrations/inventory.json @@ -144,6 +144,18 @@ "path": "migrations/postgres/0012_aggregate_schedules.sql", "sha256": "e56147e54924aadac88619ab722f005bbf127f3c009d0cdf80f7ee69b1c2823f" } + }, + { + "version": 13, + "description": "schedule claims", + "sqlite": { + "path": "migrations/sqlite/0013_schedule_claims.sql", + "sha256": "fb6f15299dad6dde0e98333c11f8cc4f9d13bbee793e856741a5c43fb6d8d48b" + }, + "postgres": { + "path": "migrations/postgres/0013_schedule_claims.sql", + "sha256": "fb6f15299dad6dde0e98333c11f8cc4f9d13bbee793e856741a5c43fb6d8d48b" + } } ] } diff --git a/migrations/postgres/0013_schedule_claims.sql b/migrations/postgres/0013_schedule_claims.sql new file mode 100644 index 000000000..36b003381 --- /dev/null +++ b/migrations/postgres/0013_schedule_claims.sql @@ -0,0 +1,7 @@ +-- Schedule worker claims (see `distributed::schedule::worker`). A worker +-- claims a due armed row by setting `claimed_until_ms` (epoch ms, database +-- clock) and dispatches it; other workers skip the row until the claim +-- expires. Settling or re-arming clears the claim. Exactly-once effects per +-- (owner, name, generation) come from the command ledger, not the claim: a +-- crashed worker's claim simply lapses and the next worker replays. +ALTER TABLE aggregate_schedules ADD COLUMN claimed_until_ms BIGINT; diff --git a/migrations/sqlite/0013_schedule_claims.sql b/migrations/sqlite/0013_schedule_claims.sql new file mode 100644 index 000000000..36b003381 --- /dev/null +++ b/migrations/sqlite/0013_schedule_claims.sql @@ -0,0 +1,7 @@ +-- Schedule worker claims (see `distributed::schedule::worker`). A worker +-- claims a due armed row by setting `claimed_until_ms` (epoch ms, database +-- clock) and dispatches it; other workers skip the row until the claim +-- expires. Settling or re-arming clears the claim. Exactly-once effects per +-- (owner, name, generation) come from the command ledger, not the claim: a +-- crashed worker's claim simply lapses and the next worker replays. +ALTER TABLE aggregate_schedules ADD COLUMN claimed_until_ms BIGINT; diff --git a/src/in_memory_repo/repository.rs b/src/in_memory_repo/repository.rs index c9592db09..be06e6ac3 100644 --- a/src/in_memory_repo/repository.rs +++ b/src/in_memory_repo/repository.rs @@ -63,9 +63,11 @@ pub struct InMemoryRepository { #[cfg_attr(not(feature = "graphql"), allow(dead_code))] causal_storage_identity: CausalStorageIdentity, /// Schedule rows per owning stream (`storage_key`), committed with the - /// owner's events. Stored, not dispatched: no - /// [`Capability::Scheduling`](crate::schedule::Capability::Scheduling). - schedules: Arc>>, + /// owner's events. Dispatched only once a schedule worker is attached + /// (`with_schedule_worker`). + schedules: Arc>>, + /// A schedule worker was attached: [`Capability::Scheduling`](crate::schedule::Capability::Scheduling). + schedule_dispatch: bool, /// The store clock: `ctx.now()` and relative schedules. clock: Arc, } @@ -100,6 +102,7 @@ impl InMemoryRepository { projection_change_retention: ProjectionChangeRetention::default(), causal_storage_identity: CausalStorageIdentity::new(), schedules: Arc::new(RwLock::new(HashMap::new())), + schedule_dispatch: false, clock: Arc::new(crate::SystemClock), } } @@ -121,7 +124,7 @@ impl InMemoryRepository { .read() .map_err(|_| RepositoryError::LockPoisoned("schedule read"))? .get(&owner.storage_key()) - .map(crate::schedule::MemorySchedules::armed) + .map(|(_, table)| table.armed()) .unwrap_or_default()) } @@ -536,11 +539,14 @@ impl InMemoryRepository { let planned_schedules = match &schedule_owner { Some(owner) => Some(( owner.storage_key(), - schedule_storage - .get(&owner.storage_key()) - .cloned() - .unwrap_or_default() - .plan(&batch.schedules)?, + ( + owner.clone(), + schedule_storage + .get(&owner.storage_key()) + .map(|(_, table)| table.clone()) + .unwrap_or_default() + .plan(&batch.schedules)?, + ), )), None => None, }; @@ -1146,12 +1152,13 @@ mod tests { } } -/// Stores schedules, does not dispatch them (no worker yet); the store clock -/// is the injected [`Clock`](crate::Clock). +/// Stores schedules; dispatches them once a schedule worker is attached +/// (`with_schedule_worker`). The store clock is the injected +/// [`Clock`](crate::Clock). impl crate::microsvc::BackendHost for InMemoryRepository { fn capabilities(&self) -> crate::schedule::Capabilities { crate::schedule::Capabilities { - scheduling: false, + scheduling: self.schedule_dispatch, clock: true, } } @@ -1162,3 +1169,78 @@ impl crate::microsvc::BackendHost for InMemoryRepository { std::future::ready(Ok(self.clock.now())) } } + +#[cfg(feature = "graphql")] +#[async_trait::async_trait] +impl crate::schedule::worker::ScheduleSource for InMemoryRepository { + async fn schedule_now(&self) -> Result { + Ok(self.clock.now()) + } + + async fn claim_due_schedules( + &self, + now_ms: i64, + claim_until_ms: i64, + limit: usize, + ) -> Result, RepositoryError> { + let mut storage = self + .schedules + .write() + .map_err(|_| RepositoryError::LockPoisoned("schedule write"))?; + let mut claimed = Vec::new(); + for (owner, table) in storage.values_mut() { + for schedule in table.claim_due(now_ms, claim_until_ms, limit) { + claimed.push((owner.clone(), schedule)); + } + } + claimed.sort_by(|(a_owner, a), (b_owner, b)| { + (a.due_at_ms, a_owner.storage_key(), &a.name).cmp(&( + b.due_at_ms, + b_owner.storage_key(), + &b.name, + )) + }); + // Claims beyond `limit` across owners go back unclaimed. + for (owner, schedule) in claimed.drain(limit.min(claimed.len())..) { + if let Some((_, table)) = storage.get_mut(&owner.storage_key()) { + table.release_claim(&schedule.name, schedule.generation); + } + } + Ok(claimed) + } + + async fn settle_schedule( + &self, + owner: &StreamIdentity, + name: &str, + generation: u64, + settlement: &crate::schedule::ScheduleSettlement, + ) -> Result<(), RepositoryError> { + if let Some((_, table)) = self + .schedules + .write() + .map_err(|_| RepositoryError::LockPoisoned("schedule write"))? + .get_mut(&owner.storage_key()) + { + table.settle(name, generation, settlement); + } + Ok(()) + } +} + +#[cfg(feature = "graphql")] +impl InMemoryRepository { + /// Attach a schedule worker: the returned repository declares + /// [`Capability::Scheduling`](crate::schedule::Capability::Scheduling), + /// and the returned [`ScheduleWorker`](crate::schedule::worker::ScheduleWorker) + /// fires due schedules by the store clock once started with the + /// service's command host. Clones share the schedule table. + pub fn with_schedule_worker( + mut self, + config: crate::schedule::worker::ScheduleWorkerConfig, + ) -> (Self, crate::schedule::worker::ScheduleWorker) { + self.schedule_dispatch = true; + let worker = crate::schedule::worker::ScheduleWorker::new(self.clone(), config); + (self, worker) + } +} diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 7fcf236e7..45255dc0b 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -66,6 +66,7 @@ impl crate::sqlx_repo::repo::SqlxRepoBackend for Postgres { EXTRACT(EPOCH FROM retention_expires_at)::double precision AS retention_expires_at, \ EXTRACT(EPOCH FROM compacted_at)::double precision AS compacted_at"; const COMMAND_LEDGER_LOCK_SUFFIX: &'static str = " FOR UPDATE"; + const SCHEDULE_CLAIM_LOCK_SUFFIX: &'static str = " FOR UPDATE SKIP LOCKED"; const COMMAND_LEDGER_COMPACTION_LOCK_SUFFIX: &'static str = " FOR UPDATE SKIP LOCKED"; const EVENT_SELECT: &'static str = "event_name, \ event_version, \ diff --git a/src/repository/sql.rs b/src/repository/sql.rs index 6d5fd560b..cfaabfe65 100644 --- a/src/repository/sql.rs +++ b/src/repository/sql.rs @@ -121,6 +121,9 @@ pub(crate) trait SqlExecutor: Send { const COMMAND_LEDGER_SELECT: &'static str; const COMMAND_LEDGER_LOCK_SUFFIX: &'static str; const COMMAND_LEDGER_COMPACTION_LOCK_SUFFIX: &'static str; + /// Row-lock clause of the schedule worker's claim subquery (`FOR UPDATE + /// SKIP LOCKED` on Postgres; SQLite serializes writers instead). + const SCHEDULE_CLAIM_LOCK_SUFFIX: &'static str = ""; fn query( &mut self, diff --git a/src/repository/sql/schedules.rs b/src/repository/sql/schedules.rs index 811a5c896..e1b5fbcf1 100644 --- a/src/repository/sql/schedules.rs +++ b/src/repository/sql/schedules.rs @@ -5,7 +5,9 @@ use super::{SqlExecutor, SqlRow, Statement}; use crate::repository::RepositoryError; -use crate::schedule::{ArmedSchedule, ScheduleError, ScheduleWrite, MAX_ARMED_SCHEDULES}; +use crate::schedule::{ + ArmedSchedule, ScheduleError, ScheduleSettlement, ScheduleWrite, MAX_ARMED_SCHEDULES, +}; use crate::StreamIdentity; /// Apply one command's resolved schedule writes for `owner`. @@ -32,7 +34,7 @@ pub(crate) async fn apply( .bind(input.as_str().into()) .sql(", ") .bind(schedule.due_at_ms()?.into()) - .sql(", 1, 0, 1) ON CONFLICT (aggregate_type, aggregate_id, name) DO UPDATE SET command = excluded.command, input = excluded.input, due_at_ms = excluded.due_at_ms, generation = aggregate_schedules.generation + 1, attempts = 0, armed = 1"), + .sql(", 1, 0, 1) ON CONFLICT (aggregate_type, aggregate_id, name) DO UPDATE SET command = excluded.command, input = excluded.input, due_at_ms = excluded.due_at_ms, generation = aggregate_schedules.generation + 1, attempts = 0, armed = 1, claimed_until_ms = NULL"), ) .await?; } @@ -40,7 +42,7 @@ pub(crate) async fn apply( executor .execute( Statement::new( - "UPDATE aggregate_schedules SET armed = 0, attempts = 0 WHERE aggregate_type = ", + "UPDATE aggregate_schedules SET armed = 0, attempts = 0, claimed_until_ms = NULL WHERE aggregate_type = ", ) .bind(owner.aggregate_type().into()) .sql(" AND aggregate_id = ") @@ -89,21 +91,103 @@ pub(crate) async fn armed( .sql(" ORDER BY due_at_ms, name"), ) .await?; - rows.iter() + rows.iter().map(armed_from_row).collect() +} + +fn armed_from_row(row: &impl SqlRow) -> Result { + let input = row.text("input")?; + Ok(ArmedSchedule { + name: row.text("name")?, + command: row.text("command")?, + input: serde_json::from_str(&input).map_err(|error| { + RepositoryError::Model(format!("schedule input is invalid JSON: {error}")) + })?, + due_at_ms: row.integer("due_at_ms")?, + generation: u64::try_from(row.integer("generation")?) + .map_err(|_| RepositoryError::Model("schedule generation is negative".into()))?, + attempts: u32::try_from(row.integer("attempts")?).unwrap_or(u32::MAX), + }) +} + +/// Claim up to `limit` armed schedules due at `now_ms` that no live claim +/// holds, until `claim_until_ms`, in one statement: concurrent workers get +/// disjoint rows (`FOR UPDATE SKIP LOCKED` on Postgres; SQLite serializes +/// the write). Returned earliest first, with their owners. +#[cfg_attr(not(feature = "graphql"), allow(dead_code))] +pub(crate) async fn claim_due( + executor: &mut E, + now_ms: i64, + claim_until_ms: i64, + limit: usize, +) -> Result, RepositoryError> { + let limit = i64::try_from(limit).unwrap_or(i64::MAX); + let rows = executor + .query( + Statement::new("UPDATE aggregate_schedules SET claimed_until_ms = ") + .bind(claim_until_ms.into()) + .sql(" WHERE (aggregate_type, aggregate_id, name) IN (SELECT aggregate_type, aggregate_id, name FROM aggregate_schedules WHERE armed = 1 AND due_at_ms <= ") + .bind(now_ms.into()) + .sql(" AND (claimed_until_ms IS NULL OR claimed_until_ms <= ") + .bind(now_ms.into()) + .sql(") ORDER BY due_at_ms, aggregate_type, aggregate_id, name LIMIT ") + .bind(limit.into()) + .sql(E::SCHEDULE_CLAIM_LOCK_SUFFIX) + .sql(") RETURNING aggregate_type, aggregate_id, name, command, input, due_at_ms, generation, attempts"), + ) + .await?; + let mut claimed = rows + .iter() .map(|row| { - let input = row.text("input")?; - Ok(ArmedSchedule { - name: row.text("name")?, - command: row.text("command")?, - input: serde_json::from_str(&input).map_err(|error| { - RepositoryError::Model(format!("schedule input is invalid JSON: {error}")) - })?, - due_at_ms: row.integer("due_at_ms")?, - generation: u64::try_from(row.integer("generation")?).map_err(|_| { - RepositoryError::Model("schedule generation is negative".into()) - })?, - attempts: u32::try_from(row.integer("attempts")?).unwrap_or(u32::MAX), - }) + let owner = + StreamIdentity::new(row.text("aggregate_type")?, row.text("aggregate_id")?)?; + Ok((owner, armed_from_row(row)?)) }) - .collect() + .collect::, RepositoryError>>()?; + claimed.sort_by(|(a_owner, a), (b_owner, b)| { + (a.due_at_ms, a_owner.storage_key(), &a.name).cmp(&( + b.due_at_ms, + b_owner.storage_key(), + &b.name, + )) + }); + Ok(claimed) +} + +/// Settle one fired arm. Only the generation that fired is touched: a +/// handler that re-armed its own name produced a newer generation, which +/// stays armed; a stale or already-settled arm is a no-op. +#[cfg_attr(not(feature = "graphql"), allow(dead_code))] +pub(crate) async fn settle( + executor: &mut impl SqlExecutor, + owner: &StreamIdentity, + name: &str, + generation: u64, + settlement: &ScheduleSettlement, +) -> Result<(), RepositoryError> { + let generation = i64::try_from(generation) + .map_err(|_| RepositoryError::Model("schedule generation exceeds SQL storage".into()))?; + let statement = match settlement { + ScheduleSettlement::Done => Statement::new( + "UPDATE aggregate_schedules SET armed = 0, attempts = 0, claimed_until_ms = NULL WHERE aggregate_type = ", + ), + ScheduleSettlement::Retry { due_at_ms } => Statement::new( + "UPDATE aggregate_schedules SET attempts = attempts + 1, claimed_until_ms = NULL, due_at_ms = ", + ) + .bind((*due_at_ms).into()) + .sql(" WHERE aggregate_type = "), + }; + executor + .execute( + statement + .bind(owner.aggregate_type().into()) + .sql(" AND aggregate_id = ") + .bind(owner.aggregate_id().into()) + .sql(" AND name = ") + .bind(name.into()) + .sql(" AND generation = ") + .bind(generation.into()) + .sql(" AND armed = 1"), + ) + .await?; + Ok(()) } diff --git a/src/schedule.rs b/src/schedule.rs index b547cd390..da8d6b45a 100644 --- a/src/schedule.rs +++ b/src/schedule.rs @@ -31,8 +31,10 @@ //! **Backends declare [`Capabilities`].** Every backend stores schedule rows //! (cells in their own `cell_alarms` table, the in-memory repository in a //! per-stream table, SQL in `aggregate_schedules`); only some *dispatch* -//! them ([`Capability::Scheduling`]: aggregate cells today — the Durable -//! Object alarm on celld, `fire_due_schedules` on the reference host). Every +//! them ([`Capability::Scheduling`]): aggregate cells (the Durable Object +//! alarm on celld, `fire_due_schedules` on the reference host), and the SQL +//! and in-memory repositories once a schedule worker is attached +//! (`with_schedule_worker`, see `schedule::worker`, `graphql` feature). Every //! backend has a clock ([`Capability::Clock`]): the cell store clock (the DO //! clock on celld), the in-memory repository's injected //! [`Clock`](crate::Clock), the database clock on SQL. Routes declare what @@ -46,6 +48,9 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use crate::microsvc::Session; + +#[cfg(feature = "graphql")] +pub mod worker; use crate::repository::RepositoryError; use crate::{ROLE_KEY, USER_ID_KEY}; @@ -549,7 +554,8 @@ where is_schedule_session(ctx.session()) } -/// Deterministic UUIDv7-shaped command id of one schedule arm. +/// Deterministic UUIDv7-shaped command id of one schedule arm of a cell +/// (the cell's own ledger scopes it to the owner). pub(crate) fn schedule_command_id(name: &str, generation: u64) -> String { let mut hasher = Sha256::new(); hasher.update(SCHEDULE_SERVICE_ID.as_bytes()); @@ -557,6 +563,32 @@ pub(crate) fn schedule_command_id(name: &str, generation: u64) -> String { hasher.update(name.as_bytes()); hasher.update([0u8]); hasher.update(generation.to_be_bytes()); + uuid_v7_shaped(hasher) +} + +/// Command id of one schedule arm held in a shared store (SQL, in-memory): +/// one command ledger serves every aggregate there, so the owning stream is +/// part of the identity. +#[cfg_attr(not(feature = "graphql"), allow(dead_code))] +pub(crate) fn owned_schedule_command_id( + owner: &crate::StreamIdentity, + name: &str, + generation: u64, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(SCHEDULE_SERVICE_ID.as_bytes()); + hasher.update([1u8]); + hasher.update(owner.aggregate_type().as_bytes()); + hasher.update([0u8]); + hasher.update(owner.aggregate_id().as_bytes()); + hasher.update([0u8]); + hasher.update(name.as_bytes()); + hasher.update([0u8]); + hasher.update(generation.to_be_bytes()); + uuid_v7_shaped(hasher) +} + +fn uuid_v7_shaped(hasher: Sha256) -> String { let digest = hasher.finalize(); let mut bytes = [0u8; 16]; bytes.copy_from_slice(&digest[..16]); @@ -613,6 +645,9 @@ pub(crate) struct MemorySchedules { struct MemoryScheduleRow { schedule: ArmedSchedule, armed: bool, + /// A schedule worker's claim (epoch ms); cleared by settling or + /// re-arming. Transient: not part of restart exports. + claimed_until_ms: Option, } impl MemorySchedules { @@ -638,12 +673,14 @@ impl MemorySchedules { attempts: 0, }, armed: true, + claimed_until_ms: None, }, ); } ScheduleWrite::Cancel(name) => { if let Some(row) = next.rows.get_mut(name) { row.armed = false; + row.claimed_until_ms = None; } } } @@ -673,6 +710,45 @@ impl MemorySchedules { .collect() } + /// Claim up to `limit` due, unclaimed (or claim-expired) rows until + /// `claim_until_ms`, earliest first (the schedule worker's claim). + #[cfg_attr(not(feature = "graphql"), allow(dead_code))] + pub(crate) fn claim_due( + &mut self, + now_ms: i64, + claim_until_ms: i64, + limit: usize, + ) -> Vec { + let mut due = self + .rows + .values() + .filter(|row| { + row.armed + && row.schedule.due_at_ms <= now_ms + && row.claimed_until_ms.is_none_or(|until| until <= now_ms) + }) + .map(|row| row.schedule.clone()) + .collect::>(); + due.sort_by(|a, b| (a.due_at_ms, &a.name).cmp(&(b.due_at_ms, &b.name))); + due.truncate(limit); + for schedule in &due { + if let Some(row) = self.rows.get_mut(&schedule.name) { + row.claimed_until_ms = Some(claim_until_ms); + } + } + due + } + + /// Drop a claim taken by [`claim_due`](Self::claim_due) without settling. + #[cfg_attr(not(feature = "graphql"), allow(dead_code))] + pub(crate) fn release_claim(&mut self, name: &str, generation: u64) { + if let Some(row) = self.rows.get_mut(name) { + if row.schedule.generation == generation { + row.claimed_until_ms = None; + } + } + } + pub(crate) fn next_due_ms(&self) -> Option { self.rows .values() @@ -688,6 +764,7 @@ impl MemorySchedules { if !row.armed || row.schedule.generation != generation { return; } + row.claimed_until_ms = None; match settlement { ScheduleSettlement::Done => { row.armed = false; @@ -721,6 +798,7 @@ impl MemorySchedules { MemoryScheduleRow { schedule: row.schedule, armed: row.armed, + claimed_until_ms: None, }, ) }) diff --git a/src/schedule/worker.rs b/src/schedule/worker.rs new file mode 100644 index 000000000..b69e11c22 --- /dev/null +++ b/src/schedule/worker.rs @@ -0,0 +1,355 @@ +//! The schedule worker for shared stores (SQL and in-memory repositories). +//! +//! Aggregate cells dispatch their own schedules on their Durable Object +//! alarm. A shared repository only *stores* `ctx.schedule` rows in its +//! `aggregate_schedules` table (or the in-memory equivalent); this worker +//! fires them: +//! +//! 1. **Claim.** Every `poll_interval`, one statement claims up to +//! `batch_size` armed rows due by the store clock (the database clock on +//! SQL) that no live claim holds, until `now + claim_ttl`. Concurrent +//! workers get disjoint rows: `FOR UPDATE SKIP LOCKED` on Postgres, the +//! single writer on SQLite. +//! 2. **Dispatch** each through a [`CommandHost`] with the trusted +//! [`schedule_session`] and a command id derived from the owning stream, +//! the schedule name and its generation +//! ([`ScheduleWorker::command_id`]). The command ledger makes that id +//! exactly-once: a re-dispatch replays the committed receipt. +//! 3. **Settle** only the generation that fired: done (disarm) on success or +//! a terminal rejection, or retry with exponential backoff on a retryable +//! failure. A handler that re-armed its own name produced a newer +//! generation, which stays armed. +//! +//! A worker that crashes between dispatch and settle leaves its claim to +//! lapse after `claim_ttl`; the next worker re-dispatches the same command +//! id, the ledger replays it, and the arm settles. Effects therefore happen +//! once per (owner, name, generation) across crashes and restarts. +//! +//! **Latency.** A due schedule fires within one `poll_interval` (default +//! 250 ms) plus dispatch time, and only while a worker runs. On celld the +//! Durable Object alarm wakes the cell at the due time itself (typically a +//! few milliseconds late) and needs no worker process. +//! +//! **Capability.** Attaching a worker (`with_schedule_worker` on +//! [`SqliteRepository`](crate::SqliteRepository), +//! [`PostgresRepository`](crate::PostgresRepository) and +//! [`InMemoryRepository`](crate::InMemoryRepository)) is what makes the +//! repository declare [`Capability::Scheduling`](super::Capability::Scheduling), +//! so routes that require it can be registered. Start the returned worker +//! with the host of the service built over that repository. +//! +//! The scheduled command must address the owning aggregate through its +//! input (the handler that called `ctx.schedule` supplies that input). + +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use async_trait::async_trait; + +use super::{ + owned_schedule_command_id, retry_delay, schedule_session, system_time_to_ms, ArmedSchedule, + FailedSchedule, FiredSchedule, ScheduleFireReport, ScheduleSettlement, SCHEDULE_FIRE_BATCH, + SCHEDULE_USER_ID, +}; +use crate::command_dispatch::CommandHost; +use crate::graphql::identity::VerifiedPrincipal; +use crate::microsvc::CausalDispatchError; +use crate::repository::RepositoryError; +use crate::StreamIdentity; + +/// Default time between claim passes. +pub const DEFAULT_SCHEDULE_POLL_INTERVAL: Duration = Duration::from_millis(250); +/// Default lifetime of a claim before another worker may take the row over. +pub const DEFAULT_SCHEDULE_CLAIM_TTL: Duration = Duration::from_secs(30); + +/// How a [`ScheduleWorker`] polls. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScheduleWorkerConfig { + /// Time between claim passes when the last pass was not full. + pub poll_interval: Duration, + /// How long a claim keeps other workers off a row. Must exceed the + /// longest expected dispatch; a lapsed claim is safe (the ledger + /// replays) but dispatches the command a second time. + pub claim_ttl: Duration, + /// Most schedules claimed per pass. + pub batch_size: usize, +} + +impl Default for ScheduleWorkerConfig { + fn default() -> Self { + Self { + poll_interval: DEFAULT_SCHEDULE_POLL_INTERVAL, + claim_ttl: DEFAULT_SCHEDULE_CLAIM_TTL, + batch_size: SCHEDULE_FIRE_BATCH, + } + } +} + +impl ScheduleWorkerConfig { + pub fn with_poll_interval(mut self, poll_interval: Duration) -> Self { + self.poll_interval = poll_interval; + self + } + + pub fn with_claim_ttl(mut self, claim_ttl: Duration) -> Self { + self.claim_ttl = claim_ttl; + self + } + + pub fn with_batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = batch_size.max(1); + self + } +} + +/// The store side of a schedule worker: its clock, the claim and the +/// settlement. Implemented by the framework's shared repositories. +#[async_trait] +pub(crate) trait ScheduleSource: Send + Sync + 'static { + /// The store clock (the database clock on SQL). + async fn schedule_now(&self) -> Result; + + /// Claim up to `limit` due, unclaimed armed schedules until + /// `claim_until_ms`, earliest first. + async fn claim_due_schedules( + &self, + now_ms: i64, + claim_until_ms: i64, + limit: usize, + ) -> Result, RepositoryError>; + + /// Settle one fired arm (only that generation). + async fn settle_schedule( + &self, + owner: &StreamIdentity, + name: &str, + generation: u64, + settlement: &ScheduleSettlement, + ) -> Result<(), RepositoryError>; +} + +/// Fires the due schedules of a shared repository through a +/// [`CommandHost`]. See the [module docs](self). +#[must_use = "a schedule worker fires nothing until it runs: call `spawn`, `run` or `run_once`"] +#[derive(Clone)] +pub struct ScheduleWorker { + source: Arc, + config: ScheduleWorkerConfig, +} + +impl std::fmt::Debug for ScheduleWorker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ScheduleWorker") + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +impl ScheduleWorker { + pub(crate) fn new(source: impl ScheduleSource, config: ScheduleWorkerConfig) -> Self { + Self { + source: Arc::new(source), + config, + } + } + + pub fn config(&self) -> ScheduleWorkerConfig { + self.config + } + + /// The command-ledger id a fired arm dispatches with: stable per + /// (owner, name, generation). + pub fn command_id(owner: &StreamIdentity, name: &str, generation: u64) -> String { + owned_schedule_command_id(owner, name, generation) + } + + /// One pass: claim the due schedules, dispatch each through `host`, and + /// settle it. Store errors abort the pass (claims lapse and are retried). + pub async fn run_once( + &self, + host: &dyn CommandHost, + ) -> Result { + let now = self.source.schedule_now().await?; + let now_ms = system_time_to_ms(now)?; + let ttl_ms = i64::try_from(self.config.claim_ttl.as_millis()).unwrap_or(i64::MAX); + let claimed = self + .source + .claim_due_schedules( + now_ms, + now_ms.saturating_add(ttl_ms), + self.config.batch_size.max(1), + ) + .await?; + let mut report = ScheduleFireReport::default(); + for (owner, schedule) in claimed { + let command_id = Self::command_id(&owner, &schedule.name, schedule.generation); + let result = host + .invoke( + &schedule.command, + &command_id, + schedule.input.clone(), + schedule_session(&schedule), + VerifiedPrincipal::from_trusted_transport(SCHEDULE_USER_ID), + None, + ) + .await; + let settlement = match result { + Ok(dispatch) => { + report.fired.push(FiredSchedule { + name: schedule.name.clone(), + generation: schedule.generation, + command: schedule.command.clone(), + replayed: dispatch.replayed(), + payload: dispatch.payload().clone(), + }); + ScheduleSettlement::Done + } + Err(error) => { + let failed = FailedSchedule { + name: schedule.name.clone(), + generation: schedule.generation, + command: schedule.command.clone(), + code: error.code(), + message: match &error { + CausalDispatchError::Internal(message) => message.clone(), + other => other.client_message(), + }, + }; + if is_retryable(&error) { + report.retried.push(failed); + let delay = i64::try_from(retry_delay(schedule.attempts).as_millis()) + .unwrap_or(i64::MAX); + ScheduleSettlement::Retry { + due_at_ms: now_ms.saturating_add(delay), + } + } else { + report.failed.push(failed); + ScheduleSettlement::Done + } + } + }; + self.source + .settle_schedule(&owner, &schedule.name, schedule.generation, &settlement) + .await?; + } + Ok(report) + } + + /// Poll until `shutdown` resolves. Store errors are logged and retried on + /// the next pass; a full pass is followed immediately by another. + pub async fn run( + self, + host: Arc, + shutdown: impl std::future::Future, + ) { + tokio::pin!(shutdown); + let mut sleep_for = Duration::ZERO; + loop { + tokio::select! { + biased; + _ = &mut shutdown => return, + _ = tokio::time::sleep(sleep_for) => {} + } + sleep_for = match self.run_once(host.as_ref()).await { + Ok(report) + if report.fired.len() + report.retried.len() + report.failed.len() + >= self.config.batch_size => + { + Duration::ZERO + } + Ok(_) => self.config.poll_interval, + Err(error) => { + eprintln!("schedule worker: {error}"); + self.config.poll_interval + } + }; + } + } + + /// Spawn [`run`](Self::run) on the current tokio runtime. + pub fn spawn(self, host: Arc) -> ScheduleWorkerHandle { + let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); + let join = tokio::spawn(self.run(host, async move { + let _ = stopped.await; + })); + ScheduleWorkerHandle { + stop: Some(stop), + join, + } + } +} + +/// A spawned [`ScheduleWorker`]. Dropping it does not stop the task; call +/// [`stop`](Self::stop). +#[derive(Debug)] +pub struct ScheduleWorkerHandle { + stop: Option>, + join: tokio::task::JoinHandle<()>, +} + +impl ScheduleWorkerHandle { + /// Stop after the current pass and wait for the task. + pub async fn stop(mut self) { + if let Some(stop) = self.stop.take() { + let _ = stop.send(()); + } + let _ = (&mut self.join).await; + } + + /// Kill the task at its next await point (a crash, for tests): a pass in + /// flight is abandoned without settling. + pub fn abort(self) { + self.join.abort(); + } +} + +fn is_retryable(error: &CausalDispatchError) -> bool { + match error { + CausalDispatchError::InProgress | CausalDispatchError::Internal(_) => true, + CausalDispatchError::Handler(error) => error.status_code() >= 500, + CausalDispatchError::Rejected { status, .. } => *status >= 500, + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::microsvc::HandlerError; + + #[test] + fn storage_and_server_failures_retry_rejections_do_not() { + assert!(is_retryable(&CausalDispatchError::InProgress)); + assert!(is_retryable(&CausalDispatchError::Internal("x".into()))); + assert!(is_retryable(&CausalDispatchError::Handler( + HandlerError::Repository(RepositoryError::Model("conflict".into())) + ))); + assert!(!is_retryable(&CausalDispatchError::Handler( + HandlerError::Rejected("no".into()) + ))); + assert!(!is_retryable(&CausalDispatchError::Forbidden)); + assert!(!is_retryable(&CausalDispatchError::CommandIdReuse)); + } + + #[test] + fn command_ids_are_scoped_to_the_owner() { + let a = StreamIdentity::new("todo", "a").unwrap(); + let b = StreamIdentity::new("todo", "b").unwrap(); + assert_eq!( + ScheduleWorker::command_id(&a, "due", 1), + ScheduleWorker::command_id(&a, "due", 1) + ); + assert_ne!( + ScheduleWorker::command_id(&a, "due", 1), + ScheduleWorker::command_id(&b, "due", 1) + ); + assert_ne!( + ScheduleWorker::command_id(&a, "due", 1), + ScheduleWorker::command_id(&a, "due", 2) + ); + assert_ne!( + ScheduleWorker::command_id(&a, "due", 1), + crate::schedule::schedule_command_id("due", 1) + ); + } +} diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index 47f442246..91aa6f56a 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -59,6 +59,7 @@ impl crate::sqlx_repo::repo::SqlxRepoBackend for Sqlite { const COMMAND_LEDGER_SELECT: &'static str = crate::repository::sqlite_codec::COMMAND_LEDGER_SELECT; const COMMAND_LEDGER_LOCK_SUFFIX: &'static str = ""; + const SCHEDULE_CLAIM_LOCK_SUFFIX: &'static str = ""; const COMMAND_LEDGER_COMPACTION_LOCK_SUFFIX: &'static str = ""; const EVENT_SELECT: &'static str = crate::repository::sqlite_codec::EVENT_SELECT; const SNAPSHOT_SELECT: &'static str = crate::repository::sqlite_codec::SNAPSHOT_SELECT; diff --git a/src/sqlx_repo/repo/backend.rs b/src/sqlx_repo/repo/backend.rs index 948e3e83a..51ef23dbb 100644 --- a/src/sqlx_repo/repo/backend.rs +++ b/src/sqlx_repo/repo/backend.rs @@ -47,7 +47,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]); assert_eq!( descriptions, vec![ @@ -62,7 +62,8 @@ mod tests { "projection delivery aliases", "coordination leases", "cell alarms", - "aggregate schedules" + "aggregate schedules", + "schedule claims" ] ); assert_eq!( @@ -116,6 +117,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/sqlite/0012_aggregate_schedules.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/sqlite/0013_schedule_claims.sql" + )), ] ); } @@ -135,7 +140,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]); assert_eq!( descriptions, vec![ @@ -150,7 +155,8 @@ mod tests { "projection delivery aliases", "coordination leases", "cell alarms", - "aggregate schedules" + "aggregate schedules", + "schedule claims" ] ); assert_eq!( @@ -204,6 +210,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/postgres/0012_aggregate_schedules.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/postgres/0013_schedule_claims.sql" + )), ] ); } @@ -253,6 +263,8 @@ pub trait SqlxRepoBackend: SqlxReadModelBackend { /// Command-ledger projection with timestamp columns normalized for /// [`decode_timestamp`](Self::decode_timestamp) and JSON surfaced as text. const COMMAND_LEDGER_SELECT: &'static str; + /// See [`SqlExecutor::SCHEDULE_CLAIM_LOCK_SUFFIX`](crate::repository::sql::SqlExecutor). + const SCHEDULE_CLAIM_LOCK_SUFFIX: &'static str; /// Row-lock suffix for a reservation/status transaction. const COMMAND_LEDGER_LOCK_SUFFIX: &'static str; /// Row-lock suffix for a bounded compaction scan. diff --git a/src/sqlx_repo/repo/commit.rs b/src/sqlx_repo/repo/commit.rs index 45be32c17..12f2aca7b 100644 --- a/src/sqlx_repo/repo/commit.rs +++ b/src/sqlx_repo/repo/commit.rs @@ -616,8 +616,8 @@ where Ok(versions) } -/// SQL stores schedules but does not dispatch them yet (the schedule worker -/// is a later addition); its clock is the database clock. +/// SQL stores schedules; it dispatches them once a schedule worker is +/// attached (`with_schedule_worker`). Its clock is the database clock. impl crate::microsvc::BackendHost for SqlxRepository where DB: SqlxRepoBackend, @@ -632,7 +632,7 @@ where { fn capabilities(&self) -> crate::schedule::Capabilities { crate::schedule::Capabilities { - scheduling: false, + scheduling: self.schedule_dispatch, clock: true, } } @@ -683,3 +683,91 @@ where .await } } + +#[cfg(feature = "graphql")] +#[async_trait::async_trait] +impl crate::schedule::worker::ScheduleSource for SqlxRepository +where + DB: SqlxRepoBackend, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type + sqlx::Decode<'q, DB>, + for<'q> String: Type + sqlx::Decode<'q, DB>, + for<'q> Vec: Type + sqlx::Decode<'q, DB>, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, + for<'r> &'r str: sqlx::ColumnIndex, +{ + async fn schedule_now(&self) -> Result { + crate::microsvc::BackendHost::host_now(self).await + } + + async fn claim_due_schedules( + &self, + now_ms: i64, + claim_until_ms: i64, + limit: usize, + ) -> Result, RepositoryError> { + let mut connection = + self.pool.acquire().await.map_err(|err| { + repository_storage_error::("acquire schedule connection", err) + })?; + crate::repository::sql::schedules::claim_due( + &mut executor::ConnectionExecutor::(&mut *connection), + now_ms, + claim_until_ms, + limit, + ) + .await + } + + async fn settle_schedule( + &self, + owner: &crate::StreamIdentity, + name: &str, + generation: u64, + settlement: &crate::schedule::ScheduleSettlement, + ) -> Result<(), RepositoryError> { + let mut connection = + self.pool.acquire().await.map_err(|err| { + repository_storage_error::("acquire schedule connection", err) + })?; + crate::repository::sql::schedules::settle( + &mut executor::ConnectionExecutor::(&mut *connection), + owner, + name, + generation, + settlement, + ) + .await + } +} + +#[cfg(feature = "graphql")] +impl SqlxRepository +where + DB: SqlxRepoBackend, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type + sqlx::Decode<'q, DB>, + for<'q> String: Type + sqlx::Decode<'q, DB>, + for<'q> Vec: Type + sqlx::Decode<'q, DB>, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, + for<'r> &'r str: sqlx::ColumnIndex, +{ + /// Attach a schedule worker: the returned repository declares + /// [`Capability::Scheduling`](crate::schedule::Capability::Scheduling), + /// and the returned [`ScheduleWorker`](crate::schedule::worker::ScheduleWorker) + /// fires its due `aggregate_schedules` rows once started with the + /// service's command host. Several processes may each run one; claims + /// keep them off each other's rows. + pub fn with_schedule_worker( + mut self, + config: crate::schedule::worker::ScheduleWorkerConfig, + ) -> (Self, crate::schedule::worker::ScheduleWorker) { + self.schedule_dispatch = true; + let worker = crate::schedule::worker::ScheduleWorker::new(self.clone(), config); + (self, worker) + } +} diff --git a/src/sqlx_repo/repo/executor.rs b/src/sqlx_repo/repo/executor.rs index 352f04d23..907254140 100644 --- a/src/sqlx_repo/repo/executor.rs +++ b/src/sqlx_repo/repo/executor.rs @@ -124,6 +124,7 @@ where const NOW: &'static str = DB::NOW; const COMMAND_LEDGER_SELECT: &'static str = DB::COMMAND_LEDGER_SELECT; const COMMAND_LEDGER_LOCK_SUFFIX: &'static str = DB::COMMAND_LEDGER_LOCK_SUFFIX; + const SCHEDULE_CLAIM_LOCK_SUFFIX: &'static str = DB::SCHEDULE_CLAIM_LOCK_SUFFIX; const COMMAND_LEDGER_COMPACTION_LOCK_SUFFIX: &'static str = DB::COMMAND_LEDGER_COMPACTION_LOCK_SUFFIX; diff --git a/src/sqlx_repo/repo/types.rs b/src/sqlx_repo/repo/types.rs index 6bf6bbb3e..191413928 100644 --- a/src/sqlx_repo/repo/types.rs +++ b/src/sqlx_repo/repo/types.rs @@ -14,6 +14,10 @@ pub struct SqlxRepository { pub(super) notify_enabled: bool, pub(super) projection_change_retention: ProjectionChangeRetention, pub(super) causal_storage_identity: CausalStorageIdentity, + /// A [`ScheduleWorker`](crate::schedule::worker::ScheduleWorker) was + /// attached (`with_schedule_worker`): routes may require + /// [`Capability::Scheduling`](crate::schedule::Capability::Scheduling). + pub(super) schedule_dispatch: bool, } impl Clone for SqlxRepository { @@ -25,6 +29,7 @@ impl Clone for SqlxRepository { notify_enabled: self.notify_enabled, projection_change_retention: self.projection_change_retention, causal_storage_identity: self.causal_storage_identity, + schedule_dispatch: self.schedule_dispatch, } } } @@ -62,6 +67,7 @@ where notify_enabled: true, projection_change_retention: ProjectionChangeRetention::default(), causal_storage_identity: CausalStorageIdentity::new(), + schedule_dispatch: false, } } diff --git a/tests/aggregate_schedules.rs b/tests/aggregate_schedules.rs index 3b2e592cb..fd7f081bd 100644 --- a/tests/aggregate_schedules.rs +++ b/tests/aggregate_schedules.rs @@ -1,8 +1,8 @@ //! Schedule rows on the shared SQL repository: stored in the command's //! transaction, owned by the batch's one stream, relative due times resolved //! by the database clock, re-arming bumps the generation, cancelling -//! disarms. (Dispatching them is a later SQL schedule worker; SQL does not -//! declare `Capability::Scheduling` yet.) +//! disarms. Dispatching them is the schedule worker's job +//! (`tests/schedule_worker`). #![cfg(feature = "sqlite")] diff --git a/tests/schedule_worker/main.rs b/tests/schedule_worker/main.rs new file mode 100644 index 000000000..c9feb9193 --- /dev/null +++ b/tests/schedule_worker/main.rs @@ -0,0 +1,608 @@ +//! The schedule worker on shared repositories: `ctx.schedule` rows stored by +//! SQLite / Postgres / the in-memory repository are fired through a +//! `CommandHost`, exactly once per (owner, name, generation), across worker +//! crashes, restarts and concurrent workers. +//! +//! Postgres runs when `DATABASE_URL` is set (each test in its own schema). + +#![cfg(all(feature = "graphql", any(feature = "sqlite", feature = "postgres")))] + +#[cfg(feature = "postgres")] +#[path = "../support/postgres.rs"] +mod postgres; + +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::graphql::identity::VerifiedPrincipal; +use distributed::graphql::protocol::ProtocolResponseAccumulator; +use distributed::microsvc::{ + CausalCommandContext, CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, + HandlerError, PortableCommand, Routes, Service, Session, +}; +use distributed::schedule::worker::ScheduleWorkerConfig; +use distributed::schedule::{schedule_guard, Capability, Schedule}; +use distributed::{ + sourced, AggregateRepository, CommandHost, Entity, LocalCommandHost, StreamIdentity, ROLE_KEY, + USER_ID_KEY, +}; + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +struct Ticker { + entity: Entity, + ticks: u32, + limit: u32, + period_ms: u64, +} + +#[sourced(entity, aggregate_type = "Ticker", events = "TickerEvent")] +impl Ticker { + #[event("ticker.started", version = 1)] + fn start(&mut self, id: String, limit: u32, period_ms: u64) { + self.entity.set_id(id); + self.limit = limit; + self.period_ms = period_ms; + } + + #[event("ticker.ticked", version = 1)] + fn tick(&mut self) { + self.ticks += 1; + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +struct StartInput { + id: String, + limit: u32, + period_ms: u64, + tick_command: String, +} + +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +struct TickInput { + id: String, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +struct TickerPayload { + id: String, + ticks: u32, +} + +const TICK: &str = "ticker.tick"; +const FLAKY: &str = "ticker.flaky"; + +struct Start; +impl PortableCommand for Start +where + D: distributed::microsvc::CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + "ticker.start", + )) + .requires(Capability::Scheduling) + .guarded( + |ctx: &CausalCommandContext<'_, Ticker>| ctx.session().user_id().is_some(), + handle_start, + ) + } +} + +async fn handle_start( + ctx: &CausalCommandContext<'_, Ticker>, + input: StartInput, +) -> Result>, HandlerError> { + let repo = ctx.repo(); + let mut ticker = repo.create(); + ticker + .start(input.id.clone(), input.limit, input.period_ms) + .map_err(|error| HandlerError::Rejected(error.to_string()))?; + ctx.schedule( + Schedule::after( + "tick", + Duration::from_millis(input.period_ms), + &input.tick_command, + &TickInput { + id: input.id.clone(), + }, + ) + .map_err(|error| HandlerError::Rejected(error.to_string()))?, + )?; + repo.commit(ticker)?.succeeded(TickerPayload { + id: input.id, + ticks: 0, + }) +} + +struct Tick; +impl PortableCommand for Tick +where + D: distributed::microsvc::CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(TICK)) + .requires(Capability::Scheduling) + .guarded(schedule_guard::, handle_tick) + } +} + +async fn handle_tick( + ctx: &CausalCommandContext<'_, Ticker>, + input: TickInput, +) -> Result>, HandlerError> { + let repo = ctx.repo(); + let mut ticker = repo + .get(&input.id) + .await? + .ok_or_else(|| HandlerError::Rejected("no ticker".into()))?; + ticker + .tick() + .map_err(|error| HandlerError::Rejected(error.to_string()))?; + if ticker.ticks < ticker.limit { + let due = ctx.scheduled_due().expect("schedule session") + + Duration::from_millis(ticker.period_ms); + ctx.schedule( + Schedule::at("tick", due, TICK, &input) + .map_err(|error| HandlerError::Rejected(error.to_string()))?, + )?; + } + let ticks = ticker.ticks; + repo.commit(ticker)?.succeeded(TickerPayload { + id: input.id, + ticks, + }) +} + +static FLAKY_FAILURES: AtomicUsize = AtomicUsize::new(0); + +/// Fails with a retryable (500) error while `FLAKY_FAILURES` > 0. +struct Flaky; +impl PortableCommand for Flaky +where + D: distributed::microsvc::CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(FLAKY)) + .requires(Capability::Scheduling) + .guarded(schedule_guard::, handle_flaky) + } +} + +async fn handle_flaky( + ctx: &CausalCommandContext<'_, Ticker>, + input: TickInput, +) -> Result>, HandlerError> { + let mut current = FLAKY_FAILURES.load(Ordering::SeqCst); + let mut fail = false; + while current > 0 { + match FLAKY_FAILURES.compare_exchange( + current, + current - 1, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => { + fail = true; + break; + } + Err(actual) => current = actual, + } + } + if fail { + return Err(HandlerError::Other("transient dependency outage".into())); + } + handle_tick(ctx, input).await +} + +fn service(repo: R) -> Arc +where + AggregateRepository: distributed::microsvc::HasRepo + + distributed::microsvc::HasOutboxStore + + distributed::microsvc::ConfigurableOutboxPublisher + + Send + + Sync + + 'static, + Routes>: Sized, + AggregateRepository: + distributed::microsvc::CausalRouteDependencies, +{ + Arc::new( + Service::new().named("tickers").routes( + Routes::new() + .with_repo(AggregateRepository::::new(repo)) + .mount(Start) + .mount(Tick) + .mount(Flaky), + ), + ) +} + +static NEXT_COMMAND: AtomicU64 = AtomicU64::new(1); + +fn command_id() -> String { + let n = NEXT_COMMAND.fetch_add(1, Ordering::SeqCst); + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64; + format!( + "0190a000-{:04x}-7000-8000-{:012x}", + n & 0xffff, + nanos & 0xffff_ffff_ffff + ) +} + +async fn start(host: &dyn CommandHost, id: &str, limit: u32, period_ms: u64, command: &str) { + let mut session = Session::new(); + session.set(USER_ID_KEY, "user-1"); + session.set(ROLE_KEY, "user"); + host.invoke( + "ticker.start", + &command_id(), + json!({ "id": id, "limit": limit, "period_ms": period_ms, "tick_command": command }), + session, + VerifiedPrincipal::test_oidc("https://issuer.test", "user-1", &["tickers"]), + None, + ) + .await + .expect("start"); +} + +fn owner(id: &str) -> StreamIdentity { + StreamIdentity::new("Ticker", id).unwrap() +} + +fn fast() -> ScheduleWorkerConfig { + ScheduleWorkerConfig::default() + .with_poll_interval(Duration::from_millis(20)) + .with_claim_ttl(Duration::from_millis(400)) +} + +/// A host that commits the command and then never returns: the process +/// "crashes" between the command's commit and the schedule's settlement. +struct CrashAfterCommit { + inner: LocalCommandHost, + committed: Arc, +} + +#[async_trait] +impl CommandHost for CrashAfterCommit { + async fn invoke( + &self, + command: &str, + command_id: &str, + input: Value, + session: Session, + principal: VerifiedPrincipal, + protocol: Option, + ) -> Result { + self.inner + .invoke(command, command_id, input, session, principal, protocol) + .await?; + self.committed.fetch_add(1, Ordering::SeqCst); + std::future::pending().await + } + + async fn status( + &self, + command_id: &str, + session: &Session, + principal: VerifiedPrincipal, + protocol: Option, + ) -> Result { + self.inner + .status(command_id, session, principal, protocol) + .await + } +} + +async fn until(what: &str, timeout: Duration, mut probe: F) +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if probe().await { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("timed out waiting for {what}"); +} + +/// The scenarios, written once over any repository with a schedule worker. +macro_rules! scenarios { + ($repo_ty:ty, $fresh:expr, $reopen:expr) => { + async fn ticks(repo: &$repo_ty, id: &str) -> u32 { + use distributed::GetStream; + let Some(entity) = repo.get_stream(&owner(id)).await.unwrap() else { + return 0; + }; + entity + .events() + .iter() + .filter(|event| event.event_name == "ticker.ticked") + .count() as u32 + } + + async fn armed(repo: &$repo_ty, id: &str) -> usize { + repo.armed_schedules(&owner(id)).await.unwrap().len() + } + + #[tokio::test] + async fn scheduling_routes_need_an_attached_worker() { + let Some(ctx) = $fresh.await else { return }; + let repo: $repo_ty = ctx.repo().await; + let unattached = repo.clone(); + let refused = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| service(unattached))); + let message = refused + .err() + .and_then(|panic| panic.downcast::().ok()) + .expect("a scheduling route on a repository without a worker must not register"); + assert!(message.contains("scheduling"), "{message}"); + let (repo, _worker) = repo.with_schedule_worker(fast()); + let _ = service(repo); + } + + #[tokio::test] + async fn fires_each_arm_once_and_reports_latency() { + let Some(ctx) = $fresh.await else { return }; + let (repo, worker) = ctx.repo().await.with_schedule_worker(fast()); + let host: Arc = Arc::new(LocalCommandHost::new(service(repo.clone()))); + let started = Instant::now(); + start(host.as_ref(), "once", 1, 200, TICK).await; + let running = worker.spawn(Arc::clone(&host)); + until("the first tick", Duration::from_secs(10), || async { + ticks(&repo, "once").await == 1 + }) + .await; + let latency = started.elapsed().saturating_sub(Duration::from_millis(200)); + eprintln!("schedule latency past due (poll 20 ms): {latency:?}"); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(ticks(&repo, "once").await, 1, "an arm fires once"); + assert_eq!(armed(&repo, "once").await, 0, "a fired arm is disarmed"); + + // Periodic: each tick re-arms the next generation until the limit. + start(host.as_ref(), "periodic", 3, 50, TICK).await; + until("three ticks", Duration::from_secs(10), || async { + ticks(&repo, "periodic").await == 3 + }) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(ticks(&repo, "periodic").await, 3); + assert_eq!(armed(&repo, "periodic").await, 0); + running.stop().await; + } + + #[tokio::test] + async fn crash_between_commit_and_settle_does_not_fire_twice() { + let Some(ctx) = $fresh.await else { return }; + let (repo, crashing_worker) = ctx.repo().await.with_schedule_worker(fast()); + let committed = Arc::new(AtomicUsize::new(0)); + let crashing: Arc = Arc::new(CrashAfterCommit { + inner: LocalCommandHost::new(service(repo.clone())), + committed: Arc::clone(&committed), + }); + start( + &LocalCommandHost::new(service(repo.clone())), + "crash", + 1, + 50, + TICK, + ) + .await; + let running = crashing_worker.spawn(crashing); + until("the tick to commit", Duration::from_secs(10), || async { + committed.load(Ordering::SeqCst) == 1 + }) + .await; + running.abort(); + assert_eq!(ticks(&repo, "crash").await, 1); + assert_eq!( + armed(&repo, "crash").await, + 1, + "the crash left the arm unsettled" + ); + + // A new process: a fresh repository on the same database. + let (repo, worker) = $reopen(&ctx).await.with_schedule_worker(fast()); + let host = LocalCommandHost::new(service(repo.clone())); + let first = worker.run_once(&host).await.unwrap(); + assert!(first.is_empty(), "the crashed worker's claim still holds"); + tokio::time::sleep(Duration::from_millis(450)).await; + let replay = worker.run_once(&host).await.unwrap(); + assert_eq!(replay.fired.len(), 1, "{replay:?}"); + assert!( + replay.fired[0].replayed, + "the ledger replays the committed tick" + ); + assert_eq!( + ticks(&repo, "crash").await, + 1, + "exactly once across the crash" + ); + assert_eq!(armed(&repo, "crash").await, 0); + assert!(worker.run_once(&host).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn concurrent_workers_fire_each_schedule_once() { + let Some(ctx) = $fresh.await else { return }; + let (repo, first) = ctx.repo().await.with_schedule_worker(fast()); + let host: Arc = Arc::new(LocalCommandHost::new(service(repo.clone()))); + for index in 0..12 { + start(host.as_ref(), &format!("many-{index}"), 1, 30, TICK).await; + } + let (other_repo, second) = $reopen(&ctx).await.with_schedule_worker(fast()); + let other_host: Arc = + Arc::new(LocalCommandHost::new(service(other_repo))); + let a = first.spawn(host); + let b = second.spawn(other_host); + until("every ticker to tick", Duration::from_secs(20), || async { + let mut all = true; + for index in 0..12 { + all &= ticks(&repo, &format!("many-{index}")).await == 1; + } + all + }) + .await; + tokio::time::sleep(Duration::from_millis(500)).await; + for index in 0..12 { + assert_eq!(ticks(&repo, &format!("many-{index}")).await, 1); + assert_eq!(armed(&repo, &format!("many-{index}")).await, 0); + } + a.stop().await; + b.stop().await; + } + + #[tokio::test] + async fn retryable_failures_back_off_then_fire_once() { + let Some(ctx) = $fresh.await else { return }; + let (repo, worker) = ctx.repo().await.with_schedule_worker(fast()); + let host = LocalCommandHost::new(service(repo.clone())); + FLAKY_FAILURES.store(1, Ordering::SeqCst); + start(&host, "flaky", 1, 0, FLAKY).await; + let report = worker.run_once(&host).await.unwrap(); + assert_eq!(report.retried.len(), 1, "{report:?}"); + let retry = repo.armed_schedules(&owner("flaky")).await.unwrap(); + assert_eq!(retry[0].attempts, 1); + assert!( + worker.run_once(&host).await.unwrap().is_empty(), + "backing off" + ); + tokio::time::sleep(Duration::from_millis(1_050)).await; + let report = worker.run_once(&host).await.unwrap(); + assert_eq!(report.fired.len(), 1, "{report:?}"); + assert_eq!(ticks(&repo, "flaky").await, 1); + assert_eq!(armed(&repo, "flaky").await, 0); + } + }; +} + +#[cfg(feature = "sqlite")] +mod sqlite { + use super::*; + use distributed::SqliteRepository; + + /// A temporary database file, so a second repository can play the + /// restarted (or concurrent) process. + pub struct Db(std::path::PathBuf); + + impl Db { + pub async fn repo(&self) -> SqliteRepository { + SqliteRepository::connect_and_migrate(&format!( + "sqlite://{}?mode=rwc", + self.0.display() + )) + .await + .unwrap() + } + } + + impl Drop for Db { + fn drop(&mut self) { + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", self.0.display())); + } + } + } + + async fn fresh() -> Option { + Some(Db(std::env::temp_dir().join(format!( + "distributed-schedule-worker-{}-{}.db", + std::process::id(), + command_id() + )))) + } + + async fn reopen(db: &Db) -> SqliteRepository { + db.repo().await + } + + scenarios!(SqliteRepository, fresh(), reopen); +} + +#[cfg(feature = "postgres")] +mod postgres_backend { + use super::*; + use crate::postgres::PostgresTestSchema; + use distributed::PostgresRepository; + + pub struct Db(PostgresTestSchema); + + impl Db { + pub async fn repo(&self) -> PostgresRepository { + self.0.repository().await + } + } + + async fn fresh() -> Option { + PostgresTestSchema::create_from_env("schedule_worker", "skipping Postgres schedule worker") + .await + .map(Db) + } + + async fn reopen(db: &Db) -> PostgresRepository { + db.repo().await + } + + scenarios!(PostgresRepository, fresh(), reopen); +} + +/// The in-memory repository fires its schedules by its injected clock. +mod memory { + use super::*; + use distributed::{GetStream, InMemoryRepository, ManualClock}; + + #[tokio::test] + async fn in_memory_fires_by_the_store_clock_once_per_arm() { + let clock = ManualClock::new(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000)); + let unattached = InMemoryRepository::new().with_clock(clock.clone()); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| service(unattached.clone()))) + .is_err(), + "no worker, no Scheduling capability" + ); + let (repo, worker) = unattached.with_schedule_worker(fast()); + let host = LocalCommandHost::new(service(repo.clone())); + start(&host, "mem", 2, 1_000, TICK).await; + assert!( + worker.run_once(&host).await.unwrap().is_empty(), + "not due yet" + ); + clock.advance(Duration::from_millis(1_000)); + let first = worker.run_once(&host).await.unwrap(); + assert_eq!(first.fired.len(), 1); + assert!(!first.fired[0].replayed); + assert!( + worker.run_once(&host).await.unwrap().is_empty(), + "re-armed for later" + ); + clock.advance(Duration::from_millis(1_000)); + assert_eq!(worker.run_once(&host).await.unwrap().fired.len(), 1); + clock.advance(Duration::from_secs(60)); + assert!(worker.run_once(&host).await.unwrap().is_empty()); + let events = repo.get_stream(&owner("mem")).await.unwrap().unwrap(); + assert_eq!( + events + .events() + .iter() + .filter(|event| event.event_name == "ticker.ticked") + .count(), + 2 + ); + assert!(repo.armed_schedules(&owner("mem")).unwrap().is_empty()); + } +} From 99e75828ee0bd5871327c4fe711cce6985a04a3d Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 18:49:12 -0500 Subject: [PATCH 31/40] feat(leaderboard): board change notices over the bus for @live LeaderboardProjector's change listener only reached a GraphQL engine in the same process, so a projector elsewhere (Mob Pit's zone server owns its Redis boards) could not drive @live in the GraphQL service. Board changes now travel on the application's bus, like every other cross-process fact: LeaderboardChangePublisher (feature leaderboard-background) is a projector listener that publishes a `distributed.leaderboard.changed` event ({ "board": name }) per changed board, coalescing bursts and retrying until the bus accepts it. GraphqlEngine::leaderboard_change_handlers(group) returns bus Handlers that republish `leaderboard:{board}` on the engine's ChangeHub; each GraphQL process subscribes under its own consumer group. A notice only names the board, so duplicates and replays push nothing new. Test: a projector and an engine on separate SQLite bus connections; the applied change pushes the new top-N to the subscriber. [[tasks/gr-p29-leaderboard-graphql]] [[tasks/gr-p40-examples-e2e-coverage]] --- docs/live-query-delivery.md | 20 +++- src/graphql/leaderboard_roots.rs | 28 ++++- src/leaderboard/mod.rs | 9 +- src/leaderboard/notify.rs | 166 ++++++++++++++++++++++++++++++ tests/graphql_leaderboard/main.rs | 95 +++++++++++++++++ 5 files changed, 313 insertions(+), 5 deletions(-) create mode 100644 src/leaderboard/notify.rs diff --git a/docs/live-query-delivery.md b/docs/live-query-delivery.md index 41b3528da..e4133f26b 100644 --- a/docs/live-query-delivery.md +++ b/docs/live-query-delivery.md @@ -129,6 +129,20 @@ queries, and a failed execution is terminal. With the causal protocol on, a leaderboard result carries an incomplete snapshot (`recordsComplete: false`, `indexesComparable: false`) and live frames use `snapshot` mode with `reset: true`: leaderboard rows are never normalized -into the replica cache or resumed from cursors. The listener is in-process; -a projector in another process must forward its board changes to the -engine (for example over the bus) to drive `@live` there. +into the replica cache or resumed from cursors. + +The listener is in-process. When the projector runs in another process +(a zone server projecting kills into Redis while a GraphQL service serves +the same boards), the change goes over the application's bus: + +- the projector's process registers `LeaderboardChangePublisher::listener()` + (feature `leaderboard-background`), which publishes a + `distributed.leaderboard.changed` event (`{ "board": "" }`) per + changed board, coalescing bursts and retrying until the bus accepts it; +- the GraphQL process runs `engine.leaderboard_change_handlers(group)` with + `bus.subscribe(..)` under a consumer group of its own, so every GraphQL + process receives every notice and republishes `leaderboard:{board}` on + its `ChangeHub`. + +A notice only names the board; the subscriber re-reads its store, so +duplicate or replayed notices push nothing new. diff --git a/src/graphql/leaderboard_roots.rs b/src/graphql/leaderboard_roots.rs index e36bc143a..edb3540f5 100644 --- a/src/graphql/leaderboard_roots.rs +++ b/src/graphql/leaderboard_roots.rs @@ -455,11 +455,37 @@ fn live_stream(request: Request) -> LeaderboardLiveStream { impl GraphqlEngine { /// A [`LeaderboardChangeListener`] that refreshes `@live` leaderboard /// subscriptions of this engine: register it on the projector writing - /// the boards (`LeaderboardProjector::with_change_listener`). + /// the boards (`LeaderboardProjector::with_change_listener`) when that + /// projector runs in this process. pub fn leaderboard_change_listener(&self) -> LeaderboardChangeListener { let hub = self.inner.change_hub.clone(); Arc::new(move |board: &Board| { hub.publish(ReadModelChange::new([leaderboard_change_key(board.name())])); }) } + + /// Bus handlers that refresh `@live` leaderboard subscriptions of this + /// engine from [`LEADERBOARD_CHANGED_EVENT`](crate::leaderboard::LEADERBOARD_CHANGED_EVENT) + /// notices, for a projector running in another process (it publishes + /// them with `LeaderboardChangePublisher`). Run them with + /// `bus.subscribe(Arc::new(handlers), RunOptions::default())` under a + /// consumer `group` of this process's own, so every GraphQL process + /// receives every notice. + pub fn leaderboard_change_handlers(&self, group: impl Into) -> crate::bus::Handlers { + let hub = self.inner.change_hub.clone(); + crate::bus::Handlers::new().named(group).on_event( + crate::leaderboard::LEADERBOARD_CHANGED_EVENT, + move |message: &crate::bus::Message| { + let notice = crate::leaderboard::LeaderboardChanged::from_message(message); + let hub = hub.clone(); + async move { + let notice = notice?; + hub.publish(ReadModelChange::new([leaderboard_change_key( + ¬ice.board, + )])); + Ok(()) + } + }, + ) + } } diff --git a/src/leaderboard/mod.rs b/src/leaderboard/mod.rs index d8594c06b..b26e31849 100644 --- a/src/leaderboard/mod.rs +++ b/src/leaderboard/mod.rs @@ -33,7 +33,10 @@ //! `@live` subscriptions, the client manifest and `leaderboards.ts`), and //! [`LeaderboardProjector::with_change_listener`] + //! `GraphqlEngine::leaderboard_change_listener` refresh live queries after an -//! applied change. See `docs/live-query-delivery.md`. +//! applied change. Across processes, `LeaderboardChangePublisher` publishes +//! [`LEADERBOARD_CHANGED_EVENT`] notices on the bus and +//! `GraphqlEngine::leaderboard_change_handlers` consumes them. See +//! `docs/live-query-delivery.md`. //! //! # How it plugs into projections //! @@ -84,6 +87,7 @@ //! makes overlapping replay safe. mod in_memory; +mod notify; mod projector; #[cfg(feature = "leaderboard-background")] @@ -96,6 +100,9 @@ use std::fmt; use async_trait::async_trait; pub use in_memory::InMemoryLeaderboard; +#[cfg(feature = "leaderboard-background")] +pub use notify::LeaderboardChangePublisher; +pub use notify::{LeaderboardChanged, LEADERBOARD_CHANGED_EVENT}; pub use projector::{ LeaderboardChange, LeaderboardChangeListener, LeaderboardProjection, LeaderboardProjector, LeaderboardRebuild, ProjectionReport, diff --git a/src/leaderboard/notify.rs b/src/leaderboard/notify.rs new file mode 100644 index 000000000..25670e460 --- /dev/null +++ b/src/leaderboard/notify.rs @@ -0,0 +1,166 @@ +//! Board change notices on the bus: `@live` boards across processes. +//! +//! A [`LeaderboardProjector`](super::LeaderboardProjector) calls its change +//! listeners in its own process. When the GraphQL engine serving the boards +//! runs elsewhere (a zone server projects kills into Redis; a GraphQL +//! service reads the same Redis), the projector's process publishes a +//! [`LEADERBOARD_CHANGED_EVENT`] on the application's bus and the GraphQL +//! process subscribes to it: +//! +//! ```rust,ignore +//! // projector process +//! let notices = LeaderboardChangePublisher::spawn(BusPublisher::new(Arc::new(bus))); +//! let projector = LeaderboardProjector::new(store, projection) +//! .with_change_listener(notices.listener()); +//! +//! // GraphQL process (its own consumer group, so every process gets every notice) +//! let handlers = engine.leaderboard_change_handlers("mobpit-service-boards"); +//! tokio::spawn(async move { bus.subscribe(Arc::new(handlers), RunOptions::default()).await }); +//! ``` +//! +//! A notice names a board, nothing else: the subscriber re-reads its store +//! (hash-gated, so an unchanged top-N pushes nothing). Notices are hints, so +//! duplicates and replays are harmless; the publisher retries until the bus +//! accepts each one, so a transient bus outage delays a push but does not +//! lose it. + +use serde::{Deserialize, Serialize}; + +use super::Board; +use crate::bus::{Message, MessageKind, TransportError}; + +/// Event name of a board change notice. +pub const LEADERBOARD_CHANGED_EVENT: &str = "distributed.leaderboard.changed"; + +/// Payload of a [`LEADERBOARD_CHANGED_EVENT`]: the board whose scores changed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LeaderboardChanged { + /// Board name (`Board::name`). + pub board: String, +} + +impl LeaderboardChanged { + /// The notice for `board`. + pub fn new(board: &Board) -> Self { + Self { + board: board.name().to_string(), + } + } + + /// The bus message of this notice (a fan-out event with a fresh id). + pub fn to_message(&self) -> Message { + let payload = serde_json::to_vec(self).expect("a board name serializes"); + Message::new(LEADERBOARD_CHANGED_EVENT, MessageKind::Event, payload) + .with_id(uuid::Uuid::now_v7().to_string()) + } + + /// Decode a delivered notice. A malformed one is a permanent failure. + pub fn from_message(message: &Message) -> Result { + let notice: Self = serde_json::from_slice(&message.payload).map_err(|error| { + TransportError::permanent(format!("malformed {LEADERBOARD_CHANGED_EVENT}: {error}")) + })?; + Board::high_first(notice.board.clone()).map_err(|error| { + TransportError::permanent(format!("malformed {LEADERBOARD_CHANGED_EVENT}: {error}")) + })?; + Ok(notice) + } +} + +#[cfg(feature = "leaderboard-background")] +pub use publisher::LeaderboardChangePublisher; + +#[cfg(feature = "leaderboard-background")] +mod publisher { + use std::collections::BTreeSet; + use std::sync::Arc; + use std::time::Duration; + + use tokio::sync::mpsc; + + use super::super::{Board, LeaderboardChangeListener}; + use super::LeaderboardChanged; + use crate::bus::MessagePublisher; + + const RETRY_FIRST: Duration = Duration::from_millis(100); + const RETRY_MAX: Duration = Duration::from_secs(5); + + /// Publishes a [`LeaderboardChanged`] notice on the bus for every board a + /// projector reports (its [`listener`](Self::listener)). Publishing runs + /// on a Tokio task: the listener never blocks the projector. Changes to + /// the same board that queue up while a publish is in flight coalesce + /// into one notice; a failed publish is retried with backoff (100 ms up + /// to 5 s) until the bus accepts it. + #[derive(Clone, Debug)] + pub struct LeaderboardChangePublisher { + tx: mpsc::UnboundedSender, + } + + impl LeaderboardChangePublisher { + /// Start publishing through `publisher` on the current Tokio runtime. + /// The task ends when every clone of this handle (and of its + /// listeners) is dropped and the queue is drained. + pub fn spawn

    (publisher: P) -> Self + where + P: MessagePublisher + 'static, + { + let (tx, mut rx) = mpsc::unbounded_channel::(); + tokio::spawn(async move { + while let Some(first) = rx.recv().await { + let mut boards = BTreeSet::from([first]); + while let Ok(next) = rx.try_recv() { + boards.insert(next); + } + for board in boards { + let message = LeaderboardChanged { board }.to_message(); + let mut delay = RETRY_FIRST; + while let Err(error) = publisher.publish(message.clone()).await { + eprintln!("leaderboard change notice: {error}; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(RETRY_MAX); + } + } + } + }); + Self { tx } + } + + /// A projector change listener that queues a notice per changed board. + pub fn listener(&self) -> LeaderboardChangeListener { + let tx = self.tx.clone(); + Arc::new(move |board: &Board| { + let _ = tx.send(board.name().to_string()); + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn notices_round_trip_and_reject_garbage() { + let board = Board::high_first("mobpit.xp").unwrap(); + let message = LeaderboardChanged::new(&board).to_message(); + assert_eq!(message.name, LEADERBOARD_CHANGED_EVENT); + assert_eq!(message.kind, MessageKind::Event); + assert!(message.id.is_some()); + assert_eq!( + LeaderboardChanged::from_message(&message).unwrap().board, + "mobpit.xp" + ); + let garbage = Message::new( + LEADERBOARD_CHANGED_EVENT, + MessageKind::Event, + b"{}".to_vec(), + ); + assert!(LeaderboardChanged::from_message(&garbage).is_err()); + let bad_board = Message::new( + LEADERBOARD_CHANGED_EVENT, + MessageKind::Event, + br#"{"board":"bad{tag}"}"#.to_vec(), + ); + assert!(LeaderboardChanged::from_message(&bad_board).is_err()); + } +} diff --git a/tests/graphql_leaderboard/main.rs b/tests/graphql_leaderboard/main.rs index 2a53fa6c5..a70316f5d 100644 --- a/tests/graphql_leaderboard/main.rs +++ b/tests/graphql_leaderboard/main.rs @@ -480,3 +480,98 @@ fn the_cli_compiles_a_framework_leaderboard_manifest_into_the_ts_client() { .contents .contains(r#"document: "query XpLeaderboardAround($radius: Int) { xpLeaderboardAround(radius: $radius) { rank member score } }","#)); } + +/// The projector runs in another process (here: another bus connection and +/// no shared engine): it publishes `distributed.leaderboard.changed` on the +/// bus, and the GraphQL process's bus handlers drive `@live` from it. +#[cfg(feature = "leaderboard-background")] +#[tokio::test] +async fn live_boards_follow_a_projector_in_another_process_over_the_bus() { + use distributed::bus::SqliteBus; + use distributed::bus::{BusConsumer, RunOptions}; + use distributed::leaderboard::LeaderboardChangePublisher; + use distributed::outbox_worker::BusPublisher; + + let path = std::env::temp_dir().join(format!( + "distributed-board-notices-{}-{}.db", + std::process::id(), + UNIX_EPOCH.elapsed().unwrap().as_nanos() + )); + let url = format!("sqlite://{}?mode=rwc", path.display()); + let open = || async { + let pool = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(2) + .connect_with( + url.parse::() + .unwrap() + .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) + .busy_timeout(Duration::from_secs(5)), + ) + .await + .unwrap(); + let bus = SqliteBus::new(pool).with_idle_poll(Duration::from_millis(20)); + bus.ensure_tables().await.unwrap(); + bus + }; + + // Shared board storage (Redis in production). + let store = Arc::new(InMemoryLeaderboard::new()); + seed(&store).await; + + // GraphQL process: engine + bus handlers under its own consumer group. + let engine = engine(store.clone()).await; + let handlers = engine.leaderboard_change_handlers("graphql-1-boards"); + let graphql_bus = open().await; + let consumer = tokio::spawn(async move { + graphql_bus + .subscribe(Arc::new(handlers), RunOptions::default()) + .await + }); + + // Projector process: no engine, only the bus. + let projector_bus = open().await; + let notices = LeaderboardChangePublisher::spawn(BusPublisher::new(Arc::new(projector_bus))); + let projection = + |event: &DomainEventOccurrence| -> Result, LeaderboardError> { + let body: XpGained = event + .decode_body() + .map_err(|error| LeaderboardError::invalid(error.to_string()))?; + Ok(vec![LeaderboardChange::new( + xp_board(), + ScoreSubmission::new(body.player, f64::from(body.xp), ScorePolicy::Increment), + )]) + }; + let projector = LeaderboardProjector::new( + store.clone() as Arc, + Arc::new(projection), + ) + .with_change_listener(notices.listener()); + + let mut stream = engine.execute_stream( + &session("user", Some("dan")), + Request::new("subscription { xpLeaderboard(first: 2) { member score } }"), + ); + let first = tokio::time::timeout(Duration::from_secs(5), stream.next()) + .await + .unwrap() + .unwrap(); + assert_eq!( + serde_json::to_value(&first.data).unwrap()["xpLeaderboard"][0]["member"], + "ann" + ); + + projector.apply_event(&gained(1, "dan", 100)).await.unwrap(); + let pushed = tokio::time::timeout(Duration::from_secs(10), stream.next()) + .await + .expect("a projector in another process drives @live over the bus") + .unwrap(); + assert_eq!( + serde_json::to_value(&pushed.data).unwrap()["xpLeaderboard"], + serde_json::json!([{ "member": "dan", "score": 120.0 }, { "member": "ann", "score": 50.0 }]) + ); + + consumer.abort(); + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); + } +} From b847e7179288e4d7bfeae79715b239be21d3a1e9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 19:12:12 -0500 Subject: [PATCH 32/40] feat: leaderboards in application manifests and SvelteKit helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An application manifest whose surface had leaderboard roots failed to build ("surface contract material"): SurfaceSpec carried no leaderboard section, so the contract rebuilt from the spec never matched the Surface IR's. SurfaceSpec now has `leaderboards` (the canonical sections, absent when empty so existing manifests keep their bytes and fingerprints), and the reconstruction includes them. JS: the replica GraphQL transport gains `query(document, variables)` and `live(document, variables, observer)` for documents that never enter the replica (server-ranked boards), over the same endpoint, credential and WebSocket. `@hops-ops/distributed/sveltekit` adds `loadDistributedLeaderboard` (SSR load of a generated `…Leaderboard` operation) and `useDistributedLeaderboard` / `createDistributedLeaderboardStore` (a store seeded with the SSR result that follows the operation's `@live` document). BREAKING CHANGE: ReplicaGraphqlTransport requires `query` and `live`. [[tasks/gr-p29-leaderboard-graphql]] [[tasks/gr-p40-examples-e2e-coverage]] --- js/src/replica/graphql-transport.ts | 85 ++++++++++- js/src/replica/index.ts | 1 + js/src/sveltekit/index.ts | 10 ++ js/src/sveltekit/leaderboard.ts | 195 ++++++++++++++++++++++++ js/tests/sveltekit-leaderboard.test.mjs | 110 +++++++++++++ src/application/manifest.rs | 9 +- src/application/module.rs | 11 ++ tests/graphql_leaderboard/main.rs | 35 +++++ 8 files changed, 451 insertions(+), 5 deletions(-) create mode 100644 js/src/sveltekit/leaderboard.ts create mode 100644 js/tests/sveltekit-leaderboard.test.mjs diff --git a/js/src/replica/graphql-transport.ts b/js/src/replica/graphql-transport.ts index cc2f424f1..5c6654d1f 100644 --- a/js/src/replica/graphql-transport.ts +++ b/js/src/replica/graphql-transport.ts @@ -28,14 +28,42 @@ export type ReplicaGraphqlTransportOptions = Readonly<{ webSocket?: WebSocketConstructor; }>; +/** + * Documents that never enter the replica cache (server-ranked leaderboards): + * the same endpoint, credentials and WebSocket as replica work, without a + * replica artifact. + */ +export type ReplicaDocumentTransport = { + /** Run one query document. */ + query< + TData = Record, + TVariables extends GraphqlVariables = GraphqlVariables + >( + document: string, + variables: TVariables, + signal?: AbortSignal + ): Promise>; + /** Run one subscription document; returns its close function. */ + live< + TData = Record, + TVariables extends GraphqlVariables = GraphqlVariables + >( + document: string, + variables: TVariables, + observer: ReplicaLiveObserver + ): () => void; +}; + /** * One concrete GraphQL transport for generated replica reads, live operations, - * command dispatch, and command-status recovery. + * command dispatch, command-status recovery, and replica-free documents. * * The transport is intentionally framework-neutral. Framework adapters inject * their request/session lifecycle but never duplicate GraphQL protocol logic. */ -export type ReplicaGraphqlTransport = ReplicaTransport & ReplicaCommandTransport; +export type ReplicaGraphqlTransport = ReplicaTransport & + ReplicaCommandTransport & + ReplicaDocumentTransport; export function createReplicaGraphqlTransport( options: ReplicaGraphqlTransportOptions @@ -117,6 +145,49 @@ export function createReplicaGraphqlTransport( return subscribeReplicaOperation(options, request, observer); }, + async query< + TData = Record, + TVariables extends GraphqlVariables = GraphqlVariables + >( + document: string, + variables: TVariables, + signal?: AbortSignal + ): Promise> { + const auth = await options.getAuth(); + const result = await requestGraphql( + options.getUrl(), + document, + auth, + variables, + { + ...(options.fetch === undefined ? {} : { fetch: options.fetch }), + ...(signal === undefined ? {} : { signal }) + } + ); + return Object.freeze({ + ...(result.data === undefined ? {} : { data: result.data }), + ...(result.errors === undefined ? {} : { errors: result.errors }), + ...(result.extensions === undefined + ? {} + : { extensions: result.extensions }) + }); + }, + + live< + TData = Record, + TVariables extends GraphqlVariables = GraphqlVariables + >( + document: string, + variables: TVariables, + observer: ReplicaLiveObserver + ): () => void { + return subscribeReplicaOperation( + options, + { document, variables }, + observer + ); + }, + dispatch( request: ReplicaCommandTransportRequest ): Promise { @@ -141,12 +212,20 @@ export function createReplicaGraphqlTransport( }); } +type LiveRequest = { + readonly document: string; + readonly variables: TVariables; + readonly extensions?: ReplicaTransportRequest['extensions']; + readonly resume?: ReplicaTransportRequest['resume']; + readonly signal?: AbortSignal; +}; + function subscribeReplicaOperation< TData, TVariables extends GraphqlVariables >( options: ReplicaGraphqlTransportOptions, - request: ReplicaTransportRequest, + request: LiveRequest, observer: ReplicaLiveObserver ): () => void { let unsubscribe = (): void => undefined; diff --git a/js/src/replica/index.ts b/js/src/replica/index.ts index c54b8a8d8..62ee54bac 100644 --- a/js/src/replica/index.ts +++ b/js/src/replica/index.ts @@ -38,6 +38,7 @@ export type { export { createReplicaGraphqlTransport } from './graphql-transport.js'; export { createReplicaUuidV7 } from './command-id.js'; export type { + ReplicaDocumentTransport, ReplicaGraphqlTransport, ReplicaGraphqlTransportOptions } from './graphql-transport.js'; diff --git a/js/src/sveltekit/index.ts b/js/src/sveltekit/index.ts index 430bf2834..c3940ecf9 100644 --- a/js/src/sveltekit/index.ts +++ b/js/src/sveltekit/index.ts @@ -1,4 +1,14 @@ export { authFromPageData, type PageGraphqlData } from './auth.js'; +export { + createDistributedLeaderboardStore, + loadDistributedLeaderboard, + useDistributedLeaderboard, + type DistributedLeaderboardLive, + type DistributedLeaderboardOperation, + type DistributedLeaderboardSnapshot, + type DistributedLeaderboardStore, + type UseDistributedLeaderboardOptions +} from './leaderboard.js'; export { distributedReloadLifecycle, registerDistributedReloadClient, diff --git a/js/src/sveltekit/leaderboard.ts b/js/src/sveltekit/leaderboard.ts new file mode 100644 index 000000000..24a816981 --- /dev/null +++ b/js/src/sveltekit/leaderboard.ts @@ -0,0 +1,195 @@ +import { requestGraphql, type FetchLike } from '../request.js'; +import type { GqlAuth, GqlError, GraphqlVariables } from '../types.js'; +import type { ReplicaDocumentTransport } from '../replica/graphql-transport.js'; +import { useDistributedSvelteKitClient } from './context.js'; + +/** + * A generated leaderboard operation (`leaderboards.ts`, re-exported by + * `$distributed`), e.g. `XpLeaderboardLeaderboard`. + */ +export type DistributedLeaderboardOperation = Readonly<{ + leaderboard: string; + operation: string; + document: string; + /** The `@live` subscription twin, or null (`…Rank(member)` is never live). */ + liveDocument: string | null; +}>; + +export type DistributedLeaderboardLive = 'off' | 'connecting' | 'active' | 'error'; + +/** The Svelte store contract (`$store` in components). */ +export type DistributedLeaderboardStore = Readonly<{ + subscribe(run: (value: T) => void): () => void; +}>; + +/** A readable store: `start` runs on the first subscriber, its stop on the last. */ +function readable( + value: T, + start: (set: (value: T) => void) => (() => void) | undefined +): DistributedLeaderboardStore { + let current = value; + const subscribers = new Set<(value: T) => void>(); + let stop: (() => void) | undefined; + const set = (next: T): void => { + current = next; + for (const run of [...subscribers]) run(current); + }; + return Object.freeze({ + subscribe(run: (value: T) => void): () => void { + subscribers.add(run); + if (subscribers.size === 1) stop = start(set); + run(current); + return () => { + if (!subscribers.delete(run) || subscribers.size > 0) return; + stop?.(); + stop = undefined; + }; + } + }); +} + +/** One leaderboard result as a component reads it. */ +export type DistributedLeaderboardSnapshot = Readonly<{ + /** The latest result (`undefined` until the first one arrives). */ + data: TData | undefined; + /** The latest GraphQL or transport error, if any. */ + error: string | null; + live: DistributedLeaderboardLive; +}>; + +export type UseDistributedLeaderboardOptions = Readonly<{ + /** The server-loaded result (`loadDistributedLeaderboard`), so SSR renders it. */ + initial?: TData; + /** Subscribe to the live document in the browser (default true). */ + live?: boolean; + /** Whether this runs in a browser (default: `typeof window !== 'undefined'`). */ + browser?: boolean; +}>; + +function message(errors: readonly GqlError[] | undefined): string | null { + if (errors === undefined || errors.length === 0) return null; + return errors.map((error) => error.message).join('; '); +} + +/** + * Server-side load of a leaderboard operation (SvelteKit `load`), with the + * same endpoint and credential the generated client uses. Leaderboard rows + * are server-ranked and never enter the replica cache, so they are passed to + * the component as page data and handed to {@link useDistributedLeaderboard} + * as `initial`. Throws on a GraphQL or HTTP error. + */ +export async function loadDistributedLeaderboard< + TData, + TVariables extends GraphqlVariables = GraphqlVariables +>( + operation: DistributedLeaderboardOperation, + variables: TVariables, + options: Readonly<{ url: string; auth: GqlAuth; fetch?: FetchLike }> +): Promise { + const result = await requestGraphql( + options.url, + operation.document, + options.auth, + variables, + options.fetch === undefined ? {} : { fetch: options.fetch } + ); + const error = message(result.errors); + if (error !== null || result.data === undefined || result.data === null) { + throw new Error( + `leaderboard ${operation.leaderboard} ${operation.operation}: ${error ?? `HTTP ${result.status}`}` + ); + } + return result.data; +} + +/** + * A readable store of one leaderboard operation for the nearest generated + * client (call during component initialization). It starts from `initial` + * (or runs the query in the browser), then follows the operation's `@live` + * subscription over the client's WebSocket; every push replaces the whole + * result (server-ranked rows are never merged into the replica). + */ +export function useDistributedLeaderboard< + TData, + TVariables extends GraphqlVariables = GraphqlVariables +>( + operation: DistributedLeaderboardOperation, + variables: TVariables, + options: UseDistributedLeaderboardOptions = {} +): DistributedLeaderboardStore> { + return createDistributedLeaderboardStore( + useDistributedSvelteKitClient().transport, + operation, + variables, + options + ); +} + +/** {@link useDistributedLeaderboard} over an explicit transport. */ +export function createDistributedLeaderboardStore< + TData, + TVariables extends GraphqlVariables = GraphqlVariables +>( + transport: ReplicaDocumentTransport, + operation: DistributedLeaderboardOperation, + variables: TVariables, + options: UseDistributedLeaderboardOptions = {} +): DistributedLeaderboardStore> { + const inBrowser = options.browser ?? typeof window !== 'undefined'; + const initial: DistributedLeaderboardSnapshot = Object.freeze({ + data: options.initial, + error: null, + live: 'off' + }); + return readable(initial, (set) => { + if (!inBrowser) return undefined; + let current = initial; + const update = (next: Partial>): void => { + current = Object.freeze({ ...current, ...next }); + set(current); + }; + const controller = new AbortController(); + let close = (): void => undefined; + if (options.initial === undefined) { + transport + .query(operation.document, variables, controller.signal) + .then((result) => { + if (controller.signal.aborted) return; + // A live result that arrived first is newer: keep it. + if (current.data !== undefined && current.live === 'active') return; + update({ + ...(result.data === undefined || result.data === null + ? {} + : { data: result.data }), + error: message(result.errors) + }); + }) + .catch((error: unknown) => { + if (!controller.signal.aborted) update({ error: String(error) }); + }); + } + if (options.live !== false && operation.liveDocument !== null) { + update({ live: 'connecting' }); + close = transport.live(operation.liveDocument, variables, { + next(result) { + const error = message(result.errors); + if (result.data === undefined || result.data === null) { + update({ error, live: error === null ? current.live : 'error' }); + return; + } + update({ data: result.data, error, live: 'active' }); + }, + error(error) { + update({ error: String(error), live: 'error' }); + }, + complete() { + update({ live: 'off' }); + } + }); + } + return () => { + controller.abort(); + close(); + }; + }); +} diff --git a/js/tests/sveltekit-leaderboard.test.mjs b/js/tests/sveltekit-leaderboard.test.mjs new file mode 100644 index 000000000..db75f15cf --- /dev/null +++ b/js/tests/sveltekit-leaderboard.test.mjs @@ -0,0 +1,110 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { createReplicaGraphqlTransport } from '../dist/replica/index.js'; +import { + createDistributedLeaderboardStore, + loadDistributedLeaderboard +} from '../dist/sveltekit/index.js'; + +const op = Object.freeze({ + leaderboard: 'xp', + operation: 'top', + document: 'query XpLeaderboard($first: Int) { xpLeaderboard(first: $first) { rank member score } }', + liveDocument: + 'subscription XpLeaderboard_Live($first: Int) { xpLeaderboard(first: $first) { rank member score } }' +}); + +function jsonResponse(body, status = 200) { + return { status, statusText: 'OK', text: async () => JSON.stringify(body) }; +} + +const tick = () => new Promise((resolve) => setImmediate(resolve)); + +test('replica-free documents share the transport endpoint and credential', async () => { + const calls = []; + const transport = createReplicaGraphqlTransport({ + getUrl: () => 'http://api.test/graphql', + getAuth: async () => ({ accessToken: 'token-1' }), + fetch: async (url, init) => { + calls.push({ url, init, body: JSON.parse(init.body) }); + return jsonResponse({ data: { xpLeaderboard: [{ rank: 0, member: 'ana', score: 10 }] } }); + } + }); + const result = await transport.query(op.document, { first: 5 }); + assert.deepEqual(result.data.xpLeaderboard[0].member, 'ana'); + assert.equal(calls[0].url, 'http://api.test/graphql'); + assert.equal(calls[0].init.headers.authorization ?? calls[0].init.headers.Authorization, 'Bearer token-1'); + assert.deepEqual(calls[0].body.variables, { first: 5 }); + + const loaded = await loadDistributedLeaderboard(op, { first: 5 }, { + url: 'http://api.test/graphql', + auth: { accessToken: 'token-1' }, + fetch: async () => jsonResponse({ data: { xpLeaderboard: [] } }) + }); + assert.deepEqual(loaded, { xpLeaderboard: [] }); + await assert.rejects( + loadDistributedLeaderboard(op, {}, { + url: 'http://api.test/graphql', + auth: {}, + fetch: async () => jsonResponse({ errors: [{ message: 'FORBIDDEN' }] }) + }), + /FORBIDDEN/ + ); +}); + +test('a leaderboard store starts from the server result and follows live pushes', async () => { + let observer; + let closed = 0; + const transport = { + query: async () => { + throw new Error('initial data was given; no query expected'); + }, + live(document, variables, next) { + assert.equal(document, op.liveDocument); + assert.deepEqual(variables, { first: 2 }); + observer = next; + return () => { + closed += 1; + }; + } + }; + const initial = { xpLeaderboard: [{ rank: 0, member: 'ana', score: 10 }] }; + const store = createDistributedLeaderboardStore(transport, op, { first: 2 }, { initial, browser: true }); + const seen = []; + const unsubscribe = store.subscribe((value) => seen.push(value)); + assert.equal(seen.at(-1).data, initial); + assert.equal(seen.at(-1).live, 'connecting'); + observer.next({ data: { xpLeaderboard: [{ rank: 0, member: 'bob', score: 30 }] } }); + assert.equal(seen.at(-1).data.xpLeaderboard[0].member, 'bob'); + assert.equal(seen.at(-1).live, 'active'); + observer.error(new Error('socket closed')); + assert.equal(seen.at(-1).live, 'error'); + unsubscribe(); + assert.equal(closed, 1); +}); + +test('without initial data the browser store queries; on the server it stays idle', async () => { + let queries = 0; + const transport = { + query: async () => { + queries += 1; + return { data: { xpLeaderboard: [{ rank: 0, member: 'cat', score: 5 }] } }; + }, + live: () => () => undefined + }; + const server = createDistributedLeaderboardStore(transport, op, {}, { browser: false }); + const serverValues = []; + server.subscribe((value) => serverValues.push(value))(); + assert.equal(queries, 0); + assert.equal(serverValues[0].data, undefined); + + const browser = createDistributedLeaderboardStore(transport, op, {}, { browser: true, live: false }); + const values = []; + const stop = browser.subscribe((value) => values.push(value)); + await tick(); + assert.equal(queries, 1); + assert.equal(values.at(-1).data.xpLeaderboard[0].member, 'cat'); + assert.equal(values.at(-1).live, 'off'); + stop(); +}); diff --git a/src/application/manifest.rs b/src/application/manifest.rs index 15b28795d..4d4869c6e 100644 --- a/src/application/manifest.rs +++ b/src/application/manifest.rs @@ -1832,7 +1832,7 @@ fn surface_contract_from_spec(surface: &SurfaceSpec) -> ApplicationResult>(); - Ok(canonical_json(&serde_json::json!({ + let mut contract = serde_json::json!({ "version": 1, "selection": selection, "dialect": surface.dialect, @@ -1847,7 +1847,12 @@ fn surface_contract_from_spec(surface: &SurfaceSpec) -> ApplicationResult>, pub commands_attached: bool, pub projectors_attached: bool, + /// Canonical leaderboard sections (`LeaderboardSurface`), in contract + /// order. Absent when empty, so manifests without leaderboards keep + /// their bytes and fingerprints. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub leaderboards: Vec, /// Canonical lossless Surface IR snapshot. It is the identity authority /// for SDL/client compilation and must never be reconstructed from tables. pub contract: serde_json::Value, @@ -344,6 +349,7 @@ impl SurfaceSpec { comparison_ops: BTreeMap::new(), commands_attached: false, projectors_attached: false, + leaderboards: Vec::new(), contract: serde_json::json!({ "version": 1, "selection": {"kind": "catalog"}, @@ -492,6 +498,11 @@ impl SurfaceSpec { comparison_ops: surface.comparison_ops.clone(), commands_attached: surface.commands_attached, projectors_attached: surface.projectors_attached, + leaderboards: surface + .leaderboards + .iter() + .map(crate::graphql::surface::LeaderboardSurface::canonical_value) + .collect(), contract: surface .canonical_contract_value() .map_err(ApplicationError::InvalidSpec)?, diff --git a/tests/graphql_leaderboard/main.rs b/tests/graphql_leaderboard/main.rs index a70316f5d..4247a92cd 100644 --- a/tests/graphql_leaderboard/main.rs +++ b/tests/graphql_leaderboard/main.rs @@ -575,3 +575,38 @@ async fn live_boards_follow_a_projector_in_another_process_over_the_bus() { let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); } } + +#[test] +fn an_application_manifest_carries_and_validates_its_leaderboards() { + let spec = distributed::SurfaceSpec::from_surface("boards", &surfaces()).unwrap(); + assert_eq!(spec.leaderboards.len(), 2); + let built = distributed::Application::new("boards") + .surface(spec.clone()) + .build() + .expect("a surface with leaderboards validates"); + let json = serde_json::to_value(built.manifest()).unwrap(); + assert!(json.to_string().contains("\"leaderboards\"")); + + // Tampering with the leaderboard section breaks the contract identity. + let mut tampered = spec; + tampered.leaderboards.pop(); + assert!(distributed::Application::new("boards") + .surface(tampered) + .build() + .is_err()); + + // Without leaderboards the section is absent (existing manifests keep + // their bytes). + let plain = distributed::SurfaceSpec::from_surface( + "plain", + &build_surface( + &distributed::ReadModelCatalog::new("plain").tables, + &SurfaceOptions::sqlite(), + ) + .unwrap(), + ) + .unwrap(); + assert!(!serde_json::to_string(&plain) + .unwrap() + .contains("leaderboards")); +} From f9c199a811b4f7ec0a833c011066d74fa1871485 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 19:12:13 -0500 Subject: [PATCH 33/40] feat(game)!: Mob Pit SvelteKit shows the Redis board live via GraphQL The SvelteKit leaderboard was a SQL query over player_stats. It is now the zone server's Redis XP board, served by mobpit-service as GraphQL board roots (xp/kills: Leaderboard, LeaderboardAround, MyRank) and rendered with the generated XpLeaderboardLeaderboard operation: SSR-loaded in the pit page load, then `@live`. The zone server's projector publishes `distributed.leaderboard.changed` on the projection bus (once attached); mobpit-service subscribes under its own consumer group and pushes. Display names still come from the player_stats read model (Leaderboard.graphql island). Board names and the Redis key prefix moved to mobpit_crawler so both processes agree; the zone server takes --redis-prefix / MOBPIT_REDIS_PREFIX. tests/projection.rs now also proves a board push that reaches the engine only through the bus notices; pit.test.mjs runs on Redis with a per-run prefix and checks the second player's board is live. BREAKING CHANGE: mobpit-service requires REDIS_URL, and `run.sh sveltekit` / `run.sh smoke` require REDIS_URL (CI sets it). [[tasks/gr-p40-examples-e2e-coverage]] [[tasks/gr-p29-leaderboard-graphql]] --- .github/workflows/integration-game-demos.yaml | 3 + examples/game/README.md | 44 +++++++++--- examples/game/run.sh | 29 ++++++-- examples/game/server/crawler/src/lib.rs | 7 ++ examples/game/server/service/Cargo.toml | 4 +- examples/game/server/service/src/lib.rs | 71 +++++++++++++++++-- examples/game/server/service/src/main.rs | 10 ++- .../game/server/service/tests/projection.rs | 65 +++++++++++++++-- examples/game/server/src/common/relay.rs | 23 +++++- examples/game/server/src/mobpit/board.rs | 25 ++++--- examples/game/server/src/mobpit/mod.rs | 4 ++ examples/game/server/src/mobpit/world.rs | 17 +++-- .../sveltekit/src/lib/pit/Leaderboard.graphql | 10 +-- .../sveltekit/src/lib/pit/Leaderboard.svelte | 32 ++++++--- .../sveltekit/src/routes/pit/+page.server.ts | 19 ++++- .../sveltekit/src/routes/pit/+page.svelte | 2 +- examples/game/sveltekit/tests/pit.test.mjs | 23 ++++-- 17 files changed, 328 insertions(+), 60 deletions(-) diff --git a/.github/workflows/integration-game-demos.yaml b/.github/workflows/integration-game-demos.yaml index f2ee5012a..2c1304964 100644 --- a/.github/workflows/integration-game-demos.yaml +++ b/.github/workflows/integration-game-demos.yaml @@ -61,6 +61,9 @@ jobs: working-directory: examples/game/sveltekit run: npm ci --no-audit --no-fund && npx playwright install --with-deps chromium - name: Run examples/game smoke + env: + # The SvelteKit leaderboard is the Redis board (GraphQL board roots). + REDIS_URL: redis://127.0.0.1:6379 run: examples/game/run.sh smoke - name: Leaderboard conformance and GraphQL boards against real Redis env: diff --git a/examples/game/README.md b/examples/game/README.md index c33bde557..bc3f1a98a 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -14,7 +14,7 @@ guests and lets any page change the network simulation. | **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | none (stateless: score is per session) | | **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | celld: `gem-pouch/{player}`, `content-log/{zone}` | | **Endless Grid** | `./run.sh grid` | Seeded procedural world (terrain/biomes/props generated bit-identically in Rust and TS, nothing static replicated), 32 m cells as zones, observer connections to the 8 neighbours, handoff across borders **and hosts** (2 processes, hash ring, Redirect + transfer tickets), hibernation to celld and restore | celld: `zone-lease/{zone}`, `zone-state/{zone}` (hibernation) | -| **Mob Pit · SvelteKit** | `./run.sh sveltekit` | The same Mob Pit server behind a SvelteKit app: Auth.js session sign-in, admission tickets minted from the session (`createGameTicketHandler`), the game client and WASM step generated by the Vite plugin (`games`), a runes HUD (`useGame`) next to a plain canvas loop, and the settled kills as a business-side read model: Crawler domain events projected by `mobpit-service` into SQL and read through generated `@load @live` GraphQL (SSR, then live). See [SvelteKit](#sveltekit) | celld (the Mob Pit cells) + `mobpit-service`'s SQL read model | +| **Mob Pit · SvelteKit** | `REDIS_URL=… ./run.sh sveltekit` | The same Mob Pit server behind a SvelteKit app: Auth.js session sign-in, admission tickets minted from the session (`createGameTicketHandler`), the game client and WASM step generated by the Vite plugin (`games`), a runes HUD (`useGame`) next to a plain canvas loop, the settled kills as a business-side read model (Crawler domain events projected by `mobpit-service` into SQL, read through generated `@load @live` GraphQL), and the zone server's Redis XP board through GraphQL board roots, live across processes over the bus. See [SvelteKit](#sveltekit) | celld (the Mob Pit cells) + `mobpit-service`'s SQL read model + the Redis boards | | **Boss Ripple** | `./run.sh ripple` | Zone hierarchy realm › city › borough › hood (`DottedTopology`), per-tier tick rates (2/5/10/30 Hz), boss defeats propagating up as tier-up events (each hop on another host), parent broadcasts reaching players through observer connections | celld: `zone-lease/{zone}`, `zone-state/{zone}` (hibernation) | `./run.sh smoke` runs every demo's smoke tests (and checks the generated @@ -238,7 +238,9 @@ hood to fight there. ## SvelteKit ```sh -examples/game/run.sh sveltekit # → http://localhost:5573/ — sign in with any username +docker compose up -d redis # at the repo root: the leaderboard is the Redis board +REDIS_URL=redis://127.0.0.1:6379 examples/game/run.sh sveltekit +# → http://localhost:5573/ — sign in with any username ``` `sveltekit/` is Mob Pit as a SvelteKit app over two backends: the Mob Pit @@ -246,8 +248,9 @@ zone server (gameplay) and `mobpit-service` (the business-side read model). The script starts `celld dev` with the Mob Pit cells (`cells/`; state in `.data/mobpit-sveltekit/celld`), then `game-demos mobpit` (no `--dev`) on `GAME_DEMOS_PORT` (default 2967 here; data in `.data/mobpit-sveltekit`) with -a fresh ticket secret and `--projection-bus sqlite:…/bus.db` (celld's outbox -Queue relays the Crawler domain events to it), then runs +a fresh ticket secret, `--redis-url "$REDIS_URL"` (its kill boards) and +`--projection-bus sqlite:…/bus.db` (celld's outbox Queue relays the Crawler +domain events to it; the board change notices go there too), then runs `distributed dev server/`: the application lifecycle generates the `$distributed` GraphQL client, starts `mobpit-service` on `MOBPIT_SERVICE_PORT` (default `GAME_DEMOS_PORT + 2`) @@ -290,10 +293,26 @@ and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: `src/lib/pit/MyStats.graphql` (my `player_stats` row; `$playerId` is bound by `MyStats.graphql.bindings.js` to `sessionClaim('user', 'id')`, so the value is the verified session's id for SSR and in the browser, never a URL - or form value) and `src/lib/pit/Leaderboard.graphql` (top 5 by XP). The - root layout's `createDistributedSvelteKitServer` runs them during SSR, the - page hydrates without refetching, and the `graphql-transport-ws` - subscription pushes every later settlement — to every signed-in player. + or form value) and `src/lib/pit/Leaderboard.graphql` (display names from + `player_stats`). The root layout's `createDistributedSvelteKitServer` runs + them during SSR, the page hydrates without refetching, and the + `graphql-transport-ws` subscription pushes every later settlement — to + every signed-in player. +- **Leaderboard (Redis, through GraphQL).** The ranking is the zone + server's Redis XP board, served by `mobpit-service` as GraphQL board roots + (`xpLeaderboard`, `xpLeaderboardAround`, `xpMyRank` and the `kills…` + twins; `leaderboard_surfaces()`), and generated into `$distributed` as + `XpLeaderboardLeaderboard` & co. (`leaderboards.ts`). The page `load` + (`routes/pit/+page.server.ts`) runs it once with + `loadDistributedLeaderboard` for SSR; `Leaderboard.svelte` follows it with + `useDistributedLeaderboard` (`@hops-ops/distributed/sveltekit`), which + subscribes to the `@live` document over the client's WebSocket. Pushes + cross processes on the bus: the zone server's `LeaderboardProjector` + publishes `distributed.leaderboard.changed` for each changed board + (`LeaderboardChangePublisher`), and `mobpit-service` subscribes with + `GraphqlEngine::leaderboard_change_handlers` (`spawn_board_notices`). + Both processes use `REDIS_URL` and the same key prefix + (`MOBPIT_REDIS_PREFIX`, default `mobpit`). The rows come from the cells (see *Mob Pit*, step 5): the zone's `profile`/`board` pushes and the `--dev` leaderboard route are not used by this app (they remain for the vanilla client pages). @@ -310,7 +329,8 @@ and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: `./run.sh smoke` runs `check:games`, then `distributed build server/` (the GraphQL client, `svelte-check`, `vite build` with the game codegen and WASM) and `tests/pit.test.mjs`: the production build (`node build`) against the -real zone server and `mobpit-service` in headless Chromium — sign-in, ticket +real zone server and `mobpit-service` (on the Redis in `REDIS_URL`, with a +per-run key prefix) in headless Chromium — sign-in, ticket rejection paths, an unauthenticated GraphQL request refused, join, keyboard movement, auto-hunt until a kill settles, the killer's stats and a second signed-in player's leaderboard updating live without a reload, the @@ -378,8 +398,10 @@ examples/game/ crawler/ crate `mobpit_crawler`: Crawler aggregate, RecordKill (XP rules), `CrawlerState` domain event (shared by the zone and the service) service/ crate `mobpit_service` (bin `mobpit-service`): player_stats read - model, modeled projector from the bus, GraphQL @live (Bearer only), - tests/projection.rs (cells → bus → table → live push, no mutations, 401) + model, modeled projector from the bus, the Redis boards as GraphQL + board roots (@live from bus notices), GraphQL @live (Bearer only), + tests/projection.rs (cells → bus → table → live push, board push + over bus notices, no mutations, 401) sim/ crate `game_sim`: the `#[predict]` steps (arena.rs movement + floor, mobpit.rs, ripple.rs inputs + steps, grid.rs world + walker step + `#[game_pure]` grid_cell / grid_sample); compiled natively for the diff --git a/examples/game/run.sh b/examples/game/run.sh index 71db0acb6..fb8e46991 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -28,7 +28,8 @@ # server NetSim; the HUD sliders change it live), MOBPIT_DATA_DIR (Mob Pit / # Leaderboard local files and their `celld dev` state, default # examples/game/.data/mobpit), REDIS_URL (Redis leaderboard instead of -# in-memory), SVELTEKIT_PORT (SvelteKit demo, default 5573; its game server +# in-memory; required by `sveltekit` and `smoke`, whose SvelteKit +# leaderboard is the Redis board served through GraphQL), SVELTEKIT_PORT (SvelteKit demo, default 5573; its game server # uses GAME_DEMOS_PORT, default 2967 there, and its read-model service # MOBPIT_SERVICE_PORT, default GAME_DEMOS_PORT + 2). # @@ -336,12 +337,25 @@ api_keys() { API_JWKS="$(sed -n 2p <<<"$keys")" } +# require_redis: the SvelteKit demo's leaderboard is the Redis board the +# zone server writes and mobpit-service serves (REDIS_URL). +require_redis() { + if [[ -z "${REDIS_URL:-}" ]]; then + echo "REDIS_URL is required: the SvelteKit leaderboard is the Redis board" >&2 + echo " (docker compose up -d redis at the repo root, then REDIS_URL=redis://127.0.0.1:6379)" >&2 + exit 1 + fi +} + # The SvelteKit demo: the Mob Pit zone server (publishing Crawler domain -# events to a SQLite bus), then `distributed dev server/`, which generates -# the `$distributed` GraphQL client, runs mobpit-service (the player_stats -# read model, GraphQL @live) and `vite dev` (the Vite plugin also generates -# src/lib/generated/mobpit and its WASM on start and on Rust changes). +# events and board change notices to a SQLite bus, kills into the Redis +# boards), then `distributed dev server/`, which generates the +# `$distributed` GraphQL client, runs mobpit-service (the player_stats read +# model and the Redis boards, GraphQL @live) and `vite dev` (the Vite plugin +# also generates src/lib/generated/mobpit and its WASM on start and on Rust +# changes). run_sveltekit() { + require_redis local port="${GAME_DEMOS_PORT:-2967}" local ui_port="${SVELTEKIT_PORT:-5573}" local service_port="${MOBPIT_SERVICE_PORT:-$((port + 2))}" @@ -356,7 +370,8 @@ run_sveltekit() { log_file="$(mktemp -t game-demos-sveltekit.XXXXXX)" secret="${DISTRIBUTED_GAME_SECRET:-$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')}" DISTRIBUTED_GAME_SECRET="$secret" "$BIN" mobpit --port "$port" \ - --data-dir "$data" --projection-bus "sqlite:$data/bus.db" >"$log_file" 2>&1 & + --data-dir "$data" --projection-bus "sqlite:$data/bus.db" \ + --redis-url "$REDIS_URL" >"$log_file" 2>&1 & local server_pid=$! trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; kill_tree "$CELLD_PID"; rm -f "$log_file"' EXIT INT TERM wait_listening "$log_file" "$server_pid" "game-demos mobpit" @@ -384,6 +399,8 @@ smoke() { # own `celld dev` (CELLD_BIN) on the cells worker. ensure_celld ensure_cells + # The SvelteKit test's leaderboard is the Redis board. + require_redis log "generated clients are current; predicted steps and pure functions compiled to WASM" gen --check log "server: fmt, clippy, tests (bots smoke)" diff --git a/examples/game/server/crawler/src/lib.rs b/examples/game/server/crawler/src/lib.rs index f014053ec..0f98f9dc9 100644 --- a/examples/game/server/crawler/src/lib.rs +++ b/examples/game/server/crawler/src/lib.rs @@ -36,6 +36,13 @@ pub const AGGREGATE_TYPE: &str = "Crawler"; pub const CRAWLER_KIND: &str = "crawler"; /// celld cell kind of per-player presence (the runtime lifecycle facts). pub const PRESENCE_KIND: &str = "presence"; +/// Redis kills board (`+1` per kill), written by the zone server's +/// leaderboard projector and served by `mobpit_service`'s GraphQL roots. +pub const KILLS_BOARD: &str = "mobpit.kills"; +/// Redis XP board (`+xp` per kill). +pub const XP_BOARD: &str = "mobpit.xp"; +/// Default Redis key prefix of the boards (zone server and service must agree). +pub const BOARD_REDIS_PREFIX: &str = "mobpit"; /// Command dispatched for every `MobKilled` fact. pub const RECORD_KILL: &str = RecordKill::COMMAND; /// Event recorded by [`RECORD_KILL`]. diff --git a/examples/game/server/service/Cargo.toml b/examples/game/server/service/Cargo.toml index 300ce37fa..ce1f902fb 100644 --- a/examples/game/server/service/Cargo.toml +++ b/examples/game/server/service/Cargo.toml @@ -20,7 +20,9 @@ path = "src/main.rs" [dependencies] mobpit_crawler = { path = "../crawler" } -distributed = { path = "../../../..", features = ["sqlite", "http", "graphql"] } +# `redis`: the Redis leaderboards the zone server writes, served as GraphQL +# board roots (`xpLeaderboard`, `killsMyRank`, … with `@live`). +distributed = { path = "../../../..", features = ["sqlite", "http", "graphql", "redis"] } axum = { version = "0.8", default-features = false, features = ["json", "tokio", "http1", "ws"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/examples/game/server/service/src/lib.rs b/examples/game/server/service/src/lib.rs index 0d8a84d9b..57c281d7b 100644 --- a/examples/game/server/service/src/lib.rs +++ b/examples/game/server/service/src/lib.rs @@ -30,12 +30,14 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; -use distributed::bus::SqliteBus; +use distributed::bus::{BusConsumer, RunOptions, SqliteBus}; +use distributed::graphql::surface::{LeaderboardGrant, LeaderboardSurface}; use distributed::graphql::{ build_surface, read, surface_for_application_contract, DistributedClientSurfaceExport, GraphqlEngine, IdentityConfig, ModelPermissions, OidcConfig, RoleGrant, Surface, SurfaceOptions, }; +use distributed::leaderboard::{Board, LeaderboardStore, RedisLeaderboard}; use distributed::microsvc::{ spawn_service_consumer_loop, CausalProjectorContext, HandlerError, ModeledProjection, Routes, Service, CONSUMER_IDLE_POLL, @@ -51,6 +53,27 @@ pub use mobpit_crawler::stats::{ }; /// The one engine role. Every signed-in player reads the whole board. pub const ROLE: &str = "user"; +/// Bus consumer group of this process's board change notices (every +/// GraphQL process subscribes under its own group). +pub const BOARD_NOTICE_GROUP: &str = "mobpit-service-boards"; + +/// The Redis boards the zone server projects kills into, as GraphQL roots: +/// `xpLeaderboard(first)`, `xpLeaderboardAround(radius)`, `xpMyRank` and +/// the `kills…` twins, all `@live`. `…MyRank` and `…Around` rank the viewer +/// (the session's user id); no root ranks an arbitrary member. +pub fn leaderboard_surfaces() -> Vec { + [ + ("xp", mobpit_crawler::XP_BOARD), + ("kills", mobpit_crawler::KILLS_BOARD), + ] + .into_iter() + .map(|(name, board)| { + LeaderboardSurface::single(name, &Board::high_first(board).expect("board name")) + .limits(5, 20, 5) + .grant(ROLE, LeaderboardGrant::read()) + }) + .collect() +} /// Read authorization: the leaderboard is public to signed-in players; /// there is nothing to write (no commands, no mutations). @@ -99,6 +122,8 @@ fn grants() -> BTreeMap> { fn full_surface() -> Surface { build_surface(&read_model_catalog().tables, &SurfaceOptions::sqlite()) .expect("mobpit Surface should build") + .with_leaderboards(leaderboard_surfaces()) + .expect("mobpit boards should attach") .with_projection_owners([projector().into()]) .expect("mobpit projector should bind") } @@ -147,13 +172,20 @@ impl BearerIdentity { } } -/// The GraphQL engine over `repo` with `@live` pushes from its change stream. +/// The GraphQL engine over `repo` (`player_stats`, `@live` from its change +/// stream) and `boards` (the Redis leaderboards, `@live` from the zone +/// server's change notices: [`spawn_board_notices`]). pub fn build_engine( repo: &SqliteRepository, + boards: Arc, identity: &BearerIdentity, protocol_key: [u8; 32], ) -> Result { - GraphqlEngine::builder(repo.pool().clone()) + let mut builder = GraphqlEngine::builder(repo.pool().clone()); + for surface in leaderboard_surfaces() { + builder = builder.leaderboard(surface, boards.clone()); + } + builder .service_id(APPLICATION) .protocol_token_key(protocol_key) .roles(&[ROLE]) @@ -182,6 +214,30 @@ pub async fn open_shared_sqlite(url: &str) -> Result, bus: sqlx::SqlitePool) { + tokio::spawn(async move { + loop { + let handlers = Arc::new(engine.leaderboard_change_handlers(BOARD_NOTICE_GROUP)); + let bus = SqliteBus::new(bus.clone()).with_idle_poll(CONSUMER_IDLE_POLL); + if let Err(error) = bus.subscribe(handlers, RunOptions::default()).await { + eprintln!("mobpit-service: board notices: {error}"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + }); +} + +/// The Redis boards at `url` under `prefix` (the zone server's). +pub fn open_boards(url: &str, prefix: &str) -> Result, String> { + Ok(Arc::new( + RedisLeaderboard::connect(url) + .map_err(|error| format!("REDIS_URL {url}: {error}"))? + .with_prefix(prefix), + )) +} + /// Runtime configuration. #[derive(Debug, Clone)] pub struct ServiceOptions { @@ -189,6 +245,10 @@ pub struct ServiceOptions { pub database_url: String, /// `sqlite:` URL of the bus the zone server publishes to. pub bus_url: String, + /// `redis://…` of the leaderboards the zone server writes. + pub redis_url: String, + /// Their key prefix (the zone server's `--redis-prefix`). + pub redis_prefix: String, /// Listen address. pub bind: String, /// Bearer verification. @@ -207,12 +267,15 @@ pub async fn run(options: ServiceOptions) -> Result<(), Box Result<(), Box> { database_url: env::var("DATABASE_URL") .unwrap_or_else(|_| "sqlite:./mobpit-service.db?mode=rwc".into()), bus_url: env::var("MOBPIT_BUS_URL").unwrap_or_else(|_| "sqlite:./mobpit-bus.db".into()), + redis_url: required("REDIS_URL")?, + redis_prefix: env::var("MOBPIT_REDIS_PREFIX") + .ok() + .filter(|prefix| !prefix.trim().is_empty()) + .unwrap_or_else(|| mobpit_crawler::BOARD_REDIS_PREFIX.into()), bind: env::var("BIND").unwrap_or_else(|_| "127.0.0.1:8791".into()), identity: BearerIdentity { issuer: required("MOBPIT_API_ISSUER")?, diff --git a/examples/game/server/service/tests/projection.rs b/examples/game/server/service/tests/projection.rs index 29659d4f7..3f56fbfc8 100644 --- a/examples/game/server/service/tests/projection.rs +++ b/examples/game/server/service/tests/projection.rs @@ -9,7 +9,10 @@ //! 2. a live leaderboard subscription pushes the next settled kill; //! 3. the rows equal the cells after a restart of the projector (cursor //! resumes, redelivery is deduplicated); -//! 4. the surface has no mutations and the HTTP endpoint refuses a request +//! 4. the leaderboard roots (`xpLeaderboard`, `@live`) read the zone +//! server's board, and its change notices travel over the bus to push +//! them (the engine shares no listener with the zone server's projector); +//! 5. the surface has no mutations and the HTTP endpoint refuses a request //! without a verified Bearer token. use std::path::PathBuf; @@ -25,7 +28,8 @@ use game_demos::devcelld::DevCelld; use game_demos::mobpit::crawler::RECORD_KILL; use game_demos::mobpit::{MobPitWorld, WorldOptions}; use mobpit_service::{ - build_engine, open_shared_sqlite, read_model_catalog, spawn_projector, BearerIdentity, + build_engine, open_shared_sqlite, read_model_catalog, spawn_board_notices, spawn_projector, + BearerIdentity, }; use serde_json::{json, Value}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -121,7 +125,13 @@ async fn settled_kills_project_into_live_player_stats() { .unwrap(); let bus = open_shared_sqlite(&bus_url).await.unwrap(); spawn_projector(repo.clone(), bus.clone()); - let engine = build_engine(&repo, &identity(), [7; 32]).unwrap(); + // The boards are the zone server's store (Redis in the demo; the same + // in-memory store here). Their `@live` pushes come only from the + // change notices on the bus. + let engine = std::sync::Arc::new( + build_engine(&repo, world.board().store().clone(), &identity(), [7; 32]).unwrap(), + ); + spawn_board_notices(engine.clone(), bus.clone()); sink.commit(&kill("k1", "dev-ana", 3, "ana")).await.unwrap(); sink.commit(&kill("k2", "dev-bo", 1, "bo")).await.unwrap(); @@ -182,6 +192,53 @@ async fn settled_kills_project_into_live_player_stats() { json!({ "player_stats": [{ "xp": 70 }] }) ); + // The XP board through the GraphQL roots, live over the bus notices. + let mut board = Box::pin(engine.execute_stream( + &user("dev-bo"), + Request::new("subscription { xpLeaderboard(first: 3) { member score } }"), + )); + eventually("the board to catch up", || async { + let now = engine + .execute( + &user("dev-bo"), + Request::new("{ xpLeaderboard(first: 3) { member score } }"), + ) + .await; + serde_json::to_value(&now.data).unwrap()["xpLeaderboard"] + == json!([{ "member": "dev-ana", "score": 85.0 }, { "member": "dev-bo", "score": 70.0 }]) + }) + .await; + let first = tokio::time::timeout(Duration::from_secs(5), board.next()) + .await + .unwrap() + .unwrap(); + assert!(first.errors.is_empty(), "{:?}", first.errors); + sink.commit(&kill("k5", "dev-bo", 3, "bo")).await.unwrap(); + let pushed = loop { + let next = tokio::time::timeout(Duration::from_secs(15), board.next()) + .await + .expect("board push after the settled kill (notice over the bus)") + .unwrap(); + let data = serde_json::to_value(&next.data).unwrap(); + if data["xpLeaderboard"][0]["member"] == "dev-bo" { + break data; + } + }; + assert_eq!( + pushed["xpLeaderboard"], + json!([{ "member": "dev-bo", "score": 130.0 }, { "member": "dev-ana", "score": 85.0 }]) + ); + let mine = engine + .execute( + &user("dev-bo"), + Request::new("{ xpMyRank { rank member score } }"), + ) + .await; + assert_eq!( + serde_json::to_value(&mine.data).unwrap()["xpMyRank"], + json!({ "rank": 0, "member": "dev-bo", "score": 130.0 }) + ); + // Read-only surface: there is no mutation to credit XP with. let forged = engine .execute( @@ -205,7 +262,7 @@ async fn settled_kills_project_into_live_player_stats() { } // HTTP: no Bearer → refused (no header-trust identity mode). - let app = mobpit_service::http::router(std::sync::Arc::new(engine), &["http://ui.test".into()]); + let app = mobpit_service::http::router(engine, &["http://ui.test".into()]); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); diff --git a/examples/game/server/src/common/relay.rs b/examples/game/server/src/common/relay.rs index 35dd6def2..a26ddac4d 100644 --- a/examples/game/server/src/common/relay.rs +++ b/examples/game/server/src/common/relay.rs @@ -18,8 +18,8 @@ use axum::http::{HeaderMap, StatusCode}; use axum::routing::post; use axum::{Json, Router}; use distributed::bus::{ - celld_queue_relay_handler, CelldQueueEnvelope, CelldQueueRelayHandler, SqliteBus, - CELLD_QUEUE_RELAY_PATH, + celld_queue_relay_handler, Bus, CelldQueueEnvelope, CelldQueueRelayHandler, Message, + MessagePublisher, SqliteBus, TransportError, CELLD_QUEUE_RELAY_PATH, }; use distributed::cell_host::{InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER}; use distributed::BusPublisher; @@ -98,6 +98,25 @@ impl ProjectionBus { } } +/// Publishes onto the projection bus once it is attached (the slot of +/// `MobPitWorld::projection`); before that every publish is a retryable +/// error, so a retrying publisher (`LeaderboardChangePublisher`) waits for +/// the bus instead of dropping the message. +#[derive(Clone)] +pub struct ProjectionPublisher(pub Arc>); + +impl MessagePublisher for ProjectionPublisher { + async fn publish(&self, message: Message) -> Result<(), TransportError> { + let bus = self + .0 + .get() + .ok_or_else(|| TransportError::retryable("projection bus not attached yet"))?; + SqliteBus::new(bus.pool().clone()) + .publish_message(message) + .await + } +} + #[derive(Clone)] struct RelayState { secret: InternalHttpSecret, diff --git a/examples/game/server/src/mobpit/board.rs b/examples/game/server/src/mobpit/board.rs index 9e976aa2d..3ae71fe0f 100644 --- a/examples/game/server/src/mobpit/board.rs +++ b/examples/game/server/src/mobpit/board.rs @@ -27,8 +27,9 @@ use std::time::{Duration, Instant, UNIX_EPOCH}; use distributed::leaderboard::{ BackgroundLeaderboard, BackgroundLeaderboardOptions, Board, InMemoryLeaderboard, - LeaderboardChange, LeaderboardEntry, LeaderboardError, LeaderboardProjection, - LeaderboardProjector, LeaderboardStore, RedisLeaderboard, ScorePolicy, ScoreSubmission, + LeaderboardChange, LeaderboardChangeListener, LeaderboardEntry, LeaderboardError, + LeaderboardProjection, LeaderboardProjector, LeaderboardStore, RedisLeaderboard, ScorePolicy, + ScoreSubmission, }; use distributed::{DomainEventOccurrence, ExternalEventSource}; use distributed_game::GameRecord; @@ -38,10 +39,7 @@ use serde_json::{json, Value}; use super::crawler::{KillReceipt, AGGREGATE_TYPE}; use super::store::CrawlerCells; -/// Kills board. -pub const KILLS_BOARD: &str = "mobpit.kills"; -/// XP board. -pub const XP_BOARD: &str = "mobpit.xp"; +pub use mobpit_crawler::{KILLS_BOARD, XP_BOARD}; /// Producer of kill occurrences. pub const PRODUCER: &str = "mobpit.crawler-cell"; /// Rows pushed to clients. @@ -238,7 +236,15 @@ pub struct Leaderboards { impl Leaderboards { /// Redis when `redis_url` is given, else in memory. Must run inside a /// Tokio runtime (spawns the background applier and the refresher). - pub fn open(redis_url: Option<&str>, redis_prefix: &str) -> Result, String> { + /// `on_change` is told every board whose scores changed (applied kills + /// and rebuild promotions): the zone server publishes it on the + /// projection bus as `distributed.leaderboard.changed`, which drives + /// `@live` boards in `mobpit-service`. + pub fn open( + redis_url: Option<&str>, + redis_prefix: &str, + on_change: Option, + ) -> Result, String> { let (store, backend): (Arc, _) = match redis_url { Some(url) => ( Arc::new( @@ -251,7 +257,10 @@ impl Leaderboards { None => (Arc::new(InMemoryLeaderboard::new()), "memory"), }; let projection = Arc::new(KillProjection::new()); - let projector: Projector = LeaderboardProjector::new(store.clone(), projection); + let mut projector: Projector = LeaderboardProjector::new(store.clone(), projection); + if let Some(listener) = on_change { + projector = projector.with_change_listener(listener); + } let background = BackgroundLeaderboard::spawn( projector.clone(), BackgroundLeaderboardOptions::default(), diff --git a/examples/game/server/src/mobpit/mod.rs b/examples/game/server/src/mobpit/mod.rs index 83c4d2c82..988b3d690 100644 --- a/examples/game/server/src/mobpit/mod.rs +++ b/examples/game/server/src/mobpit/mod.rs @@ -64,6 +64,9 @@ pub struct MobPitArgs { /// Redis for the leaderboard (`redis://127.0.0.1:6379`); default in memory. #[arg(long, env = "REDIS_URL")] pub redis_url: Option, + /// Redis key prefix of the boards (`mobpit-service` reads the same). + #[arg(long, env = "MOBPIT_REDIS_PREFIX", default_value = mobpit_crawler::BOARD_REDIS_PREFIX)] + pub redis_prefix: String, /// Start with this artificial storage latency (ms). #[arg(long, env = "MOBPIT_STORAGE_LATENCY_MS", default_value_t = 0)] pub storage_latency_ms: u64, @@ -147,6 +150,7 @@ pub async fn run(mut args: MobPitArgs) -> std::io::Result<()> { let celld = store::CelldOptions::new(&args.celld_url, secret).map_err(std::io::Error::other)?; let mut options = WorldOptions::new(&data_dir, celld); options.redis_url = args.redis_url.clone(); + options.redis_prefix = args.redis_prefix.clone(); options.projection_bus = args.projection_bus.clone(); let world = MobPitWorld::start(options) .await diff --git a/examples/game/server/src/mobpit/world.rs b/examples/game/server/src/mobpit/world.rs index 78f561a6a..c56f16785 100644 --- a/examples/game/server/src/mobpit/world.rs +++ b/examples/game/server/src/mobpit/world.rs @@ -18,8 +18,9 @@ use super::presence::Presence; use super::store::{CelldOptions, CrawlerCells, MobPitFactSink, SinkState}; use super::zone::{MobPit, MobPitConfig}; use super::{routes, MobPitDemo}; -use crate::common::relay::{relay_routes, ProjectionBus}; +use crate::common::relay::{relay_routes, ProjectionBus, ProjectionPublisher}; use crate::common::{random_hex, Demo, DemoRoutes, Guests}; +use distributed::leaderboard::LeaderboardChangePublisher; /// Unsettled facts one zone may hold before `emit_fact` reports backpressure. pub const FACT_CAPACITY: usize = 256; @@ -51,7 +52,7 @@ impl WorldOptions { data_dir: data_dir.into(), celld, redis_url: None, - redis_prefix: "mobpit".into(), + redis_prefix: mobpit_crawler::BOARD_REDIS_PREFIX.into(), projection_bus: None, read_models: None, } @@ -119,7 +120,15 @@ impl MobPitWorld { pub fn open(options: WorldOptions) -> Result, String> { std::fs::create_dir_all(&options.data_dir) .map_err(|e| format!("create {}: {e}", options.data_dir.display()))?; - let board = Leaderboards::open(options.redis_url.as_deref(), &options.redis_prefix)?; + // Board changes go out on the projection bus (once attached) so the + // GraphQL service's `@live` boards follow them. + let projection: Arc> = Arc::default(); + let notices = LeaderboardChangePublisher::spawn(ProjectionPublisher(projection.clone())); + let board = Leaderboards::open( + options.redis_url.as_deref(), + &options.redis_prefix, + Some(notices.listener()), + )?; let sink = Arc::new(SinkState::default()); let presence = Arc::new(Presence::default()); let fact_sink = MobPitFactSink::new( @@ -143,7 +152,7 @@ impl MobPitWorld { board, zone_pending: AtomicUsize::new(0), profiles: Arc::default(), - projection: Arc::default(), + projection, presence, })) } diff --git a/examples/game/sveltekit/src/lib/pit/Leaderboard.graphql b/examples/game/sveltekit/src/lib/pit/Leaderboard.graphql index 2bc0556a8..76f9b92ea 100644 --- a/examples/game/sveltekit/src/lib/pit/Leaderboard.graphql +++ b/examples/game/sveltekit/src/lib/pit/Leaderboard.graphql @@ -1,8 +1,10 @@ -# The leaderboard: a query over the player_stats read model (top 5 by XP). -query Leaderboard($limit: Int! = 5) @load @live { - player_stats(order_by: [{ xp: desc }, { player_id: asc }], limit: $limit) { +# The leaderboard component's read-model island: display names. The ranking +# itself is the Redis XP board through the GraphQL board roots +# (`xpLeaderboard`, see Leaderboard.svelte); names live in the player_stats +# read model, keyed by the same principal. +query Leaderboard($limit: Int! = 100) @load @live { + player_stats(order_by: [{ player_id: asc }], limit: $limit) { player_id name - xp } } diff --git a/examples/game/sveltekit/src/lib/pit/Leaderboard.svelte b/examples/game/sveltekit/src/lib/pit/Leaderboard.svelte index 34ca8d2e8..4959485b8 100644 --- a/examples/game/sveltekit/src/lib/pit/Leaderboard.svelte +++ b/examples/game/sveltekit/src/lib/pit/Leaderboard.svelte @@ -1,20 +1,36 @@ -

      - {#each rows as row (row.player_id)} -
    1. {row.name} — {row.xp} XP
      2. +
        + {#each rows as row (row.member)} +
      1. {nameOf.get(row.member) ?? row.member} — {row.score} XP
        2. {/each}
      +{#if $board.error}

      {$board.error}

      {/if} diff --git a/examples/game/sveltekit/src/routes/pit/+page.server.ts b/examples/game/sveltekit/src/routes/pit/+page.server.ts index 5d7188d63..e987cd5fb 100644 --- a/examples/game/sveltekit/src/routes/pit/+page.server.ts +++ b/examples/game/sveltekit/src/routes/pit/+page.server.ts @@ -1,13 +1,30 @@ import { redirect } from '@sveltejs/kit'; import { env } from '$env/dynamic/private'; +import { loadDistributedLeaderboard } from '@hops-ops/distributed/sveltekit'; +import { XpLeaderboardLeaderboard, type XpLeaderboardData } from '$distributed'; import { tickets } from '$lib/server/game'; +import { graphqlUrl } from '$lib/server/distributed'; export const load = async (event) => { const joined = await tickets.resolve(event); if (joined === undefined) redirect(303, '/'); + // The XP board is server-ranked (Redis), not a replica query: load it + // here so the first HTML has it; the component then follows `@live`. + const session = await event.locals.auth(); + let board: XpLeaderboardData | undefined; + try { + board = await loadDistributedLeaderboard(XpLeaderboardLeaderboard, { first: 5 }, { + url: graphqlUrl(), + auth: { accessToken: session?.accessToken ?? null }, + fetch: event.fetch + }); + } catch (error) { + console.error(`leaderboard SSR: ${error}`); + } return { zone: joined.zone, zonesUrl: env.GAME_ZONES_URL ?? 'ws://127.0.0.1:2967', - principal: joined.principal + principal: joined.principal, + board }; }; diff --git a/examples/game/sveltekit/src/routes/pit/+page.svelte b/examples/game/sveltekit/src/routes/pit/+page.svelte index 486748e57..f3d8fde83 100644 --- a/examples/game/sveltekit/src/routes/pit/+page.svelte +++ b/examples/game/sveltekit/src/routes/pit/+page.svelte @@ -56,7 +56,7 @@ (projected from the Crawler cells' domain events): SSR, then @live. -->
      saving {unsaved}
      - + {#each saved.items.slice(0, 3) as s (s.key)}
      +{s.xp} XP saved ✓ {s.waitedMs} ms
      {/each} diff --git a/examples/game/sveltekit/tests/pit.test.mjs b/examples/game/sveltekit/tests/pit.test.mjs index 8bda959fa..14c0db5ce 100644 --- a/examples/game/sveltekit/tests/pit.test.mjs +++ b/examples/game/sveltekit/tests/pit.test.mjs @@ -5,9 +5,12 @@ * auto-hunt kills a mob → the celld Crawler cell commits it, its domain event * goes through the cell outbox and celld's Queue to the zone server's relay, * onto the bus, and is projected into `player_stats` → the `@live` GraphQL queries - * (MyStats, bound to the session; Leaderboard) update without a reload, for - * the killer and for a second signed-in player → the SSR HTML after a reload - * has the settled numbers → no page errors. + * (MyStats, bound to the session; the leaderboard's names) update without a + * reload; the zone server's Redis XP board reaches the page through + * mobpit-service's GraphQL board roots (`xpLeaderboard`, SSR-seeded, `@live` + * from the board change notices on the bus) for the killer and for a second + * signed-in player → the SSR HTML after a reload has the settled numbers → + * no page errors. Needs REDIS_URL. * * Ports: GAME_DEMOS_PORT (default 2967), SVELTEKIT_PORT (default 5573), * MOBPIT_SERVICE_PORT (default GAME_DEMOS_PORT + 2). Needs the lifecycle @@ -74,6 +77,11 @@ function start(command, args, options, ready) { } test('sign in, join, move, kill, live read model updates for two players', { timeout: 600_000 }, async () => { + // The leaderboard is the Redis board the zone server writes and + // mobpit-service serves through its GraphQL board roots. + const redisUrl = process.env.REDIS_URL; + assert.ok(redisUrl, 'REDIS_URL is required: the SvelteKit leaderboard is the Redis board (docker compose up -d redis)'); + const redisPrefix = `mobpit-sveltekit-${randomBytes(6).toString('hex')}`; const secret = randomBytes(32).toString('hex'); const dataDir = mkdtempSync(join(tmpdir(), 'mobpit-sveltekit-')); const busUrl = `sqlite:${join(dataDir, 'bus.db')}`; @@ -82,7 +90,7 @@ test('sign in, join, move, kill, live read model updates for two players', { tim // The Crawler cells run on celld; its outbox Queue relays the domain // events to the game server, which publishes them onto the bus. const celld = await startCelld({ relayPort: GAME_PORT }); - const gameEnv = { ...process.env, ...celld.env, DISTRIBUTED_GAME_SECRET: secret }; + const gameEnv = { ...process.env, ...celld.env, DISTRIBUTED_GAME_SECRET: secret, REDIS_URL: redisUrl, MOBPIT_REDIS_PREFIX: redisPrefix }; const gameArgs = ['mobpit', '--port', String(GAME_PORT), '--data-dir', join(dataDir, 'local'), '--projection-bus', busUrl]; const game = bin ? start(bin, gameArgs, { env: gameEnv }, /LISTENING/) @@ -92,6 +100,8 @@ test('sign in, join, move, kill, live read model updates for two players', { tim BIND: `127.0.0.1:${SERVICE_PORT}`, DATABASE_URL: `sqlite:${join(dataDir, 'service.db')}?mode=rwc`, MOBPIT_BUS_URL: busUrl, + REDIS_URL: redisUrl, + MOBPIT_REDIS_PREFIX: redisPrefix, MOBPIT_API_ISSUER: 'mobpit-sveltekit', MOBPIT_API_AUDIENCE: 'mobpit-service', MOBPIT_API_JWKS: keys.jwks, @@ -229,7 +239,10 @@ test('sign in, join, move, kill, live read model updates for two players', { tim assert.ok([10, 25, 60].some((tier) => xp >= tier) && kills >= 1, `xp ${xp} kills ${kills}`); assert.deepEqual(navigations, [], 'the read model updated without a reload'); - // The other player's leaderboard updated live too. + // The other player's leaderboard (the Redis board via the GraphQL + // board roots) updated live too: the zone server's board change + // notice crossed the bus to mobpit-service, which pushed it. + assert.equal(await bob.getAttribute('[data-testid=leaderboard]', 'data-live'), 'active'); await bob.waitForFunction( (expected) => [...document.querySelectorAll('[data-testid=leaderboard] li')].some((li) => li.textContent === expected), `ana — ${xp} XP`, From 68359a2d6e62dcd9af70e2ef7bae5e1cfaf682bb Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 19:29:34 -0500 Subject: [PATCH 34/40] test: expect migration 13 in the migration version assertions The schedule claims migration (0013) made three version assertions stale: the SQLite and Postgres upgrade tests and the CLI's migration inventory order test. [[tasks/gr-p39-postgres-game-adapters]] --- distributed_cli/src/contracts/tests.rs | 2 +- tests/postgres_repository/main.rs | 2 +- tests/sqlite_repository/main.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distributed_cli/src/contracts/tests.rs b/distributed_cli/src/contracts/tests.rs index 65105aa22..8b56b33d5 100644 --- a/distributed_cli/src/contracts/tests.rs +++ b/distributed_cli/src/contracts/tests.rs @@ -1259,7 +1259,7 @@ fn migration_inventory_is_deterministic_and_preserves_runtime_order() { .iter() .map(|migration| migration.version) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]); assert_eq!( inventory.canonical_bytes().expect("canonical inventory"), inventory diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index e77d6efed..763fa7e50 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -234,7 +234,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_and_preserve_checks() { .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 12); + assert_eq!(latest_version, 13); let invalid_service = sqlx::query( r#" diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index e44a89fc4..26035bf04 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -202,7 +202,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_without_schema_drift() .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 12); + assert_eq!(latest_version, 13); let created_at_type: String = sqlx::query_scalar( "SELECT typeof(created_at) FROM command_ledger WHERE service_id = 'service'", From 8138354e1b43c0f2b98d5808e1f0081a60a005fa Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 19:37:59 -0500 Subject: [PATCH 35/40] test(schedule): make the worker crash test robust on Postgres The crash test assumed the restarted worker's first pass ran within the crashed worker's 400 ms claim; opening and migrating a Postgres pool sometimes took longer. The claim is now 3 s and the test waits from the observed commit. The flaky-command counter was a process-wide static shared by the SQLite and Postgres variants running in parallel; each test now marks its own ticker id. [[tasks/gr-p39-postgres-game-adapters]] --- tests/schedule_worker/main.rs | 58 +++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/tests/schedule_worker/main.rs b/tests/schedule_worker/main.rs index c9feb9193..7d89c2d1b 100644 --- a/tests/schedule_worker/main.rs +++ b/tests/schedule_worker/main.rs @@ -162,9 +162,12 @@ async fn handle_tick( }) } -static FLAKY_FAILURES: AtomicUsize = AtomicUsize::new(0); +/// Ticker ids whose next `ticker.flaky` fails once (tests on different +/// backends run in parallel, so each uses its own id). +static FLAKY_ONCE: std::sync::Mutex> = + std::sync::Mutex::new(std::collections::BTreeSet::new()); -/// Fails with a retryable (500) error while `FLAKY_FAILURES` > 0. +/// Fails with a retryable (500) error once per id in `FLAKY_ONCE`. struct Flaky; impl PortableCommand for Flaky where @@ -182,23 +185,7 @@ async fn handle_flaky( ctx: &CausalCommandContext<'_, Ticker>, input: TickInput, ) -> Result>, HandlerError> { - let mut current = FLAKY_FAILURES.load(Ordering::SeqCst); - let mut fail = false; - while current > 0 { - match FLAKY_FAILURES.compare_exchange( - current, - current - 1, - Ordering::SeqCst, - Ordering::SeqCst, - ) { - Ok(_) => { - fail = true; - break; - } - Err(actual) => current = actual, - } - } - if fail { + if FLAKY_ONCE.lock().unwrap().remove(&input.id) { return Err(HandlerError::Other("transient dependency outage".into())); } handle_tick(ctx, input).await @@ -389,7 +376,13 @@ macro_rules! scenarios { #[tokio::test] async fn crash_between_commit_and_settle_does_not_fire_twice() { let Some(ctx) = $fresh.await else { return }; - let (repo, crashing_worker) = ctx.repo().await.with_schedule_worker(fast()); + // A claim long enough to still hold after the restart below + // (opening and migrating a Postgres pool takes a while). + let claim_ttl = Duration::from_secs(3); + let (repo, crashing_worker) = ctx + .repo() + .await + .with_schedule_worker(fast().with_claim_ttl(claim_ttl)); let committed = Arc::new(AtomicUsize::new(0)); let crashing: Arc = Arc::new(CrashAfterCommit { inner: LocalCommandHost::new(service(repo.clone())), @@ -408,6 +401,9 @@ macro_rules! scenarios { committed.load(Ordering::SeqCst) == 1 }) .await; + // The claim was taken before the commit, so it lapses before + // `committed_at + claim_ttl`. + let committed_at = Instant::now(); running.abort(); assert_eq!(ticks(&repo, "crash").await, 1); assert_eq!( @@ -420,8 +416,15 @@ macro_rules! scenarios { let (repo, worker) = $reopen(&ctx).await.with_schedule_worker(fast()); let host = LocalCommandHost::new(service(repo.clone())); let first = worker.run_once(&host).await.unwrap(); - assert!(first.is_empty(), "the crashed worker's claim still holds"); - tokio::time::sleep(Duration::from_millis(450)).await; + assert!( + committed_at.elapsed() >= claim_ttl - Duration::from_millis(500) + || first.is_empty(), + "the crashed worker's claim still holds: {first:?}" + ); + tokio::time::sleep( + (claim_ttl + Duration::from_millis(100)).saturating_sub(committed_at.elapsed()), + ) + .await; let replay = worker.run_once(&host).await.unwrap(); assert_eq!(replay.fired.len(), 1, "{replay:?}"); assert!( @@ -472,11 +475,12 @@ macro_rules! scenarios { let Some(ctx) = $fresh.await else { return }; let (repo, worker) = ctx.repo().await.with_schedule_worker(fast()); let host = LocalCommandHost::new(service(repo.clone())); - FLAKY_FAILURES.store(1, Ordering::SeqCst); - start(&host, "flaky", 1, 0, FLAKY).await; + let flaky = format!("flaky-{}", command_id()); + FLAKY_ONCE.lock().unwrap().insert(flaky.clone()); + start(&host, &flaky, 1, 0, FLAKY).await; let report = worker.run_once(&host).await.unwrap(); assert_eq!(report.retried.len(), 1, "{report:?}"); - let retry = repo.armed_schedules(&owner("flaky")).await.unwrap(); + let retry = repo.armed_schedules(&owner(&flaky)).await.unwrap(); assert_eq!(retry[0].attempts, 1); assert!( worker.run_once(&host).await.unwrap().is_empty(), @@ -485,8 +489,8 @@ macro_rules! scenarios { tokio::time::sleep(Duration::from_millis(1_050)).await; let report = worker.run_once(&host).await.unwrap(); assert_eq!(report.fired.len(), 1, "{report:?}"); - assert_eq!(ticks(&repo, "flaky").await, 1); - assert_eq!(armed(&repo, "flaky").await, 0); + assert_eq!(ticks(&repo, &flaky).await, 1); + assert_eq!(armed(&repo, &flaky).await, 0); } }; } From 0c4a940ec264fe8a7cf438ce59031fa0e0114d6e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Tue, 20 Oct 2026 19:50:34 -0500 Subject: [PATCH 36/40] fix(game): give REDIS_URL only to the SvelteKit test in run.sh smoke With REDIS_URL exported for the whole smoke run, the other demos' tests (which assert their in-memory boards) got Redis too. smoke now requires REDIS_URL, unsets it, and passes it to the SvelteKit test only. [[tasks/gr-p40-examples-e2e-coverage]] --- examples/game/run.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/game/run.sh b/examples/game/run.sh index fb8e46991..51cd396b2 100755 --- a/examples/game/run.sh +++ b/examples/game/run.sh @@ -399,8 +399,11 @@ smoke() { # own `celld dev` (CELLD_BIN) on the cells worker. ensure_celld ensure_cells - # The SvelteKit test's leaderboard is the Redis board. + # The SvelteKit test's leaderboard is the Redis board. Only that test gets + # REDIS_URL: the other demos' tests expect their in-memory boards. require_redis + local sveltekit_redis="$REDIS_URL" + unset REDIS_URL log "generated clients are current; predicted steps and pure functions compiled to WASM" gen --check log "server: fmt, clippy, tests (bots smoke)" @@ -421,7 +424,7 @@ smoke() { "$CLI" build "$SERVER" >"$build_log" 2>&1 || { cat "$build_log"; rm -f "$build_log"; exit 1; } rm -f "$build_log" (cd "$SVELTEKIT" && npx playwright install chromium >/dev/null && - GAME_DEMOS_BIN="$BIN" MOBPIT_SERVICE_BIN="$SERVICE_BIN" npm test --silent) + REDIS_URL="$sveltekit_redis" GAME_DEMOS_BIN="$BIN" MOBPIT_SERVICE_BIN="$SERVICE_BIN" npm test --silent) log "smoke OK" } From ab368c57b202c70c53f6aed2084a78e6c701c4c0 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 21 Oct 2026 05:55:15 -0500 Subject: [PATCH 37/40] test(js): expect leaderboard and ticket exports in pack smoke The sveltekit surface gained the leaderboard helpers (b847e717) and the game server surface the ticket extension checks (7a32cc87); the pack smoke's exact export lists still had the old surfaces. --- js/scripts/pack-smoke.mjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/js/scripts/pack-smoke.mjs b/js/scripts/pack-smoke.mjs index 553ec4ba6..403b0f5a0 100644 --- a/js/scripts/pack-smoke.mjs +++ b/js/scripts/pack-smoke.mjs @@ -584,6 +584,7 @@ assert.deepEqual(Object.keys(sveltekitSurface).sort(), [ 'authFromPageData', 'bindSveltekitOperation', 'constant', + 'createDistributedLeaderboardStore', 'createDistributedSvelteKit', 'createDistributedSvelteKitServer', 'createPageDataSessionSource', @@ -593,6 +594,7 @@ assert.deepEqual(Object.keys(sveltekitSurface).sort(), [ 'defineGraphqlIslandBindings', 'distributedReloadLifecycle', 'forwardedProp', + 'loadDistributedLeaderboard', 'omitVariable', 'parseDistributedGenerationEnvelope', 'provideDistributedSvelteKitClient', @@ -603,6 +605,7 @@ assert.deepEqual(Object.keys(sveltekitSurface).sort(), [ 'searchParam', 'sessionClaim', 'sessionSourceFromPageData', + 'useDistributedLeaderboard', 'useDistributedSvelteKitClient', 'useDistributedSvelteKitCommands', 'validateDistributedReloadLocation', @@ -629,8 +632,11 @@ const sveltekitGameServerSurface = await import('@hops-ops/distributed/sveltekit assert.deepEqual(Object.keys(sveltekitGameServerSurface).sort(), [ 'GAME_TICKET_RATE_LIMIT_SCRIPT', 'GameTicketRateLimiterUnavailable', + 'MAX_GAME_TICKET_EXTENSION_BYTES', 'MAX_GAME_TICKET_TTL_SECONDS', 'MIN_GAME_TICKET_SECRET_BYTES', + 'TRUSTED_FIELD_DENYLIST', + 'checkGameTicketExtensions', 'createGameTicketHandler', 'createGameTicketRateLimiter', 'createRedisGameTicketRateLimiter', From b397cffd27cf04d7f3232e157f151c04807ad68a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 21 Oct 2026 05:55:15 -0500 Subject: [PATCH 38/40] fix(game): release a refused transfer before rejecting the client The target released its reservation in a spawned task after replying, so a client that immediately returned to its source could still find the transfer reserved ("transfer in progress"). Await the release first. --- distributed_game/src/net/mod.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index 637b5e6fb..dbc7e22fb 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -1857,15 +1857,17 @@ where } let result = reply_rx.await; if let Some((issuer, nonce)) = reserved.take() { - let op = if matches!(result, Ok(Ok(_))) { - TransferOp::Confirm + if matches!(result, Ok(Ok(_))) { + let inner = inner.clone(); + tokio::spawn(async move { + let _ = transfer_op(&inner, &issuer, TransferOp::Confirm, &nonce).await; + }); } else { - TransferOp::Release - }; - let inner = inner.clone(); - tokio::spawn(async move { - let _ = transfer_op(&inner, &issuer, op, &nonce).await; - }); + // Release before the client hears the refusal: it may present + // the same ticket to its source right away, and the issuer only + // accepts that return once the reservation is released. + let _ = transfer_op(&inner, &issuer, TransferOp::Release, &nonce).await; + } } let client = match result { Ok(Ok(joined)) => joined.client, From c274d0751183597a6ef17bd112716be5334f796f Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 21 Oct 2026 05:55:16 -0500 Subject: [PATCH 39/40] test(game): wait for every kill to settle before board checks One swing can kill several mobs, so the hunt can stop at two settled kills while a third is still in flight; the board then jumps past the captured profile and never matches. Wait for acked inputs and settled kills first. --- .../game/client/tests/leaderboard-headless.test.mjs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/game/client/tests/leaderboard-headless.test.mjs b/examples/game/client/tests/leaderboard-headless.test.mjs index 4a72e22e8..6d07a8eef 100644 --- a/examples/game/client/tests/leaderboard-headless.test.mjs +++ b/examples/game/client/tests/leaderboard-headless.test.mjs @@ -33,7 +33,18 @@ test('board pushes match the cells and rebuild is identical', { timeout: 600_000 await pit.client.connect('pit-1'); await pit.ready; const me = api.last.principal; + const kills = []; + pit.on('kill', (k) => k.killer === pit.hello().entity && kills.push(k)); await hunt(pit, () => (pit.profile()?.kills ?? 0) >= 2); + // One swing can kill several mobs, and kills keep landing from inputs + // sent before the hunt saw the second settlement: wait until every + // input is acknowledged and every own kill settled before comparing. + await until(() => pit.client.predictor.pending.length === 0, 5_000, 'inputs acknowledged'); + await until( + () => pit.unsaved.size === 0 && pit.profile()?.kills === kills.length, + 10_000, + `settled profile ${JSON.stringify(pit.profile())} for ${kills.length} kills` + ); const profile = pit.profile(); // Pushed, not polled: the observer sees my kills without asking. From 13973be70f0c52d3e0705f9c6d5e8ab7b35f7298 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 21 Oct 2026 05:56:51 -0500 Subject: [PATCH 40/40] test(cli): run the lifecycle executor fixture through sh Exec-ing a just-written script can fail with ETXTBSY when another test thread forks while the script's write fd is open. Pass the script to /bin/sh so it is only read, never exec'd. --- distributed_cli/src/lifecycle/build.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/distributed_cli/src/lifecycle/build.rs b/distributed_cli/src/lifecycle/build.rs index 2f3f65a61..032c0dcda 100644 --- a/distributed_cli/src/lifecycle/build.rs +++ b/distributed_cli/src/lifecycle/build.rs @@ -1384,8 +1384,6 @@ mod tests { #[cfg(unix)] #[test] fn lifecycle_executor_receives_exact_cli_path_including_spaces() { - use std::os::unix::fs::PermissionsExt; - let fixture = tempfile::tempdir().expect("create executor fixture"); let stage = fixture.path().join("stage"); fs::create_dir_all(&stage).expect("create executor stage"); @@ -1395,9 +1393,6 @@ mod tests { "#!/bin/sh\nprintf '%s' \"$DISTRIBUTED_LIFECYCLE_CLI_EXECUTABLE\"\n", ) .expect("write executor script"); - let mut permissions = fs::metadata(&script).unwrap().permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&script, permissions).expect("make executor executable"); let cli = fixture.path().join("cli with spaces").join("distributed"); let mut outputs = BTreeSet::new(); outputs.insert("cli-path.txt".to_string()); @@ -1409,8 +1404,11 @@ mod tests { &outputs, &LifecycleExecutor { identity: "sha256:executor".to_string(), - program: script.to_string_lossy().into_owned(), - args: Vec::new(), + // Run the script through `sh` instead of exec-ing it: a + // freshly written executable can fail with ETXTBSY while a + // concurrently forked test child still holds its write fd. + program: "/bin/sh".to_string(), + args: vec![script.to_string_lossy().into_owned()], stdout: Some("cli-path.txt".to_string()), }, false,