diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index 5f196a208..2e9b3da62 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -42,6 +42,7 @@ jobs: tests/celld/worker -> tests/celld/worker/target tests/celld/alarm-worker -> tests/celld/alarm-worker/target tests/celld/fact-worker -> tests/celld/fact-worker/target + tests/celld/zone-worker -> tests/celld/zone-worker/target shared-key: celld-storage - uses: actions/setup-node@v4 with: @@ -82,6 +83,13 @@ jobs: run: | cargo test -p distributed_game --features celld,runtime --test celld_zone_state cargo test -p distributed_game --features celld,net,lease --test world + - name: Build the celld zone host test Worker + working-directory: tests/celld/zone-worker + run: worker-build --release + - name: Zones as celld objects (celld dev; 3-node fleet on RustFS, crash, facts exactly once) + env: + CELLD_ZONE_REQUIRED: "1" + run: cargo test -p distributed_game --features celld,net --test celld_zone - name: Retain storage proof results and logs if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/integration-game-demos.yaml b/.github/workflows/integration-game-demos.yaml index 2c1304964..eb3055ecc 100644 --- a/.github/workflows/integration-game-demos.yaml +++ b/.github/workflows/integration-game-demos.yaml @@ -7,9 +7,11 @@ name: Game demos smoke # headless tests, and the SvelteKit demo end to end: `distributed build # examples/game/server` (GraphQL client, svelte-check, vite build), then # Playwright against the production build, a real Mob Pit server and the real -# mobpit-service read model. Mob Pit's durable cells run on celld: the tests -# start their own `celld dev` (CELLD_BIN, pinned in /.celld-version; no Docker) serving -# examples/game/cells, built with worker-build. +# mobpit-service read model. Mob Pit's rooms and durable cells run on celld: +# the tests start their own `celld dev` (CELLD_BIN, pinned in +# /.celld-version) serving examples/game/cells, built with worker-build; the +# SvelteKit Playwright test runs the rooms on a 3-node fleet (RustFS via +# docker compose and the aws CLI, both on the runner; required here). on: workflow_call: @@ -64,6 +66,7 @@ jobs: env: # The SvelteKit leaderboard is the Redis board (GraphQL board roots). REDIS_URL: redis://127.0.0.1:6379 + MOBPIT_FLEET_REQUIRED: "1" run: examples/game/run.sh smoke - name: Leaderboard conformance and GraphQL boards against real Redis env: diff --git a/Cargo.toml b/Cargo.toml index 9b341917f..6fdd03778 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = ["distributed_macros", "distributed_cli", "distributed_game", "distributed_game_macros"] -exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/celld/fact-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"] +exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/celld/fact-worker", "tests/celld/zone-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"] resolver = "2" [workspace.package] diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index 9116199d7..198ed9eab 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -16,9 +16,9 @@ default = [] runtime = ["dep:tokio"] # Adapter that commits facts into a `distributed` AggregateCell. cell = ["dep:distributed", "dep:sha2", "dep:base64"] -# `command_sink::CommandFactSink`: facts dispatched as -# commands through any command host (a local service over any repository, -# or celld cells). +# `command_sink::CommandFactSink` over `CelldCommandHost` / +# `LocalCommandHost` (a local service over any repository, or celld cells +# over HTTP). The sink itself is portable (`cell`). command-host = ["cell", "distributed/graphql"] # celld clients: `CelldFactSink` (the command sink over a remote-only # `CelldCommandHost`), `CellZoneOwnership`, `CellZoneStateStore`. @@ -26,12 +26,18 @@ celld = ["cell", "command-host", "dep:reqwest", "dep:uuid"] # celld worker glue (wasm32 Durable Objects): `world::zone_state::celld_worker`, # the zone-state HTTP surface over a cell and an R2 bucket binding. workers-rs = ["cell", "dep:worker", "distributed/workers-rs"] +# Zones as celld Durable Objects (wasm32): `celld_zone::ZoneObject`, +# `CelldObjectHost` (facts object to object), `celld_zone_object!`. +celld-zone = ["cell", "ticket", "workers-rs", "dep:js-sys", "dep:wasm-bindgen", "dep:wasm-bindgen-futures"] # Per-zone Prometheus metrics (dependency-free text exposition). metrics = [] # Load-test bots (library + `game-bots` binary). bots = ["net", "dep:tokio-tungstenite", "dep:futures-util", "tokio/rt-multi-thread"] # axum WebSocket zone server with signed-ticket admission (+ /metrics). -net = ["runtime", "metrics", "dep:axum", "dep:hmac", "dep:sha2", "dep:base64", "tokio/net", "tokio/macros", "tokio/io-util"] +# HMAC-signed admission tickets (`ticket`) without the axum server (the +# celld zone host admits players with it). +ticket = ["dep:hmac", "dep:sha2", "dep:base64"] +net = ["runtime", "metrics", "ticket", "dep:axum", "dep:hmac", "dep:sha2", "dep:base64", "tokio/net", "tokio/macros", "tokio/io-util"] # `world::FencedLeaseOwnership`: zone ownership over `distributed::lease`. lease = ["dep:distributed"] @@ -50,6 +56,9 @@ futures-util = { version = "0.3", default-features = false, features = ["sink", reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true } uuid = { version = "1", features = ["v7"], optional = true } worker = { version = "0.8", optional = true } +js-sys = { version = "0.3", optional = true } +wasm-bindgen = { version = "0.2", optional = true } +wasm-bindgen-futures = { version = "0.4", optional = true } [dev-dependencies] base64 = "0.23" @@ -118,6 +127,11 @@ name = "celld_facts" path = "tests/celld_facts/main.rs" required-features = ["celld", "runtime"] +[[test]] +name = "celld_zone" +path = "tests/celld_zone/main.rs" +required-features = ["celld", "net"] + [[test]] name = "inbox" path = "tests/inbox/main.rs" diff --git a/distributed_game/README.md b/distributed_game/README.md index 616f1da4b..b76fba409 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -26,7 +26,8 @@ the browser, but the client SDK has no rendering dependency. - [Build a zone](#build-a-zone) - [Guides](#guides): game loop · input · what clients see · the client SDK · generated clients · SvelteKit apps · client prediction (Rust → WebAssembly) · shared - logic and data (no TypeScript twins) · saving progress · scaling out · + logic and data (no TypeScript twins) · saving progress · scaling out + (zones as celld objects, the native multi-host server) · live updates · lag compensation · testing and operations · security - [Design decisions](#design-decisions) - [Status and limits](#status-and-limits) @@ -100,7 +101,7 @@ each built to show one part of the engine: | Demo | Run | Shows | |---|---|---| | Arena | `examples/game/run.sh arena` | ticks, deltas, prediction on/off, interest radius vs. global entities, simulated latency | -| Mob Pit | `run.sh mobpit` | combat whose rewards are saved without stalling; "slow storage"; `kill -9` crash recovery | +| Mob Pit | `run.sh mobpit` (`--fleet` for several celld nodes) | rooms as celld objects joined by doorways, sleeping when empty and restored on return; combat whose rewards are saved exactly once without stalling | | Leaderboard | `run.sh leaderboard` | a live ranking fed from saved kills, rebuilt from history | | Shooting Gallery | `run.sh gallery` | lag compensation on/off under latency | | The Freeze | `run.sh freeze` | pausing every zone on two servers, updating content, resuming; safe-room reloads | @@ -291,7 +292,7 @@ A zone is in one of three **tiers**: Idle forever (a city clock, say) or from `has_pending_work` to stay Hot. - **Dormant**: idle for `dormant_after` (30 s); no ticks. With hibernation on, a Dormant zone is saved and unloaded (see - [Scaling out](#scaling-out-many-zones-many-servers)). + [Scaling out](#scaling-out-zones-as-celld-objects)). `on_tier_changed` tells you about transitions. Timers (`ctx.set_timeout(after, value)`, `ctx.set_interval(every, value)`, @@ -758,7 +759,9 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), - Keys must be unique across restarts; `ctx.fact_key` includes the activation's boot epoch for you (see [Durable facts](#durable-facts)). - If the process crashes, each in-flight fact is either fully saved or - not saved at all. Mob Pit shows this with `kill -9`. + not saved at all. A zone hosted on celld (`celld_zone`) also keeps its + emitted facts in an outbox in object storage, so they are resent after + a crash: exactly once (Mob Pit's rooms). - **Player lifecycle facts.** With `HostConfig::lifecycle_facts = true` the host itself emits `lifecycle::PlayerJoined` (ticket, rejoin or handoff), `ConnectionLost` (grace started; dropped or told to reload), @@ -786,7 +789,66 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), B" sorts after "left A" even when it commits first or the two hosts' clocks disagree. -### Scaling out: many zones, many servers +### Scaling out: zones as celld objects + +On celld every zone is a Durable Object (feature `celld-zone`, +[`celld_zone`](src/celld_zone/mod.rs)): one object per zone id, so a room +*is* a cell. Game code implements `CelldZone` for its `Zone` and declares +the class; nothing in the game or its API names a server. + +```rust,ignore +impl CelldZone for MobPit { + fn object_config(zone: &str, env: &ObjectEnv) -> Result { /* … */ } + fn fact_routes(sink: ObjectFactSink) -> ObjectFactSink { + sink.route(FactRoute::by_principal::("crawler")) + } +} +distributed_game::celld_zone_object!(PitRoom, MobPit); + +// worker entry: /rooms/{zone} → the zone's object (WebSocket or …/info) +if let Some(response) = forward_to_zone(req, &env, "PIT_ROOM", "/rooms/").await? { … } +``` + +**What celld handles:** where a zone runs (first touch; overflow to the +least-loaded node with `CELLD_MAX_RESIDENT_CELLS`), forwarding a WebSocket +from any node to the owner, one live instance per zone (fenced by its own +epochs), evicting idle objects, moving idle ones to balance load, draining +a stopping node and recovering a crashed node's objects elsewhere. + +**What the framework handles:** the WebSocket join (signed ticket, transfer +ticket of a doorway handoff — single use across activations — or reconnect +token), a self-correcting tick loop that runs only while someone is +connected, the zone's snapshot in object storage (every +`ZONE_PERSIST_MS` while occupied, and when it empties), dormancy and +hibernation (`ZONE_HIBERNATE_AFTER_MS` later, by object alarm, so it also +happens after an eviction), restore on the next join, facts to the cells +of the same worker object to object (`CelldObjectHost` under the usual +`CommandFactSink` routes; a durable outbox in the object's storage makes +them exactly once across crashes), and the zone's lifecycle reports +(`active` / `dormant` / `hibernated`) into its +[`zone_activity`](src/world/zone_activity.rs) cell, which the +`zone_status` read model projects for minimaps. + +**Limits.** A live zone never moves (only idle objects are rebalanced), so +load spreads by first touch and node caps. Objects of one class share V8 +isolates: a slow zone delays its neighbours. A node crash takes about the +node lease TTL (`CELLD_TTL_MS`, 10 s) to recover and loses at most the +snapshot interval of zone state; facts survive in the outbox. A socket +that outlives its activation closes with 1012 and the SDK rejoins with a +fresh ticket. celld does not tell code which node it runs on: the node +shown in debug views comes from an opt-in operator lookup +(`CELLD_ZONE_NODES`), never used for routing. + +Measured on celld 0.6.2 (gr-p42 spike, a loaded 18-core laptop): about 150 +one-player rooms or 4–5 fifty-player rooms per core at 30 Hz; forwarding +through another node adds under a millisecond; a hibernated room wakes in +35–130 ms (p50). + +### Scaling out: the native multi-host server + +The `net` server can also spread zones over several processes itself +(Endless Grid and Boss Ripple use it): placement, ownership and +hibernation are then the framework's job. - **Placement.** A `ZoneDirectory` (`HashRing`) maps each zone id to a host. A client that connects to the wrong host is told where to go @@ -942,7 +1004,10 @@ trait and its ABI (`run_abi`, `call_bytes`). | *(default)* | Zone/host runtime, facts queue, protocol, schema, replication, authoring derives, game surface, predicted steps (compiles to `wasm32-unknown-unknown`, where `#[predict]` exports its steps) | `serde`, `serde_json`, `distributed_game_macros` | | `runtime` | `spawn_fact_worker` on Tokio | `tokio` (rt, time, sync) | | `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native, unit tests); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record`; the `world::zone_lease` and `world::zone_state` cell aggregates | `distributed`, `sha2`, `base64` | -| `command-host` | `command_sink::CommandFactSink`: facts → commands through any command host, routed by command (`FactRoute`) | `cell` + `distributed/graphql` | +| `cell` (also) | `command_sink::CommandFactSink`: facts → commands through any fact host, routed by command (`FactRoute`); `world::zone_activity` | | +| `command-host` | `FactHost` for `CelldCommandHost` / `LocalCommandHost` (`CelldFactSink`) | `cell` + `distributed/graphql` | +| `celld-zone` | `celld_zone`: any `Zone` as a celld Durable Object (`ZoneObject`, `celld_zone_object!`, `forward_to_zone`), facts object to object (`CelldObjectHost`), `serve_cell` / `ZoneActivityCell` for the worker's cells (wasm32) | `cell`, `ticket`, `workers-rs` + `js-sys`, `wasm-bindgen(-futures)` | +| `ticket` | `ticket`: HS256 admission and transfer tickets without the axum server | `hmac`, `sha2`, `base64` | | `celld` | `celld::CelldFactSink` (the command sink over a remote-only `CelldCommandHost`): facts → celld cells over the internal HTTP path; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells; `celld::CellZoneStateStore`: hibernation snapshots in celld `ZoneState` cells | `command-host` + `reqwest`, `uuid` | | `workers-rs` | `world::zone_state::celld_worker`: the `zone-state/{zone}` route of a celld worker (cell + R2 bucket binding for large snapshots) | `cell` + `worker` | | `metrics` | `metrics::GameMetrics` per-zone Prometheus text | – | diff --git a/distributed_game/src/bots.rs b/distributed_game/src/bots.rs index 8f76e9a8e..d5c3ac662 100644 --- a/distributed_game/src/bots.rs +++ b/distributed_game/src/bots.rs @@ -113,8 +113,13 @@ pub fn encode_i8_pair(dir: [f32; 2]) -> Vec { /// Bot run settings. #[derive(Clone)] pub struct BotConfig { - /// Server base URL, e.g. `ws://127.0.0.1:2567` (`/zones/{zone}` is appended). + /// Server base URL, e.g. `ws://127.0.0.1:2567` (`{path}/{zone}` is + /// appended). pub url: String, + /// Path the zone id is appended to: `/zones` on the `net` server, + /// whatever the worker routes to its zone objects on celld (e.g. + /// `/rooms`). + pub path: String, /// Zone to join. pub zone: String, /// Number of bots. @@ -145,6 +150,7 @@ impl BotConfig { pub fn new(url: impl Into, zone: impl Into, ticket_key: TicketKey) -> Self { Self { url: url.into(), + path: "/zones".into(), zone: zone.into(), bots: 10, input_rate: 20.0, @@ -364,7 +370,12 @@ async fn drive( name: None, extensions: Default::default(), }); - let url = format!("{}/zones/{}", config.url.trim_end_matches('/'), config.zone); + let url = format!( + "{}/{}/{}", + config.url.trim_end_matches('/'), + config.path.trim_matches('/'), + config.zone + ); let (mut ws, _) = tokio_tungstenite::connect_async(url.as_str()) .await .map_err(|e| format!("connect: {e}"))?; diff --git a/distributed_game/src/celld_zone/cells.rs b/distributed_game/src/celld_zone/cells.rs new file mode 100644 index 000000000..b03f7bce5 --- /dev/null +++ b/distributed_game/src/celld_zone/cells.rs @@ -0,0 +1,189 @@ +//! Serving fact cells next to celld zones (wasm32): the internal wait-path +//! route every fact cell of a game worker needs, and the zone-activity cell. + +use distributed::cell_host::{ + AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, + InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, +}; +use distributed::microsvc::Session; +use distributed::Aggregate; +use serde_json::{json, Value}; +use worker::{Env, Method, Request, Response, Result, State}; + +use crate::cell::fact_session_from_headers; +use crate::world::zone_activity::{mount_zone_activity, ZoneActivity}; + +/// Builds the handler session from trusted request headers (after the +/// internal secret was checked); `None` refuses the command. +pub type Sessions = fn(&dyn Fn(&str) -> Option) -> Option; + +/// Facts only: the principal and the `x-game-*` stamp +/// ([`fact_session_from_headers`]). +pub fn fact_sessions(header: &dyn Fn(&str) -> Option) -> Option { + fact_session_from_headers(header) +} + +/// Check `x-distributed-internal-secret` against +/// `DISTRIBUTED_INTERNAL_SECRET`. +pub fn authenticate(req: &Request, env: &Env) -> std::result::Result<(), CellDispatchError> { + let configured = env + .secret(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + .or_else(|_| { + env.var(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + }) + .map_err(|_| CellDispatchError::Internal("internal secret is required".into()))?; + let secret = InternalHttpSecret::new(configured).map_err(CellDispatchError::Internal)?; + let candidate = req + .headers() + .get(CELL_INTERNAL_SECRET_HEADER) + .ok() + .flatten() + .ok_or(CellDispatchError::Unauthorized)?; + if secret.matches(&candidate) { + Ok(()) + } else { + Err(CellDispatchError::Unauthorized) + } +} + +/// A JSON response with `status`. +pub fn json_status(body: Value, status: u16) -> Result { + Ok(Response::from_json(&body)?.with_status(status)) +} + +/// The wire form of a dispatch error. +pub fn error_response(error: CellDispatchError) -> Result { + json_status( + json!({ "error": error.client_message(), "code": error.code() }), + error.status_code(), + ) +} + +/// One cell's internal routes: `GET /{kind}/{shard}` → `read(cell)`, `POST +/// /{kind}/{shard}/{command}` → the idempotent wait-path dispatch (identity +/// from the service-id and principal-partition headers, the session from +/// `sessions`), then a best-effort outbox drain. Every request must carry +/// the internal secret. +pub async fn serve_cell( + env: &Env, + cell: &AggregateCell, + mut req: Request, + read: F, + sessions: Sessions, +) -> Result +where + A: Aggregate + Send + Sync + 'static, + F: AsyncFnOnce(&AggregateCell) -> std::result::Result, +{ + if let Err(error) = authenticate(&req, env) { + return error_response(error); + } + let url = req.url()?; + let parts: Vec = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .map(str::to_string) + .collect(); + match (req.method(), parts.get(2)) { + (Method::Get, None) => match read(cell).await { + Ok(body) => json_status(body, 200), + Err(error) => error_response(CellDispatchError::Internal(error)), + }, + (Method::Post, Some(command)) => { + let command = command.clone(); + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => { + return error_response(CellDispatchError::BadRequest("invalid JSON".into())) + } + }; + let parsed = match CellWaitPathRequest::parse(body) { + Ok(parsed) => parsed, + Err(error) => return error_response(CellDispatchError::BadRequest(error)), + }; + let header = |name: &str| req.headers().get(name).ok().flatten(); + let identity = match ( + header(CELL_SERVICE_ID_HEADER), + header(CELL_PRINCIPAL_PARTITION_HEADER), + ) { + (Some(service), Some(partition)) => { + match CellCommandIdentity::new(service, partition, &parsed.command_id) { + Ok(identity) => identity, + Err(error) => return error_response(error), + } + } + _ => return error_response(CellDispatchError::Unauthorized), + }; + let Some(session) = sessions(&header) else { + return error_response(CellDispatchError::Forbidden); + }; + let result = cell + .dispatch_idempotent(&command, &identity, parsed.input, session) + .await; + if let Err(error) = cell.drain_outbox(env).await { + worker::console_error!("post-commit drain deferred: {error}"); + } + match result { + Ok(dispatch) => json_status(dispatch.wait_path_wire(), 200), + Err(error) => error_response(error), + } + } + _ => json_status(json!({ "error": "not found" }), 404), + } +} + +/// A cell's alarm: due schedules, then the outbox drain. +pub async fn cell_alarm(env: &Env, cell: &AggregateCell) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let wake = cell + .alarm(env) + .await + .map_err(|error| worker::Error::RustError(error.to_string()))?; + for error in &wake.drain.deferred { + worker::console_error!("drain deferred: {error}"); + } + Response::ok("ok") +} + +/// The zone-activity cell of a game worker (`ZoneActivity:{zone}`, kind +/// `zone-activity`, binding `ZONE_ACTIVITY`): the zone's own lifecycle +/// facts; its `zone.activity_changed` events leave through `outbox` (the +/// worker's Queue) for the zone status read model. `GET` returns the row. +pub struct ZoneActivityCell { + cell: AggregateCell, + env: Env, +} + +impl ZoneActivityCell { + /// Open the cell of `state`, publishing its events to `outbox` (the + /// worker's Queue producer binding; cells commit only with one). + pub fn new(state: State, env: Env, outbox: CelldOutbox) -> Self { + let cell = mount_zone_activity( + AggregateCell::::from_state_with_snapshots(state, 20) + .expect("zone activity cell identity"), + ) + .with_celld_outbox(outbox); + Self { cell, env } + } + + /// Durable Object `fetch`. + pub async fn fetch(&self, req: Request) -> Result { + serve_cell(&self.env, &self.cell, req, read_activity, fact_sessions).await + } + + /// Durable Object `alarm`. + pub async fn alarm(&self) -> Result { + cell_alarm(&self.env, &self.cell).await + } +} + +async fn read_activity(cell: &AggregateCell) -> std::result::Result { + let activity = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ "activity": activity.map(|a| a.row()) })) +} diff --git a/distributed_game/src/celld_zone/mod.rs b/distributed_game/src/celld_zone/mod.rs new file mode 100644 index 000000000..e7b50c7ad --- /dev/null +++ b/distributed_game/src/celld_zone/mod.rs @@ -0,0 +1,85 @@ +//! Zones as celld Durable Objects (feature `celld-zone`): host any [`Zone`] +//! as one object per zone id. celld decides where a zone runs, forwards +//! every connection to it, moves idle zones, evicts sleeping ones and +//! recovers them after a node crash; game code and game APIs never name a +//! server. +//! +//! ```text +//! client ──WebSocket /rooms/{zone}──▶ any celld node ──forward──▶ the zone's object +//! │ ZoneHost (this activation) +//! Hello(ticket | transfer ticket | reconnect token) ┤ deadline tick loop while occupied +//! │ snapshot every few s + when empty +//! facts (outbox in object storage) ───┼──▶ cell objects (CommandFactSink +//! │ over CelldObjectHost, exactly once) +//! zone_activity.record (active / dormant / hibernated)┘──▶ zone-activity/{zone} cell +//! └ outbox ▶ Queue ▶ zone_status read model +//! ``` +//! +//! What the framework does per object ([`ZoneObject`], wasm32): +//! - **Join.** A WebSocket upgrade is accepted as a hibernatable socket; +//! its attachment holds the client id and the activation id. Hello is +//! verified with the shared HS256 [`TicketKey`](crate::ticket::TicketKey) +//! (env `GAME_TICKET_SECRET`): a join ticket, a transfer ticket from a +//! neighbouring zone's handoff (single use, also across activations) or a +//! reconnect token inside the grace window. +//! - **Ticks.** A self-correcting `setTimeout` loop over absolute deadlines +//! ([`TickSchedule`]; `setInterval` drifts) drives +//! [`ZoneHost::advance`](crate::ZoneHost::advance) while anyone is +//! connected, and stops when the zone is empty. +//! - **State.** The zone's [`Zone::hibernate`] snapshot is written to object +//! storage every `ZONE_PERSIST_MS` while occupied and when the last +//! player leaves (`dormant`); after `ZONE_HIBERNATE_AFTER_MS` empty (an +//! object alarm, so it survives eviction) the zone is dropped from memory +//! (`hibernated`). The next join wakes it and restores the snapshot. +//! - **Restarts.** A socket that outlives its activation (eviction, a move, +//! a crash) is closed with [`CLOSE_RESTARTED`] (1012); the client rejoins +//! with a fresh ticket. +//! - **Facts.** Zone facts go to cells of the same worker object to object +//! ([`CelldObjectHost`] under [`CommandFactSink`](crate::command_sink::CommandFactSink), +//! the same routes and command ids as over HTTP). Each fact is written to +//! the object's outbox before it is sent and deleted when settled; the +//! next activation or the outbox alarm resends what is left, and the +//! cells' command ledger turns a resend into a replay: exactly once. +//! - **Status.** The zone reports its lifecycle as facts into its +//! [`zone_activity`](crate::world::zone_activity) cell, projected into the +//! [`zone_status`](crate::world::zone_status) read model (minimaps). +//! +//! What celld does: placement (first touch, overflow by +//! `CELLD_MAX_RESIDENT_CELLS`), forwarding from any node, idle eviction, +//! rebalancing of idle objects, graceful drain and crash recovery (fenced +//! by its own epochs; one instance per object name). +//! +//! Limits: a live zone never moves (only idle ones are rebalanced), so +//! spreading load depends on first touch and node caps; objects of one +//! class share V8 isolates (a slow zone delays its neighbours); a crash +//! loses at most the snapshot interval of zone state (facts are kept by +//! the outbox) and takes the node lease TTL (~10 s) to recover. +//! +//! Debug only: celld does not tell application code which node runs an +//! object. With `CELLD_ZONE_NODES` set (operator listeners) the object asks +//! each node whether it holds it; `CELLD_ZONE_NODE` names a fixed node. +//! Both are off by default; nothing routes by them. + +pub mod plan; + +#[cfg(target_arch = "wasm32")] +mod cells; +#[cfg(target_arch = "wasm32")] +mod object; + +pub use plan::{ + cell_binding, ObjectOptions, RoomState, StoredFact, TickSchedule, CLOSE_RESTARTED, ZONE_HEADER, +}; + +#[cfg(target_arch = "wasm32")] +pub use cells::{ + authenticate, cell_alarm, error_response, fact_sessions, json_status, serve_cell, Sessions, + ZoneActivityCell, +}; +#[cfg(target_arch = "wasm32")] +pub use object::{ + forward_to_zone, CelldObjectHost, CelldZone, LocalFuture, ObjectEnv, ObjectFactSink, ZoneObject, +}; + +#[cfg(doc)] +use crate::Zone; diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs new file mode 100644 index 000000000..9e3478529 --- /dev/null +++ b/distributed_game/src/celld_zone/object.rs @@ -0,0 +1,1410 @@ +//! The Durable Object side of the celld zone host (wasm32 only). + +use std::cell::RefCell; +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::rc::Rc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use distributed::cell_host::{ + CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, + CELL_SERVICE_ID_HEADER, +}; +use distributed::USER_ID_KEY; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use worker::send::{SendFuture, SendWrapper}; +use worker::{ + Delay, Env, Headers, Method, Request, RequestInit, Response, Result, State, Storage, WebSocket, + WebSocketIncomingMessage, WebSocketPair, +}; + +use super::plan::{ + cell_binding, classify_wait_path, zone_from_path, ObjectOptions, RoomState, StoredFact, + TickSchedule, Window, CLOSE_RESTARTED, OUTBOX_PREFIX, ZONE_HEADER, +}; +use crate::command_sink::{CommandFactSink, FactCall, FactHost, FactRoute}; +use crate::facts::{ + fact_channel, BoxFuture, Fact, FactError, FactQueue, FactReceipt, FactSink, RetryPolicy, +}; +use crate::handoff::{HandoffRequest, Transfer}; +use crate::inbox::ZoneInbound; +use crate::protocol::{error, ClientMessage, Role, ServerMessage, PROTOCOL_VERSION}; +use crate::stamp::FactStamp; +use crate::ticket::{NonceCache, TicketKey, VerifiedTicket}; +use crate::world::zone_activity::{ + zone_principal, RecordActivity, ACTIVITY_ACTIVE, ACTIVITY_DORMANT, ACTIVITY_HIBERNATED, + ZONE_ACTIVITY_KIND, +}; +use crate::{ClientId, Clock, HostConfig, Outbound, TicketClaims, Zone, ZoneHost}; + +/// A future that runs on the object's (single) thread. +pub type LocalFuture = Pin + 'static>>; + +/// The zone's fact sink on celld: commands to cells, object to object. +pub type ObjectFactSink = CommandFactSink; + +const SNAPSHOT_KEY: &str = "zone:snapshot"; +const STATUS_KEY: &str = "zone:status"; +const NONCE_PREFIX: &str = "nonce:"; +/// Transfer tickets (doorways) live this long. +const TRANSFER_TTL_SECS: u64 = 30; +/// Unsettled facts one activation may hold. +const FACT_CAPACITY: usize = 512; +/// Retry stored facts this often while any is unsettled (ms). +const OUTBOX_RETRY_MS: f64 = 10_000.0; + +/// A game zone the framework can host as a celld Durable Object: one object +/// per zone id (celld places it, forwards to it, hibernates and moves it). +pub trait CelldZone: Zone { + /// The zone's construction parameters for `zone_id` (validate the id + /// here: `Err` refuses the join). + fn object_config(zone_id: &str, env: &ObjectEnv) -> std::result::Result; + + /// Host tuning; the object sets the tick and patch rates, reconnect + /// grace and boot epoch from its options. Default: [`HostConfig`] + /// defaults. + fn host_config(_zone_id: &str, _env: &ObjectEnv) -> HostConfig { + HostConfig::default() + } + + /// Fact routes of the zone (lifecycle facts too, if the host config + /// emits them). The zone-activity route is added by the framework. + fn fact_routes(sink: ObjectFactSink) -> ObjectFactSink { + sink + } + + /// Async work after a player joined (e.g. read the player's profile + /// from its cell); the message it returns is delivered into the zone + /// ([`Zone::on_message`]). Runs off the tick; the zone must not wait + /// for it. + fn after_join( + _env: &ObjectEnv, + _principal: &str, + ) -> Option>> { + None + } +} + +// ------------------------------------------------------------------ env + +/// The object's environment for game hooks: vars and cell reads. +#[derive(Clone)] +pub struct ObjectEnv { + env: Env, +} + +impl ObjectEnv { + /// Wrap a worker env. + pub fn new(env: Env) -> Self { + Self { env } + } + + /// A var or secret. + pub fn var(&self, name: &str) -> Option { + self.env + .var(name) + .map(|v| v.to_string()) + .or_else(|_| self.env.secret(name).map(|v| v.to_string())) + .ok() + } + + /// The worker env. + pub fn env(&self) -> &Env { + &self.env + } + + /// `GET /{kind}/{shard}` on a cell of this worker (its read route), + /// object to object, with the internal secret. + pub async fn read_cell(&self, kind: &str, shard: &str) -> std::result::Result { + let secret = self + .var(CELL_INTERNAL_SECRET_ENV) + .ok_or_else(|| format!("{CELL_INTERNAL_SECRET_ENV} is required"))?; + let headers = Headers::new(); + headers + .set(CELL_INTERNAL_SECRET_HEADER, &secret) + .map_err(|e| e.to_string())?; + let mut init = RequestInit::new(); + init.with_method(Method::Get).with_headers(headers); + let url = format!("https://cell/{kind}/{}", encode_segment(shard)); + let request = Request::new_with_init(&url, &init).map_err(|e| e.to_string())?; + let stub = stub_for(&self.env, kind, shard).map_err(|e| e.to_string())?; + let mut response = stub + .fetch_with_request(request) + .await + .map_err(|e| e.to_string())?; + let status = response.status_code(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if (200..300).contains(&status) { + Ok(body) + } else { + Err(format!("HTTP {status}: {body}")) + } + } +} + +/// The stub of cell `kind/shard` (binding [`cell_binding`]`(kind)`). +fn stub_for(env: &Env, kind: &str, shard: &str) -> Result { + let namespace = env.durable_object(&cell_binding(kind))?; + let id = namespace.id_from_name(shard)?; + id.get_stub() +} + +fn encode_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b':') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +// ------------------------------------------------------------------ fact host + +/// [`FactHost`] over the cells of the same worker: the cell wait-path +/// request (`POST /{kind}/{shard}/{command}`, internal secret, principal, +/// `x-game-*` stamp) sent straight to the cell's Durable Object stub — +/// object to object, no HTTP hop through a node's public listener. The +/// binding of kind `k` is [`cell_binding`]`(k)`. +pub struct CelldObjectHost { + env: SendWrapper, + secret: String, + service_id: String, +} + +impl CelldObjectHost { + /// A host for `env`'s cells under command-ledger `service_id`. + pub fn new(env: Env, secret: impl Into, service_id: impl Into) -> Self { + Self { + env: SendWrapper::new(env), + secret: secret.into(), + service_id: service_id.into(), + } + } + + async fn post(&self, call: &FactCall<'_>) -> std::result::Result { + let retry = |e: worker::Error| FactError::Retryable(e.to_string()); + let headers = Headers::new(); + let set = |name: &str, value: &str| { + headers + .set(name, value) + .map_err(|e| FactError::Permanent(format!("header {name}: {e}"))) + }; + set(CELL_INTERNAL_SECRET_HEADER, &self.secret)?; + set(CELL_SERVICE_ID_HEADER, &self.service_id)?; + set(CELL_PRINCIPAL_PARTITION_HEADER, &call.principal)?; + set(USER_ID_KEY, &call.principal)?; + set("content-type", "application/json")?; + for (name, value) in call.session.variables() { + if name.starts_with(crate::stamp::SESSION_PREFIX) { + set(name, value)?; + } + } + let body = json!({ "commandId": call.command_id, "input": call.input }).to_string(); + let mut init = RequestInit::new(); + init.with_method(Method::Post) + .with_headers(headers) + .with_body(Some(body.into())); + let url = format!( + "https://cell/{}/{}/{}", + encode_segment(call.kind), + encode_segment(&call.shard), + encode_segment(call.command) + ); + let request = Request::new_with_init(&url, &init).map_err(retry)?; + let stub = stub_for(&self.env, call.kind, &call.shard) + .map_err(|e| FactError::Permanent(format!("cell `{}`: {e}", call.kind)))?; + let mut response = stub.fetch_with_request(request).await.map_err(retry)?; + let status = response.status_code(); + let body: Value = response.json().await.unwrap_or(Value::Null); + classify_wait_path(status, &body) + } +} + +impl FactHost for CelldObjectHost { + fn dispatch_fact<'a>( + &'a self, + call: FactCall<'a>, + ) -> BoxFuture<'a, std::result::Result> { + Box::pin(SendFuture::new(async move { self.post(&call).await })) + } +} + +/// The durable fact outbox around the sink: every emitted fact is written +/// to object storage right after the call that emitted it (from the fact +/// queues' journals, [`FactQueue::with_journal`]) and deleted here once it +/// settled (committed or permanently refused). The next activation (or the +/// outbox alarm) resends what is left; the cells' command ledger makes a +/// resend a replay. +#[derive(Clone)] +struct DurableFacts { + sink: ObjectFactSink, + storage: SendWrapper>, +} + +impl FactSink for DurableFacts { + fn commit<'a>( + &'a self, + fact: &'a Fact, + ) -> BoxFuture<'a, std::result::Result> { + Box::pin(SendFuture::new(async move { + let result = self.sink.commit(fact).await; + match &result { + Err(FactError::Permanent(e)) => { + worker::console_error!("fact {} refused: {e}", fact.key()) + } + Err(FactError::Retryable(e)) => { + worker::console_warn!("fact {} will be retried: {e}", fact.key()) + } + Ok(_) => {} + } + if !matches!(result, Err(FactError::Retryable(_))) { + let _ = self + .storage + .delete(&StoredFact::storage_key(fact.key())) + .await; + } + result + })) + } +} + +// ------------------------------------------------------------------ time + +fn now_ms() -> f64 { + js_sys::Date::now() +} + +/// Monotonic host clock over `Date.now()` (`Instant` panics on +/// wasm32-unknown-unknown). +struct JsClock { + origin_ms: f64, + last_us: AtomicU64, +} + +impl JsClock { + fn new() -> Self { + Self { + origin_ms: now_ms(), + last_us: AtomicU64::new(0), + } + } +} + +impl Clock for JsClock { + fn now(&self) -> Duration { + let us = ((now_ms() - self.origin_ms).max(0.0) * 1000.0) as u64; + Duration::from_micros(self.last_us.fetch_max(us, Ordering::Relaxed).max(us)) + } + + fn unix_ms(&self) -> u64 { + self.origin_ms as u64 + self.now().as_millis() as u64 + } +} + +/// `n` random bytes as hex, from `crypto.getRandomValues`. +fn random_hex(n: usize) -> String { + let bytes = js_sys::Uint8Array::new_with_length(n as u32); + let crypto = js_sys::Reflect::get(&js_sys::global(), &"crypto".into()).ok(); + let filled = crypto.and_then(|crypto| { + let get = js_sys::Reflect::get(&crypto, &"getRandomValues".into()).ok()?; + let get: js_sys::Function = get.into(); + get.call1(&crypto, &bytes).ok() + }); + if filled.is_none() { + for i in 0..n { + bytes.set_index(i as u32, (js_sys::Math::random() * 256.0) as u8); + } + } + bytes.to_vec().iter().map(|b| format!("{b:02x}")).collect() +} + +async fn yield_now() { + let _ = wasm_bindgen_futures::JsFuture::from(js_sys::Promise::resolve( + &wasm_bindgen::JsValue::NULL, + )) + .await; +} + +thread_local! { + /// One id per V8 isolate (module instance), for the debug info. + static ISOLATE: String = random_hex(4); +} + +// ------------------------------------------------------------------ object + +#[derive(Serialize, Deserialize)] +struct Attachment { + client: u32, + activation: String, +} + +#[derive(Serialize, Deserialize)] +struct Snapshot { + version: String, + state: String, +} + +#[derive(Default)] +struct TickStats { + late: Window, + work: Window, + dropped: u64, + iterations: u64, +} + +struct Inner { + zone_id: String, + activation: String, + activation_epoch: u64, + options: Option, + key: Option, + host: Option>, + restored: bool, + wakes: u64, + state: RoomState, + sockets: HashMap, + ticking: bool, + nonces: NonceCache, + sink: Option, + activity: Option, + activity_seq: u64, + outbox_loaded: bool, + last_status: String, + node: String, + stats: TickStats, + last_persist_ms: f64, + spooled: u64, +} + +impl Inner { + fn occupied(&self) -> bool { + self.host + .as_ref() + .is_some_and(|h| h.player_count() > 0 || h.client_count() > 0) + } + + fn flush(&mut self) { + let Some(host) = self.host.as_mut() else { + return; + }; + for out in host.drain_outbound() { + match out { + Outbound::Frame { client, frame } => { + if let Some(ws) = self.sockets.get(&client.0) { + let _ = ws.send_with_bytes(frame); + } + } + Outbound::Close { client } => { + if let Some(ws) = self.sockets.remove(&client.0) { + let _ = ws.close(Some(1000), Some("closed by zone")); + } + } + } + } + } + + /// Facts emitted since the last call (zone and activity queues). + fn emitted(&mut self) -> Vec { + let mut facts = self + .host + .as_mut() + .map(ZoneHost::take_emitted_facts) + .unwrap_or_default(); + if let Some(queue) = self.activity.as_mut() { + facts.extend(queue.take_journal()); + } + facts + } + + fn snapshot(&self) -> Option { + let host = self.host.as_ref()?; + let state = host.hibernate_state()?; + use base64::Engine as _; + Some(Snapshot { + version: host.config().content_version.clone(), + state: base64::engine::general_purpose::STANDARD.encode(state), + }) + } + + /// Queue an activity report (a zone fact into its zone-activity cell). + fn report(&mut self, status: &str) { + if let Some(queue) = self.activity.as_mut() { + queue.drain(); + } + let Some(queue) = self.activity.as_mut() else { + return; + }; + self.activity_seq += 1; + let input = RecordActivity { + status: status.to_string(), + activation: self.activation.clone(), + restored: self.restored, + players: self.host.as_ref().map_or(0, |h| h.player_count() as u32), + node: self.node.clone(), + isolate: ISOLATE.with(|i| i.clone()), + }; + let stamp = FactStamp { + zone_id: self.zone_id.clone(), + tick: self.host.as_ref().map_or(0, |h| h.tick()), + emitted_at_ms: now_ms() as u64, + activation: self.activation_epoch, + seq: self.activity_seq, + ..FactStamp::default() + }; + let key = format!( + "zone_activity.record:{}:{}:{}", + self.zone_id, self.activation_epoch, self.activity_seq + ); + match serde_json::to_value(&input) { + Ok(value) => { + let fact = Fact::trusted( + key, + crate::world::zone_activity::RECORD, + value, + Some(zone_principal(&self.zone_id)), + stamp, + ); + if let Err(e) = queue.try_send(fact) { + worker::console_error!("zone {}: activity report dropped: {e:?}", self.zone_id); + } + } + Err(e) => worker::console_error!("zone activity encode: {e}"), + } + self.last_status = status.to_string(); + } +} + +/// Hosts one [`CelldZone`] as a celld Durable Object. The game's +/// `#[durable_object]` delegates to it (see [`crate::celld_zone_object!`]). +/// +/// Routes (on the object, via [`forward_to_zone`]): a WebSocket upgrade +/// joins the zone (the `distributed_game` wire protocol: Hello with a +/// ticket, a transfer ticket or a reconnect token); `GET …/info` returns +/// debug JSON (activation, node, isolate, tick timing) without waking a +/// hibernated zone. +pub struct ZoneObject { + state: Rc, + env: ObjectEnv, + inner: Rc>>, +} + +impl ZoneObject { + /// The object for `state` (constructor of the Durable Object). + pub fn new(state: State, env: Env) -> Self { + let zone_id = state.id().name().unwrap_or_default(); + let env = ObjectEnv::new(env); + let options = ObjectOptions::from_vars(|name| env.var(name)); + if let Err(e) = &options { + worker::console_error!("celld zone: {e}"); + } + let options = options.ok(); + let key = options + .as_ref() + .map(|o| TicketKey::new(o.ticket_secret.clone().into_bytes())); + let node = options + .as_ref() + .and_then(|o| o.node.clone()) + .unwrap_or_default(); + let inner = Inner { + zone_id, + activation: random_hex(6), + activation_epoch: (now_ms() * 1000.0) as u64, + options, + key, + host: None, + restored: false, + wakes: 0, + state: RoomState::Cold, + sockets: HashMap::new(), + ticking: false, + nonces: NonceCache::default(), + sink: None, + activity: None, + activity_seq: 0, + outbox_loaded: false, + last_status: String::new(), + node, + stats: TickStats::default(), + last_persist_ms: 0.0, + spooled: 0, + }; + Self { + state: Rc::new(state), + env, + inner: Rc::new(RefCell::new(inner)), + } + } + + /// The game hook environment. + pub fn env(&self) -> &ObjectEnv { + &self.env + } + + /// Durable Object `fetch`. + pub async fn fetch(&self, req: Request) -> Result { + if let Some(zone) = req.headers().get(ZONE_HEADER)? { + let mut inner = self.inner.borrow_mut(); + if inner.zone_id.is_empty() { + inner.zone_id = zone; + } + } + if self.inner.borrow().zone_id.is_empty() { + return Response::error("zone id required", 400); + } + let path = req.url()?.path().to_string(); + if path.ends_with("/info") { + // Debug JSON without secrets; pages on other origins may read it. + let response = Response::from_json(&self.info())?; + response.headers().set("access-control-allow-origin", "*")?; + return Ok(response); + } + if req.headers().get("upgrade")?.as_deref() != Some("websocket") { + return Response::error("expected a WebSocket upgrade", 426); + } + if self.inner.borrow().options.is_none() { + return Response::error("zone host is not configured", 503); + } + self.wake().await?; + let pair = WebSocketPair::new()?; + self.state.accept_web_socket(&pair.server); + Response::from_websocket(pair.client) + } + + /// Durable Object `websocket_message`. + pub async fn websocket_message( + &self, + ws: WebSocket, + message: WebSocketIncomingMessage, + ) -> Result<()> { + let WebSocketIncomingMessage::Binary(bytes) = message else { + let _ = ws.close(Some(1003), Some("binary frames only")); + return Ok(()); + }; + let attached: Option = ws.deserialize_attachment().ok().flatten(); + let activation = self.inner.borrow().activation.clone(); + let client = match attached { + Some(a) if a.activation == activation => Some(a.client), + // The socket outlived the activation that admitted it. + Some(_) => { + let _ = ws.close(Some(CLOSE_RESTARTED), Some("zone restarted; rejoin")); + return Ok(()); + } + None => None, + }; + let msg = match ClientMessage::decode(&bytes) { + Ok(m) => m, + Err(_) => { + reject(&ws, error::BAD_MESSAGE, "bad message"); + return Ok(()); + } + }; + match (client, msg) { + (None, ClientMessage::Hello(hello)) => self.hello(&ws, hello).await?, + ( + Some(client), + ClientMessage::Input { + seq, + client_tick, + payload, + }, + ) => { + let mut inner = self.inner.borrow_mut(); + match inner.host.as_mut() { + Some(host) => { + host.push_input(ClientId(client), seq, client_tick, &payload); + } + None => { + drop(inner); + let _ = ws.close(Some(CLOSE_RESTARTED), Some("zone asleep; rejoin")); + } + } + } + (_, ClientMessage::Ping { client_time }) => { + let server_time = self + .inner + .borrow() + .host + .as_ref() + .map_or(0.0, |h| h.server_time_ms()); + if let Ok(pong) = (ServerMessage::Pong { + client_time, + server_time, + }) + .encode(&()) + { + let _ = ws.send_with_bytes(pong); + } + } + (Some(client), ClientMessage::Leave) => { + { + let mut inner = self.inner.borrow_mut(); + inner.sockets.remove(&client); + if let Some(host) = inner.host.as_mut() { + host.disconnect(ClientId(client), true); + } + inner.flush(); + } + self.spool().await; + let _ = ws.close(Some(1000), Some("left")); + } + _ => reject(&ws, error::BAD_MESSAGE, "unexpected message"), + } + Ok(()) + } + + /// Durable Object `websocket_close`. + pub async fn websocket_close( + &self, + ws: WebSocket, + _code: usize, + _reason: String, + _was_clean: bool, + ) -> Result<()> { + self.drop_socket(&ws); + Ok(()) + } + + /// Durable Object `websocket_error`. + pub async fn websocket_error(&self, ws: WebSocket, _error: worker::Error) -> Result<()> { + self.drop_socket(&ws); + Ok(()) + } + + /// Durable Object `alarm`: hibernate a room that stayed empty, resend + /// unsettled facts, and after a crash mark the room asleep. + pub async fn alarm(&self) -> Result { + if self.inner.borrow().options.is_none() { + return Response::ok("unconfigured"); + } + self.load_zone_id().await; + self.start_facts().await; + let (cold, due, hibernate_after) = { + let inner = self.inner.borrow(); + let after = inner + .options + .as_ref() + .map_or(Duration::from_secs(10), |o| o.hibernate_after); + ( + inner.host.is_none(), + !inner.occupied() && inner.state.hibernate_due(now_ms(), after), + after, + ) + }; + if due { + self.hibernate().await; + } else if cold { + // A fresh activation woken by its alarm: the last one crashed or + // was evicted while awake. Its snapshot is what it left. + let stored: Option = self.state.storage().get(STATUS_KEY).await.ok().flatten(); + if stored.as_deref().is_some_and(|s| s != ACTIVITY_HIBERNATED) { + self.inner.borrow_mut().report(ACTIVITY_HIBERNATED); + self.store_status().await; + } + } else { + let since = match self.inner.borrow().state { + RoomState::Dormant { since_ms } => Some(since_ms), + _ => None, + }; + if let Some(since) = since { + let wait = since + hibernate_after.as_millis() as f64 - now_ms(); + self.arm(wait.max(10.0)).await; + } + } + self.arm_outbox().await; + Response::ok("ok") + } + + // -------------------------------------------------------------- internals + + /// Sockets accepted before an eviction reach a new instance without a + /// zone id from a request; the id is kept in storage. + async fn load_zone_id(&self) { + if !self.inner.borrow().zone_id.is_empty() { + return; + } + if let Ok(Some(zone)) = self.state.storage().get::("zone:id").await { + let mut inner = self.inner.borrow_mut(); + if inner.zone_id.is_empty() { + inner.zone_id = zone; + } + } + } + + /// Start the fact pipeline once per activation and resend what the + /// previous activation left in the outbox. + async fn start_facts(&self) { + if self.inner.borrow().sink.is_some() { + return; + } + let (zone_id, service, secret) = { + let inner = self.inner.borrow(); + let Some(options) = inner.options.as_ref() else { + return; + }; + ( + inner.zone_id.clone(), + options.fact_service.clone(), + options.internal_secret.clone(), + ) + }; + let Some(secret) = secret else { + worker::console_error!( + "zone {zone_id}: {CELL_INTERNAL_SECRET_ENV} is not set; facts are disabled" + ); + return; + }; + let host = CelldObjectHost::new(self.env.env().clone(), secret, &service); + let sink = match ObjectFactSink::new(host, service, zone_principal(&zone_id)) { + Ok(sink) => { + Z::fact_routes(sink.route(FactRoute::by_zone::(ZONE_ACTIVITY_KIND))) + } + Err(e) => { + worker::console_error!("zone {zone_id}: fact sink: {e}"); + return; + } + }; + let durable = DurableFacts { + sink, + storage: SendWrapper::new(Rc::new(self.state.storage())), + }; + let (queue, feed) = fact_channel(FACT_CAPACITY, retry_policy()); + let queue = queue.with_journal(); + wasm_bindgen_futures::spawn_local( + feed.with_max_concurrent_commits(4) + .run(durable.clone(), sleep), + ); + { + let mut inner = self.inner.borrow_mut(); + if inner.sink.is_some() { + return; + } + inner.sink = Some(durable); + inner.activity = Some(queue); + } + let _ = self.state.storage().put("zone:id", &zone_id).await; + // Resend the previous activations' unsettled facts. + let stored = self + .state + .storage() + .list_with_options(worker::ListOptions::new().prefix(OUTBOX_PREFIX)) + .await; + let mut inner = self.inner.borrow_mut(); + inner.outbox_loaded = true; + if let Ok(map) = stored { + let Some(queue) = inner.activity.as_mut() else { + return; + }; + for value in map.values().into_iter().flatten() { + let Some(stored) = value + .as_string() + .and_then(|text| serde_json::from_str::(&text).ok()) + else { + continue; + }; + if let Some(fact) = stored.into_fact() { + let _ = queue.try_send(fact); + } + } + // Already stored. + queue.take_journal(); + } + } + + /// Make sure the zone is in memory (restoring its snapshot). + async fn wake(&self) -> Result<()> { + self.start_facts().await; + if self.inner.borrow().host.is_some() { + return Ok(()); + } + let stored: Option = self.state.storage().get(SNAPSHOT_KEY).await.ok().flatten(); + let host = { + let inner = self.inner.borrow(); + if inner.host.is_some() { + return Ok(()); + } + let options = inner + .options + .as_ref() + .ok_or_else(|| worker::Error::RustError("zone host is not configured".into()))?; + let zone_id = inner.zone_id.clone(); + let config = Z::object_config(&zone_id, &self.env).map_err(worker::Error::RustError)?; + let mut host_config = Z::host_config(&zone_id, &self.env); + host_config.tick_rate = options.tick_hz; + host_config.patch_rate = options.patch_hz; + // The object stops ticking when empty instead of idling. + host_config.idle_tick_rate = None; + host_config.reconnect_grace = options.reconnect_grace; + host_config.boot_epoch = (now_ms() * 1000.0) as u64; + let key = inner.key.clone().expect("ticket key with options"); + let issuer_zone = zone_id.clone(); + let generation = inner.activation_epoch; + let mut host = + ZoneHost::::new(zone_id, host_config, Arc::new(JsClock::new()), config) + .with_token_source(|| random_hex(24)) + .with_transfer_issuer(move |request: &HandoffRequest| { + let expires_at = (now_ms() / 1000.0) as u64 + TRANSFER_TTL_SECS; + Some(key.sign_transfer(&Transfer { + claims: TicketClaims { + principal: request.principal.clone(), + zone: request.target_zone.clone(), + expires_at, + nonce: random_hex(16), + role: Role::Player, + name: request.name.clone(), + extensions: request.extensions.clone(), + }, + source_zone: request.source_zone.clone(), + issuer: issuer_zone.clone(), + generation, + carried: request.carried.clone(), + departed_at_ms: request.departed_at_ms, + })) + }); + let (queue, feed) = fact_channel(FACT_CAPACITY, retry_policy()); + if let Some(sink) = inner.sink.clone() { + wasm_bindgen_futures::spawn_local( + feed.with_max_concurrent_commits(4).run(sink, sleep), + ); + host = host.with_facts(queue.with_journal()); + } + let restored = match &stored { + Some(snapshot) => { + use base64::Engine as _; + match base64::engine::general_purpose::STANDARD.decode(&snapshot.state) { + Ok(state) => { + host.restore(&state, &snapshot.version); + true + } + Err(_) => false, + } + } + None => false, + }; + (host, restored) + }; + let (host, restored) = host; + { + let mut inner = self.inner.borrow_mut(); + if inner.host.is_some() { + return Ok(()); + } + inner.host = Some(host); + inner.restored = restored; + inner.wakes += 1; + // Each wake is a new activation: sockets of an earlier one are + // told to rejoin (1012). + if inner.wakes > 1 { + inner.activation = random_hex(6); + } + inner.state = RoomState::Active; + inner.last_persist_ms = now_ms(); + inner.report(ACTIVITY_ACTIVE); + } + self.store_status().await; + self.find_node(); + Ok(()) + } + + async fn hello(&self, ws: &WebSocket, hello: crate::protocol::Hello) -> Result<()> { + if hello.protocol_version != PROTOCOL_VERSION { + reject(ws, error::VERSION_MISMATCH, "protocol version"); + return Ok(()); + } + if let Err(e) = self.wake().await { + reject(ws, error::NOT_FOUND, &e.to_string()); + return Ok(()); + } + let now = (now_ms() / 1000.0) as u64; + // Transfer tickets are single use across activations too. + let mut principal = None; + let joined = if hello.reconnect_token.is_empty() { + let verified = { + let inner = self.inner.borrow(); + let key = inner.key.as_ref().expect("configured"); + key.verify_ticket(&hello.ticket, now) + }; + let verified = match verified { + Ok(v) => v, + Err(e) => { + reject(ws, error::UNAUTHORIZED, &format!("{e:?}")); + return Ok(()); + } + }; + let zone_id = self.inner.borrow().zone_id.clone(); + let (claims, transfer) = match verified { + VerifiedTicket::Join(claims) => (claims, None), + VerifiedTicket::Transfer(transfer) => (transfer.claims.clone(), Some(transfer)), + }; + if claims.zone != zone_id { + reject(ws, error::UNAUTHORIZED, "ticket is for another zone"); + return Ok(()); + } + let replayed = self + .inner + .borrow_mut() + .nonces + .check_and_insert(&claims.nonce, claims.expires_at, now) + .is_err(); + let nonce_key = format!("{NONCE_PREFIX}{}", claims.nonce); + let used_before = transfer.is_some() + && self + .state + .storage() + .get::(&nonce_key) + .await + .ok() + .flatten() + .is_some(); + if replayed || used_before { + reject(ws, error::UNAUTHORIZED, "ticket already used"); + return Ok(()); + } + if transfer.is_some() { + let _ = self + .state + .storage() + .put(&nonce_key, claims.expires_at) + .await; + } + principal = Some(claims.principal.clone()); + let mut inner = self.inner.borrow_mut(); + let host = inner.host.as_mut().expect("woken"); + match &transfer { + Some(transfer) => { + host.join_transfer(transfer, hello.capabilities, &hello.content_version) + } + None => host.join(&claims, hello.capabilities, &hello.content_version), + } + } else { + let mut inner = self.inner.borrow_mut(); + let host = inner.host.as_mut().expect("woken"); + host.reconnect_with_version( + &hello.reconnect_token, + hello.capabilities, + &hello.content_version, + ) + }; + match joined { + Ok(joined) => { + { + let mut inner = self.inner.borrow_mut(); + let activation = inner.activation.clone(); + ws.serialize_attachment(Attachment { + client: joined.client.0, + activation, + })?; + inner.sockets.insert(joined.client.0, ws.clone()); + inner.flush(); + if inner.state != RoomState::Active { + inner.state = RoomState::Active; + inner.report(ACTIVITY_ACTIVE); + } + } + self.store_status().await; + self.start_loop(); + if let Some(principal) = principal { + self.after_join(principal); + } + } + Err(e) => reject(ws, e.code, &e.message), + } + Ok(()) + } + + fn after_join(&self, principal: String) { + let Some(work) = Z::after_join(&self.env, &principal) else { + return; + }; + let inner = self.inner.clone(); + wasm_bindgen_futures::spawn_local(async move { + if let Some(message) = work.await { + let mut inner = inner.borrow_mut(); + if let Some(host) = inner.host.as_mut() { + host.deliver(ZoneInbound::Message(message)); + inner.flush(); + } + // Facts the message produced are written on the next tick. + } + }); + } + + fn drop_socket(&self, ws: &WebSocket) { + let attached: Option = ws.deserialize_attachment().ok().flatten(); + let mut inner = self.inner.borrow_mut(); + if let Some(a) = attached.filter(|a| a.activation == inner.activation) { + inner.sockets.remove(&a.client); + if let Some(host) = inner.host.as_mut() { + host.disconnect(ClientId(a.client), false); + } + inner.flush(); + drop(inner); + self.spool_later(); + } + } + + fn start_loop(&self) { + { + let mut inner = self.inner.borrow_mut(); + if inner.ticking || inner.host.is_none() { + return; + } + inner.ticking = true; + } + let this = self.handle(); + wasm_bindgen_futures::spawn_local(async move { this.tick_loop().await }); + } + + fn handle(&self) -> Self { + Self { + state: self.state.clone(), + env: self.env.clone(), + inner: self.inner.clone(), + } + } + + async fn tick_loop(&self) { + let (hz, persist_every) = { + let inner = self.inner.borrow(); + let options = inner.options.as_ref().expect("configured"); + (options.tick_hz, options.persist_every.as_millis() as f64) + }; + let mut schedule = TickSchedule::new(now_ms(), hz); + self.arm(persist_every * 3.0).await; + loop { + Delay::from(Duration::from_micros( + (schedule.wait_ms(now_ms()) * 1000.0) as u64, + )) + .await; + let woke = now_ms(); + let late = schedule.woke(woke); + let (stop, persist) = { + let mut inner = self.inner.borrow_mut(); + let Some(host) = inner.host.as_mut() else { + inner.ticking = false; + return; + }; + let report = host.advance(); + inner.flush(); + let stats = &mut inner.stats; + stats.iterations += 1; + stats.late.push(late); + stats.work.push(now_ms() - woke); + stats.dropped += u64::from(report.dropped_ticks); + if let Some(queue) = inner.activity.as_mut() { + queue.drain(); + } + let stop = !inner.occupied(); + let persist = stop || woke - inner.last_persist_ms >= persist_every; + if stop { + inner.ticking = false; + } + (stop, persist) + }; + self.spool().await; + if persist { + self.persist().await; + if !stop { + self.arm(persist_every * 3.0).await; + } + } + if stop { + self.go_dormant().await; + return; + } + } + } + + /// Write the zone's snapshot (after the fact outbox writes queued by + /// this tick, so a crash never keeps an outcome without its fact). + async fn persist(&self) { + yield_now().await; + let snapshot = { + let mut inner = self.inner.borrow_mut(); + inner.last_persist_ms = now_ms(); + inner.snapshot() + }; + if let Some(snapshot) = snapshot { + if let Err(e) = self.state.storage().put(SNAPSHOT_KEY, snapshot).await { + worker::console_error!("zone snapshot write failed: {e}"); + } + } + } + + async fn go_dormant(&self) { + let after = { + let mut inner = self.inner.borrow_mut(); + if inner.occupied() { + return; + } + inner.state = RoomState::Dormant { since_ms: now_ms() }; + inner.report(ACTIVITY_DORMANT); + inner + .options + .as_ref() + .map_or(10_000.0, |o| o.hibernate_after.as_millis() as f64) + }; + self.store_status().await; + self.arm(after).await; + } + + async fn hibernate(&self) { + self.persist().await; + { + let mut inner = self.inner.borrow_mut(); + if inner.occupied() { + return; + } + inner.report(ACTIVITY_HIBERNATED); + // Dropped, so the next join restores from storage. + inner.host = None; + inner.state = RoomState::Cold; + inner.ticking = false; + } + self.store_status().await; + } + + /// Write the facts emitted since the last call to the outbox. + async fn spool(&self) { + let facts = self.inner.borrow_mut().emitted(); + if facts.is_empty() { + return; + } + let storage = self.state.storage(); + // One batched write (the storage API takes at most 128 keys). + for chunk in facts.chunks(128) { + let batch = js_sys::Object::new(); + for fact in chunk { + let text = serde_json::to_string(&StoredFact::of(fact)).unwrap_or_default(); + let _ = js_sys::Reflect::set( + &batch, + &StoredFact::storage_key(fact.key()).into(), + &text.into(), + ); + } + if let Err(e) = storage.put_multiple_raw(batch).await { + worker::console_error!("fact outbox write failed: {e}"); + } + } + self.inner.borrow_mut().spooled += facts.len() as u64; + self.arm_outbox().await; + } + + /// [`spool`](Self::spool) from a synchronous handler. + fn spool_later(&self) { + let this = self.handle(); + wasm_bindgen_futures::spawn_local(async move { this.spool().await }); + } + + async fn store_status(&self) { + self.spool().await; + let status = self.inner.borrow().last_status.clone(); + if !status.is_empty() { + let _ = self.state.storage().put(STATUS_KEY, status).await; + } + } + + /// Set the object's alarm `in_ms` from now. + async fn arm(&self, in_ms: f64) { + if let Err(e) = self + .state + .storage() + .set_alarm(Duration::from_millis(in_ms.max(1.0) as u64)) + .await + { + worker::console_error!("zone alarm: {e}"); + } + } + + /// Keep an alarm while unsettled facts wait in the outbox. + async fn arm_outbox(&self) { + let pending = { + let inner = self.inner.borrow(); + inner.activity.as_ref().map_or(0, |q| q.in_flight()) + + inner.host.as_ref().map_or(0, |h| h.facts_in_flight()) + }; + let alarm = self.state.storage().get_alarm().await.ok().flatten(); + if pending > 0 && alarm.is_none() { + self.arm(OUTBOX_RETRY_MS).await; + } + } + + /// Debug only, off by default: ask each configured operator listener + /// whether it holds this object. + fn find_node(&self) { + let (nodes, id) = { + let inner = self.inner.borrow(); + let Some(options) = inner.options.as_ref() else { + return; + }; + if options.nodes.is_empty() || options.node.is_some() { + return; + } + (options.nodes.clone(), self.state.id().to_string()) + }; + let inner = self.inner.clone(); + wasm_bindgen_futures::spawn_local(async move { + for (node, addr) in nodes { + let Ok(url) = format!("http://{addr}/state").parse() else { + continue; + }; + let Ok(mut response) = worker::Fetch::Url(url).send().await else { + continue; + }; + let Ok(state) = response.json::().await else { + continue; + }; + let mine = state["deployment"]["cells"] + .as_object() + .is_some_and(|cells| cells.keys().any(|k| k.ends_with(&id))); + if mine { + let mut inner = inner.borrow_mut(); + inner.node = node; + if inner.state == RoomState::Active { + inner.report(ACTIVITY_ACTIVE); + } + return; + } + } + }); + } + + fn info(&self) -> Value { + let inner = self.inner.borrow(); + let host = inner.host.as_ref(); + let stats = &inner.stats; + json!({ + "zone": inner.zone_id, + "status": inner.last_status, + "awake": host.is_some(), + "activation": inner.activation, + "restored": inner.restored, + "wakes": inner.wakes, + "node": inner.node, + "isolate": ISOLATE.with(|i| i.clone()), + "players": host.map_or(0, |h| h.player_count()), + "tick": host.map_or(0, |h| h.tick()), + "hz": inner.options.as_ref().map_or(0, |o| o.tick_hz), + "ticking": inner.ticking, + "factsInFlight": host.map_or(0, |h| h.facts_in_flight()) + + inner.activity.as_ref().map_or(0, |q| q.in_flight()), + "spooledFacts": inner.spooled, + "iterations": stats.iterations, + "droppedTicks": stats.dropped, + "lateMs": { "p50": stats.late.percentile(0.5), "p99": stats.late.percentile(0.99) }, + "workMs": { "p50": stats.work.percentile(0.5), "p99": stats.work.percentile(0.99) }, + }) + } +} + +fn retry_policy() -> RetryPolicy { + RetryPolicy::until_settled(Duration::from_millis(100), Duration::from_secs(5)) +} + +fn sleep(d: Duration) -> Delay { + Delay::from(d) +} + +fn reject(ws: &WebSocket, code: u32, message: &str) { + if let Ok(frame) = (ServerMessage::Error { + code, + message: message.into(), + }) + .encode(&()) + { + let _ = ws.send_with_bytes(frame); + } + let _ = ws.close(Some(1008), Some(message)); +} + +/// Worker entry helper: route `"{prefix}{zone}[/…]"` to the zone's object +/// in Durable Object namespace `binding` (one object per zone id; celld +/// places it). Returns `None` when the path is not under `prefix`. +pub async fn forward_to_zone( + req: Request, + env: &Env, + binding: &str, + prefix: &str, +) -> Result> { + let path = req.url()?.path().to_string(); + let Some((zone, _)) = zone_from_path(prefix, &path) else { + return Ok(None); + }; + let stub = env + .durable_object(binding)? + .id_from_name(&zone)? + .get_stub()?; + let headers = req.headers().clone(); + headers.set(ZONE_HEADER, &zone)?; + let mut init = RequestInit::new(); + init.with_method(req.method()).with_headers(headers); + let forwarded = Request::new_with_init(req.url()?.as_str(), &init)?; + stub.fetch_with_request(forwarded).await.map(Some) +} + +/// Declare the Durable Object class hosting zone type `$zone`: +/// +/// ```ignore +/// distributed_game::celld_zone_object!(PitRoom, MobPit); +/// ``` +/// +/// The crate needs `worker` as a dependency. Bind the class in +/// `wrangler.jsonc` (a SQLite class) and route to it with +/// [`forward_to_zone`]. +#[macro_export] +macro_rules! celld_zone_object { + ($name:ident, $zone:ty) => { + #[::worker::durable_object] + pub struct $name { + zone: $crate::celld_zone::ZoneObject<$zone>, + } + + impl ::worker::DurableObject for $name { + fn new(state: ::worker::State, env: ::worker::Env) -> Self { + Self { + zone: $crate::celld_zone::ZoneObject::new(state, env), + } + } + + async fn fetch(&self, req: ::worker::Request) -> ::worker::Result<::worker::Response> { + self.zone.fetch(req).await + } + + async fn websocket_message( + &self, + ws: ::worker::WebSocket, + message: ::worker::WebSocketIncomingMessage, + ) -> ::worker::Result<()> { + self.zone.websocket_message(ws, message).await + } + + async fn websocket_close( + &self, + ws: ::worker::WebSocket, + code: usize, + reason: String, + was_clean: bool, + ) -> ::worker::Result<()> { + self.zone.websocket_close(ws, code, reason, was_clean).await + } + + async fn websocket_error( + &self, + ws: ::worker::WebSocket, + error: ::worker::Error, + ) -> ::worker::Result<()> { + self.zone.websocket_error(ws, error).await + } + + async fn alarm(&self) -> ::worker::Result<::worker::Response> { + self.zone.alarm().await + } + } + }; +} diff --git a/distributed_game/src/celld_zone/plan.rs b/distributed_game/src/celld_zone/plan.rs new file mode 100644 index 000000000..c9e4ab28e --- /dev/null +++ b/distributed_game/src/celld_zone/plan.rs @@ -0,0 +1,473 @@ +//! The runtime-free parts of the celld zone host: options, the room's +//! lifecycle states, the self-correcting tick schedule, the durable fact +//! outbox encoding and the cell wait-path answer classification. Compiled +//! on every target so they are unit-tested natively; the Durable Object glue +//! ([`super::ZoneObject`], wasm32) drives them. + +use std::collections::HashMap; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::facts::{Fact, FactError, FactReceipt}; +use crate::stamp::FactStamp; + +/// Env var: HS256 secret admission (and transfer) tickets are signed with. +pub const TICKET_SECRET_VAR: &str = "GAME_TICKET_SECRET"; +/// Env var: tick rate (Hz, default 30). +pub const TICK_HZ_VAR: &str = "ZONE_TICK_HZ"; +/// Env var: patch rate (Hz, default 20, at most the tick rate). +pub const PATCH_HZ_VAR: &str = "ZONE_PATCH_HZ"; +/// Env var: snapshot interval while occupied (ms, default 5000). +pub const PERSIST_MS_VAR: &str = "ZONE_PERSIST_MS"; +/// Env var: how long an empty room stays dormant before it hibernates (ms, +/// default 10000). +pub const HIBERNATE_AFTER_MS_VAR: &str = "ZONE_HIBERNATE_AFTER_MS"; +/// Env var: reconnect grace (ms, default 5000). +pub const RECONNECT_GRACE_MS_VAR: &str = "ZONE_RECONNECT_GRACE_MS"; +/// Env var: command-ledger service id of the zone's facts (default +/// `celld-zone`). +pub const FACT_SERVICE_VAR: &str = "ZONE_FACT_SERVICE"; +/// Env var (debug only, off by default): `name=host:port,…` celld operator +/// listeners asked which node holds the room (celld does not tell +/// application code its node id). +pub const NODES_VAR: &str = "CELLD_ZONE_NODES"; +/// Env var (debug only): a fixed node name (e.g. `dev` for `celld dev`). +pub const NODE_VAR: &str = "CELLD_ZONE_NODE"; + +/// Request header naming the zone (set by [`super::forward_to_zone`]; the +/// object's id has no name on some celld paths). +pub const ZONE_HEADER: &str = "x-celld-zone"; + +/// Close code for sockets of an earlier activation (the room restarted or +/// moved): the client rejoins with a fresh ticket. +pub const CLOSE_RESTARTED: u16 = 1012; + +/// Object settings, from the worker's vars and secrets. +#[derive(Debug, Clone, PartialEq)] +pub struct ObjectOptions { + /// Ticket key secret. + pub ticket_secret: String, + /// Internal secret for cell calls (`DISTRIBUTED_INTERNAL_SECRET`). + pub internal_secret: Option, + /// Hot tick rate. + pub tick_hz: u32, + /// Patch rate. + pub patch_hz: u32, + /// Snapshot interval while occupied. + pub persist_every: Duration, + /// Dormant → hibernated. + pub hibernate_after: Duration, + /// Reconnect grace window. + pub reconnect_grace: Duration, + /// Fact command-ledger service id. + pub fact_service: String, + /// Debug node lookup: `(name, operator host:port)`. + pub nodes: Vec<(String, String)>, + /// Debug fixed node name. + pub node: Option, +} + +fn parse( + get: &impl Fn(&str) -> Option, + name: &str, + default: T, +) -> Result { + match get(name).filter(|value| !value.trim().is_empty()) { + None => Ok(default), + Some(value) => value + .trim() + .parse() + .map_err(|_| format!("{name}: not a number: {value}")), + } +} + +impl ObjectOptions { + /// Read the options through `get` (vars, then secrets). + pub fn from_vars(get: impl Fn(&str) -> Option) -> Result { + let ticket_secret = get(TICKET_SECRET_VAR) + .filter(|secret| secret.len() >= 16) + .ok_or_else(|| format!("{TICKET_SECRET_VAR} (>= 16 bytes) is required"))?; + let tick_hz: u32 = parse(&get, TICK_HZ_VAR, 30)?; + if !(1..=120).contains(&tick_hz) { + return Err(format!("{TICK_HZ_VAR} must be within 1..=120")); + } + let patch_hz: u32 = parse(&get, PATCH_HZ_VAR, 20)?; + let nodes = get(NODES_VAR) + .unwrap_or_default() + .split(',') + .filter_map(|entry| { + let (name, addr) = entry.trim().split_once('=')?; + (!name.is_empty() && !addr.is_empty()).then(|| (name.into(), addr.into())) + }) + .collect(); + Ok(Self { + ticket_secret, + internal_secret: get(distributed::cell_host::CELL_INTERNAL_SECRET_ENV) + .filter(|secret| !secret.is_empty()), + tick_hz, + patch_hz: patch_hz.clamp(1, tick_hz), + persist_every: Duration::from_millis(parse(&get, PERSIST_MS_VAR, 5_000)?), + hibernate_after: Duration::from_millis(parse(&get, HIBERNATE_AFTER_MS_VAR, 10_000)?), + reconnect_grace: Duration::from_millis(parse(&get, RECONNECT_GRACE_MS_VAR, 5_000)?), + fact_service: get(FACT_SERVICE_VAR) + .filter(|service| !service.trim().is_empty()) + .unwrap_or_else(|| "celld-zone".into()), + nodes, + node: get(NODE_VAR).filter(|node| !node.is_empty()), + }) + } +} + +/// Where a room is in its life (per object activation). +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum RoomState { + /// No zone in memory (fresh activation, or hibernated and dropped). + Cold, + /// Zone in memory with players (or just woke up). + Active, + /// Empty since `since_ms` (Unix ms); snapshot written. + Dormant { + /// When the last player left. + since_ms: f64, + }, +} + +impl RoomState { + /// True when an empty room should hibernate at `now_ms`. + pub fn hibernate_due(&self, now_ms: f64, after: Duration) -> bool { + match self { + Self::Dormant { since_ms } => now_ms + 1.0 >= since_ms + after.as_millis() as f64, + _ => false, + } + } +} + +/// A self-correcting fixed-rate schedule over absolute deadlines +/// (`setInterval` drifts: 33.3 ms gave 35 ms on celld). Each wake-up aims at +/// `start + k × step`; deadlines already missed are skipped (the fixed +/// timestep catches up inside one `advance`). +#[derive(Debug, Clone)] +pub struct TickSchedule { + start_ms: f64, + step_ms: f64, + k: u64, +} + +impl TickSchedule { + /// A schedule at `hz` from `now_ms`. + pub fn new(now_ms: f64, hz: u32) -> Self { + Self { + start_ms: now_ms, + step_ms: 1000.0 / f64::from(hz.max(1)), + k: 1, + } + } + + /// The next deadline. + pub fn deadline(&self) -> f64 { + self.start_ms + self.k as f64 * self.step_ms + } + + /// How long to wait from `now_ms` (never negative). + pub fn wait_ms(&self, now_ms: f64) -> f64 { + (self.deadline() - now_ms).max(0.0) + } + + /// Record a wake-up at `now_ms`: returns how late it was, and moves to + /// the first deadline after `now_ms`. + pub fn woke(&mut self, now_ms: f64) -> f64 { + let late = (now_ms - self.deadline()).max(0.0); + let next = ((now_ms - self.start_ms) / self.step_ms).floor() as u64 + 1; + self.k = next.max(self.k + 1); + late + } +} + +/// A fact in the object's durable outbox (written before it is sent, +/// deleted once settled, resent by the next activation). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StoredFact { + /// Idempotency key. + pub key: String, + /// Command. + pub command: String, + /// Input. + pub input: Value, + /// Bound principal. + pub principal: Option, + /// Stamp as its `x-game-*` session variables. + pub stamp: HashMap, +} + +impl StoredFact { + /// Storage key of a fact. + pub fn storage_key(key: &str) -> String { + format!("{OUTBOX_PREFIX}{key}") + } + + /// Encode `fact`. + pub fn of(fact: &Fact) -> Self { + Self { + key: fact.key().to_string(), + command: fact.command().to_string(), + input: fact.input().clone(), + principal: fact.principal().map(str::to_string), + stamp: fact.stamp().session_vars().into_iter().collect(), + } + } + + /// Decode (None for a malformed stamp). + pub fn into_fact(self) -> Option { + let stamp = FactStamp::from_session_vars(&self.stamp)?; + Some(Fact::trusted( + self.key, + self.command, + self.input, + self.principal, + stamp, + )) + } +} + +/// Storage key prefix of the durable fact outbox. +pub const OUTBOX_PREFIX: &str = "fact:"; + +/// Durable Object binding of a cell kind: upper case, `-` → `_` +/// (`zone-activity` → `ZONE_ACTIVITY`, `crawler` → `CRAWLER`). +pub fn cell_binding(kind: &str) -> String { + kind.chars() + .map(|c| { + if c == '-' { + '_' + } else { + c.to_ascii_uppercase() + } + }) + .collect() +} + +/// Classify a cell wait-path answer (`status`, JSON `body`). +pub fn classify_wait_path(status: u16, body: &Value) -> Result { + if (200..300).contains(&status) { + return Ok(FactReceipt { + payload: body.get("payload").cloned().unwrap_or(Value::Null), + replayed: body + .pointer("/receipt/replayed") + .and_then(Value::as_bool) + .unwrap_or(false), + }); + } + let code = body.get("code").and_then(Value::as_str).unwrap_or(""); + let message = body + .get("error") + .and_then(Value::as_str) + .unwrap_or("rejected"); + let detail = format!("HTTP {status} {code}: {message}"); + if status >= 500 || matches!(status, 408 | 429) || code == "COMMAND_IN_PROGRESS" { + Err(FactError::Retryable(detail)) + } else { + Err(FactError::Permanent(detail)) + } +} + +/// `"{prefix}{zone}[/rest]"` → `(zone, rest)` (percent-decoded zone; None +/// when the path does not match or the zone is empty / not printable). +pub fn zone_from_path<'a>(prefix: &str, path: &'a str) -> Option<(String, &'a str)> { + let rest = path.strip_prefix(prefix)?; + let (raw, tail) = rest.split_once('/').unwrap_or((rest, "")); + let zone = percent_decode(raw)?; + (!zone.is_empty() + && zone.len() <= 128 + && zone.bytes().all(|byte| (0x21..=0x7e).contains(&byte))) + .then_some((zone, tail)) +} + +fn percent_decode(segment: &str) -> Option { + let bytes = segment.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + out.push(u8::from_str_radix(segment.get(i + 1..i + 3)?, 16).ok()?); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).ok() +} + +/// Percentiles of the last values (tick timing in the debug info). +#[derive(Debug, Clone, Default)] +pub struct Window { + values: std::collections::VecDeque, +} + +impl Window { + const MAX: usize = 900; + + /// Add a value. + pub fn push(&mut self, value: f64) { + if self.values.len() == Self::MAX { + self.values.pop_front(); + } + self.values.push_back(value); + } + + /// The `p` percentile (0 when empty). + pub fn percentile(&self, p: f64) -> f64 { + if self.values.is_empty() { + return 0.0; + } + let mut sorted: Vec = self.values.iter().copied().collect(); + sorted.sort_by(f64::total_cmp); + sorted[((sorted.len() - 1) as f64 * p.clamp(0.0, 1.0)).round() as usize] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn vars(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let map: HashMap = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + move |name| map.get(name).cloned() + } + + #[test] + fn options_need_a_ticket_secret_and_default_the_rest() { + assert!(ObjectOptions::from_vars(vars(&[])).is_err()); + assert!(ObjectOptions::from_vars(vars(&[(TICKET_SECRET_VAR, "short")])).is_err()); + let options = + ObjectOptions::from_vars(vars(&[(TICKET_SECRET_VAR, "0123456789abcdef")])).unwrap(); + assert_eq!((options.tick_hz, options.patch_hz), (30, 20)); + assert_eq!(options.hibernate_after, Duration::from_secs(10)); + assert!(options.nodes.is_empty(), "node lookup is off by default"); + assert_eq!(options.fact_service, "celld-zone"); + let options = ObjectOptions::from_vars(vars(&[ + (TICKET_SECRET_VAR, "0123456789abcdef"), + (TICK_HZ_VAR, "10"), + (PATCH_HZ_VAR, "20"), + (NODES_VAR, "node-a=127.0.0.1:1, bad ,node-b=127.0.0.1:2"), + ("DISTRIBUTED_INTERNAL_SECRET", "s"), + ])) + .unwrap(); + assert_eq!(options.patch_hz, 10, "patches never outpace ticks"); + assert_eq!(options.nodes.len(), 2); + assert_eq!(options.internal_secret.as_deref(), Some("s")); + assert!(ObjectOptions::from_vars(vars(&[ + (TICKET_SECRET_VAR, "0123456789abcdef"), + (TICK_HZ_VAR, "0"), + ])) + .is_err()); + } + + #[test] + fn the_tick_schedule_corrects_itself_and_skips_missed_deadlines() { + let mut schedule = TickSchedule::new(1000.0, 20); + assert_eq!(schedule.deadline(), 1050.0); + assert_eq!(schedule.wait_ms(1010.0), 40.0); + // Woke 3 ms late: the next deadline stays on the grid (no drift). + assert_eq!(schedule.woke(1053.0), 3.0); + assert_eq!(schedule.deadline(), 1100.0); + // A 130 ms stall skips the missed deadlines. + assert_eq!(schedule.woke(1230.0), 130.0); + assert_eq!(schedule.deadline(), 1250.0); + assert_eq!(schedule.wait_ms(1300.0), 0.0); + // Early wake-ups still advance by one step. + let before = schedule.deadline(); + schedule.woke(before - 5.0); + assert_eq!(schedule.deadline(), before + 50.0); + } + + #[test] + fn dormant_rooms_hibernate_after_the_delay() { + let after = Duration::from_millis(500); + assert!(!RoomState::Active.hibernate_due(10_000.0, after)); + assert!(!RoomState::Cold.hibernate_due(10_000.0, after)); + let dormant = RoomState::Dormant { since_ms: 1_000.0 }; + assert!(!dormant.hibernate_due(1_400.0, after)); + assert!(dormant.hibernate_due(1_500.0, after)); + } + + #[test] + fn stored_facts_round_trip() { + let fact = Fact::trusted( + "kill:pit:1:m1:0", + "crawler.record_kill", + json!({ "mob": 1 }), + Some("alice".into()), + FactStamp { + zone_id: "pit.0.0".into(), + tick: 7, + activation: 9, + seq: 2, + emitted_at_ms: 1_700_000_000_000, + ..FactStamp::default() + }, + ); + let stored = StoredFact::of(&fact); + let text = serde_json::to_string(&stored).unwrap(); + let back: StoredFact = serde_json::from_str(&text).unwrap(); + assert_eq!(back.into_fact().unwrap(), fact); + assert_eq!(StoredFact::storage_key("k"), format!("{OUTBOX_PREFIX}k")); + } + + #[test] + fn wait_path_answers_are_classified() { + let ok = classify_wait_path( + 200, + &json!({ "payload": { "xp": 3 }, "receipt": { "replayed": true } }), + ) + .unwrap(); + assert_eq!(ok.payload, json!({ "xp": 3 })); + assert!(ok.replayed); + let retry = |status, code: &str| { + matches!( + classify_wait_path(status, &json!({ "code": code })), + Err(FactError::Retryable(_)) + ) + }; + assert!(retry(503, "INTERNAL")); + assert!(retry(409, "COMMAND_IN_PROGRESS")); + assert!(retry(429, "")); + assert!(!retry(401, "UNAUTHORIZED")); + assert!(!retry(422, "REJECTED")); + } + + #[test] + fn bindings_and_paths() { + assert_eq!(cell_binding("zone-activity"), "ZONE_ACTIVITY"); + assert_eq!(cell_binding("crawler"), "CRAWLER"); + assert_eq!( + zone_from_path("/rooms/", "/rooms/pit.1.-2/info"), + Some(("pit.1.-2".to_string(), "info")) + ); + assert_eq!( + zone_from_path("/rooms/", "/rooms/a%20b"), + None, + "zones are printable ASCII without spaces" + ); + assert_eq!(zone_from_path("/rooms/", "/rooms/"), None); + assert_eq!(zone_from_path("/rooms/", "/other/x"), None); + } + + #[test] + fn windows_report_percentiles() { + let mut window = Window::default(); + assert_eq!(window.percentile(0.5), 0.0); + for v in 1..=100 { + window.push(f64::from(v)); + } + assert_eq!(window.percentile(0.0), 1.0); + assert_eq!(window.percentile(1.0), 100.0); + assert!((window.percentile(0.99) - 99.0).abs() <= 1.0); + } +} diff --git a/distributed_game/src/command_sink.rs b/distributed_game/src/command_sink.rs index da08e751e..b5acc84b1 100644 --- a/distributed_game/src/command_sink.rs +++ b/distributed_game/src/command_sink.rs @@ -1,26 +1,27 @@ -//! [`CommandFactSink`]: a zone server's facts dispatched as commands through -//! any [`CommandHost`] — the same durable aggregates whether they run in -//! celld cells ([`CelldCommandHost`], `POST {celld}/{kind}/{shard}/{command}` -//! over the internal path) or in an ordinary service over any repository -//! ([`LocalCommandHost`]: in-memory, SQLite, Postgres). +//! [`CommandFactSink`]: a zone's facts dispatched as commands through any +//! [`FactHost`] — the same durable aggregates whether they are reached over +//! celld's internal HTTP path ([`CelldCommandHost`], feature `command-host`, +//! `POST {celld}/{kind}/{shard}/{command}`), object-to-object from a zone +//! running as a celld Durable Object +//! ([`CelldObjectHost`](crate::celld_zone::CelldObjectHost), feature +//! `celld-zone`, wasm32), or in an ordinary service over any repository +//! ([`LocalCommandHost`], feature `command-host`). //! -//! One sink per zone server routes every fact by its command +//! One sink per zone (host) routes every fact by its command //! ([`FactCommand::COMMAND`](crate::FactCommand::COMMAND)) to a registered //! [`FactRoute`]: a cell kind and a shard function of the fact's input and //! trusted session (principal + [`FactStamp`](crate::FactStamp)). A host -//! that addresses cells uses both ([`FactRouting`]); a service routes by -//! command and its handler picks the stream from the same session or input. -//! Player lifecycle facts (`zone.*`, [`crate::lifecycle`]) route to the -//! presence aggregate of the player's account ([`CommandFactSink::lifecycle_route`]). +//! that addresses cells uses both; a service routes by command and its +//! handler picks the stream from the same session or input. Player +//! lifecycle facts (`zone.*`, [`crate::lifecycle`]) route to the presence +//! aggregate of the player's account ([`CommandFactSink::lifecycle_route`]). //! //! - **Exactly one commit per key.** The command id is //! [`command_id_for`]`(service_id, fact key)`, so a retry after a timeout //! replays through the aggregate's command ledger (`replayed` receipt). //! - **Trusted identity.** The principal (bound by the zone host from the -//! verified ticket) is the session's `x-user-id` and the verified -//! principal ([`VerifiedPrincipal::from_trusted_transport`]); the stamp -//! travels as `x-game-*` session variables (forwarded to cells as trusted -//! metadata, [`CelldCommandHost::forward_session_prefix`]). +//! verified ticket) is the session's `x-user-id`; the stamp travels as +//! `x-game-*` session variables (forwarded to cells as trusted metadata). //! - **No local durable fallback.** A host that is slow or unreachable, a //! 5xx, or a command still in progress is [`FactError::Retryable`]: the fact //! stays in the zone's queue (use @@ -28,17 +29,24 @@ //! unrouted or rejected command is [`FactError::Permanent`]. use std::sync::Arc; +#[cfg(feature = "command-host")] use std::time::Duration; +#[cfg(feature = "command-host")] use distributed::cell_host::{CelldCommandHost, InternalHttpSecret}; +#[cfg(feature = "command-host")] use distributed::command_dispatch::{CommandHost, LocalCommandHost}; +#[cfg(feature = "command-host")] use distributed::graphql::identity::VerifiedPrincipal; -use distributed::microsvc::{CausalDispatchError, Session}; +#[cfg(feature = "command-host")] +use distributed::microsvc::CausalDispatchError; +use distributed::microsvc::Session; use serde_json::Value; use crate::cell::{command_id_for, session_stamp, trusted_session}; use crate::facts::{BoxFuture, Fact, FactCommand, FactError, FactReceipt, FactSink}; use crate::lifecycle::LIFECYCLE_COMMANDS; +#[cfg(feature = "command-host")] use crate::stamp::SESSION_PREFIX; type ShardFn = Arc Option + Send + Sync>; @@ -109,26 +117,41 @@ impl FactRoute { } } -/// A [`CommandHost`] that learns the fact routes it needs. -pub trait FactRouting { - /// Register `route` (a host that addresses aggregates by kind and shard - /// records it; one that routes by command ignores it). - fn register_fact_route(&mut self, route: &FactRoute); +/// One routed fact, ready for a [`FactHost`]. +#[derive(Debug, Clone)] +pub struct FactCall<'a> { + /// Durable command name. + pub command: &'a str, + /// Aggregate kind of the route. + pub kind: &'a str, + /// Validated shard (non-empty, printable ASCII, ≤ 256 bytes). + pub shard: String, + /// Deterministic command id ([`command_id_for`]). + pub command_id: String, + /// Command input. + pub input: &'a Value, + /// Verified principal of the fact (or the sink's system principal). + pub principal: String, + /// Trusted session: `x-user-id` + the `x-game-*` stamp. + pub session: Session, } -/// A service routes by command: its handler selects the stream. -impl FactRouting for LocalCommandHost { +/// What a [`CommandFactSink`] dispatches through: anything that runs one +/// durable command idempotently by command id. Implemented for +/// [`CelldCommandHost`] and [`LocalCommandHost`] (feature `command-host`) +/// and for the celld object host (feature `celld-zone`). +pub trait FactHost: Send + Sync + 'static { + /// Learn `route` (a host that addresses aggregates by kind and shard + /// records it; one that routes by command ignores it). fn register_fact_route(&mut self, _route: &FactRoute) {} -} -/// celld addresses cells: `{kind}/{shard}/{command}`. -impl FactRouting for CelldCommandHost { - fn register_fact_route(&mut self, route: &FactRoute) { - let shard = route.shard.clone(); - self.add_cell_route(route.command(), route.kind(), move |input, session| { - shard(input, session) - }); - } + /// Run `call` once per command id. Classify failures: unreachable, + /// 5xx, in progress → [`FactError::Retryable`]; anything else → + /// [`FactError::Permanent`]. + fn dispatch_fact<'a>( + &'a self, + call: FactCall<'a>, + ) -> BoxFuture<'a, Result>; } /// See the module docs. @@ -150,7 +173,7 @@ impl Clone for CommandFactSink { } } -impl CommandFactSink { +impl CommandFactSink { /// A sink dispatching through `host` under command-ledger `service_id`; /// zone facts without a player use `system_principal`. pub fn new( @@ -202,6 +225,11 @@ impl CommandFactSink { &self.host } + /// The command-ledger service id. + pub fn service_id(&self) -> &str { + &self.service_id + } + /// The route a fact would take, if any. pub fn route_for(&self, command: &str) -> Option<&FactRoute> { self.routes.iter().find(|route| route.command == command) @@ -209,6 +237,15 @@ impl CommandFactSink { /// `(principal, trusted session, command id)` of a routed fact. pub fn prepare(&self, fact: &Fact) -> Result<(String, Session, String), FactError> { + let (_, _, principal, session, command_id) = self.resolve(fact)?; + Ok((principal, session, command_id)) + } + + #[allow(clippy::type_complexity)] + fn resolve( + &self, + fact: &Fact, + ) -> Result<(&FactRoute, String, String, Session, String), FactError> { let route = self.route_for(fact.command()).ok_or_else(|| { FactError::Permanent(format!("no route for fact command `{}`", fact.command())) })?; @@ -217,7 +254,7 @@ impl CommandFactSink { .unwrap_or(&self.system_principal) .to_string(); let session = trusted_session(&principal, fact); - route + let shard = route .shard(fact.input(), &session) .filter(|shard| { !shard.is_empty() @@ -232,6 +269,8 @@ impl CommandFactSink { )) })?; Ok(( + route, + shard, principal, session, command_id_for(&self.service_id, fact.key()), @@ -239,7 +278,27 @@ impl CommandFactSink { } } +impl FactSink for CommandFactSink { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (route, shard, principal, session, command_id) = self.resolve(fact)?; + self.host + .dispatch_fact(FactCall { + command: fact.command(), + kind: route.kind(), + shard, + command_id, + input: fact.input(), + principal, + session, + }) + .await + }) + } +} + /// celld slow, down or still running the command: keep the fact queued. +#[cfg(feature = "command-host")] fn classify(error: CausalDispatchError) -> FactError { let detail = format!("{}: {error:?}", error.code()); match &error { @@ -258,36 +317,69 @@ fn classify(error: CausalDispatchError) -> FactError { } } -impl FactSink for CommandFactSink { - fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { - Box::pin(async move { - let (principal, session, command_id) = self.prepare(fact)?; - let result = self - .host - .invoke( - fact.command(), - &command_id, - fact.input().clone(), - session, - VerifiedPrincipal::from_trusted_transport(&principal), - None, - ) - .await - .map_err(classify)?; - Ok(FactReceipt { - payload: result.payload().clone(), - replayed: result.replayed(), - }) +/// Any [`CommandHost`]: `invoke` with the fact's trusted session. +#[cfg(feature = "command-host")] +fn invoke_command_host<'a, H: CommandHost>( + host: &'a H, + call: FactCall<'a>, +) -> BoxFuture<'a, Result> { + Box::pin(async move { + let result = host + .invoke( + call.command, + &call.command_id, + call.input.clone(), + call.session, + VerifiedPrincipal::from_trusted_transport(&call.principal), + None, + ) + .await + .map_err(classify)?; + Ok(FactReceipt { + payload: result.payload().clone(), + replayed: result.replayed(), }) + }) +} + +/// A service routes by command: its handler selects the stream. +#[cfg(feature = "command-host")] +impl FactHost for LocalCommandHost { + fn dispatch_fact<'a>( + &'a self, + call: FactCall<'a>, + ) -> BoxFuture<'a, Result> { + invoke_command_host(self, call) + } +} + +/// celld addresses cells: `{kind}/{shard}/{command}`. +#[cfg(feature = "command-host")] +impl FactHost for CelldCommandHost { + fn register_fact_route(&mut self, route: &FactRoute) { + let shard = route.shard.clone(); + self.add_cell_route(route.command(), route.kind(), move |input, session| { + shard(input, session) + }); + } + + fn dispatch_fact<'a>( + &'a self, + call: FactCall<'a>, + ) -> BoxFuture<'a, Result> { + invoke_command_host(self, call) } } /// Default per-request timeout of [`CelldFactSink`]. +#[cfg(feature = "command-host")] pub const DEFAULT_CELLD_FACT_TIMEOUT: Duration = Duration::from_secs(10); /// Facts into celld cells over the internal path. +#[cfg(feature = "command-host")] pub type CelldFactSink = CommandFactSink; +#[cfg(feature = "command-host")] impl CommandFactSink { /// A sink for the cells at `celld_url` under command-ledger /// `service_id`: a remote-only [`CelldCommandHost`] forwarding the fact @@ -324,7 +416,7 @@ impl CommandFactSink { } } -#[cfg(test)] +#[cfg(all(test, feature = "command-host"))] mod tests { use super::*; use crate::stamp::FactStamp; diff --git a/distributed_game/src/facts.rs b/distributed_game/src/facts.rs index baf00eb59..66e784662 100644 --- a/distributed_game/src/facts.rs +++ b/distributed_game/src/facts.rs @@ -320,6 +320,7 @@ pub struct FactQueue { in_flight: HashSet, capacity: usize, script: Option, + journal: Option>, } /// Worker-side handle consumed by [`FactFeed::run`]. @@ -342,6 +343,7 @@ pub fn fact_channel(capacity: usize, policy: RetryPolicy) -> (FactQueue, FactFee in_flight: HashSet::new(), capacity: capacity.max(1), script: None, + journal: None, }, FactFeed { shared, @@ -362,6 +364,9 @@ impl FactQueue { return Err(EmitError::Backpressure); } self.in_flight.insert(fact.key.clone()); + if let Some(journal) = self.journal.as_mut() { + journal.push(fact.clone()); + } if self.script.is_some() { return Ok(()); } @@ -374,6 +379,22 @@ impl FactQueue { Ok(()) } + /// Keep a copy of every accepted fact until [`take_journal`](Self::take_journal): + /// a host that keeps its own durable outbox writes them there right + /// after the tick that emitted them (the celld zone host does). + pub fn with_journal(mut self) -> Self { + self.journal = Some(Vec::new()); + self + } + + /// Facts accepted since the last call (empty without a journal). + pub fn take_journal(&mut self) -> Vec { + self.journal + .as_mut() + .map(std::mem::take) + .unwrap_or_default() + } + /// Settlements that arrived since the last call. pub fn drain(&mut self) -> Vec { let settled: Vec<_> = self.settlements.try_iter().collect(); diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index a2cb7b046..9d798bae6 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -1014,6 +1014,16 @@ impl ZoneHost { self.core.in_flight() } + /// Facts emitted since the last call, when the fact queue keeps a + /// journal ([`FactQueue::with_journal`]); empty otherwise. + pub fn take_emitted_facts(&mut self) -> Vec { + self.core + .facts + .as_mut() + .map(FactQueue::take_journal) + .unwrap_or_default() + } + /// True after [`dispose`](Self::dispose). pub fn is_disposed(&self) -> bool { self.disposed @@ -1847,7 +1857,12 @@ impl ZoneHost { } fn tick_once(&mut self, dt: Duration) { + // wasm32-unknown-unknown (celld) has no `Instant`: measure with the + // host clock there (the celld zone host). + #[cfg(not(target_arch = "wasm32"))] let started = std::time::Instant::now(); + #[cfg(target_arch = "wasm32")] + let started = self.core.clock.now(); self.core.tick += 1; self.core.sim_time += dt; self.core.dt = dt; @@ -1867,7 +1882,12 @@ impl ZoneHost { .rewind .record(self.core.tick, self.core.sim_time, entities); } + #[cfg(not(target_arch = "wasm32"))] self.pending.tick_durations.push(started.elapsed()); + #[cfg(target_arch = "wasm32")] + self.pending + .tick_durations + .push(self.core.clock.now().saturating_sub(started)); if self.hash_ticks { let hash = self.state_hash(); self.pending.tick_hashes.push((self.core.tick, hash)); diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index 721d48d87..03fe2fdb8 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -13,6 +13,9 @@ //! priority accumulation. Wire format: [`protocol`] / `PROTOCOL.md`. //! - Transport (`net` feature): axum WebSocket server hosting many zones //! with HMAC-signed ticket admission ([`net`]). +//! - celld (`celld-zone` feature, wasm32): each zone is a celld Durable +//! Object; celld places, forwards, hibernates and recovers it +//! ([`celld_zone`]). //! //! - Authoring derives (re-exported from `distributed_game_macros`): //! `#[derive(Replicated)]` views generate a kind's schema and field @@ -56,10 +59,12 @@ pub mod bots; pub mod cell; #[cfg(feature = "celld")] pub mod celld; +#[cfg(feature = "celld-zone")] +pub mod celld_zone; pub mod claims; pub mod clock; pub mod codec; -#[cfg(feature = "command-host")] +#[cfg(feature = "cell")] pub mod command_sink; pub mod entity; pub mod facts; @@ -83,6 +88,11 @@ pub mod rng; pub mod schema; pub mod stamp; pub mod surface; +/// Signed admission tickets (also `net::ticket`); usable without the axum +/// server, e.g. by a celld zone host on wasm32. +#[cfg(feature = "ticket")] +#[path = "net/ticket.rs"] +pub mod ticket; pub mod timers; pub mod timestep; pub mod world; diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index dbc7e22fb..cac7a3174 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -45,7 +45,7 @@ pub mod inbox; pub mod peer; -pub mod ticket; +pub use crate::ticket; pub mod transfer; pub mod world; diff --git a/distributed_game/src/world/mod.rs b/distributed_game/src/world/mod.rs index 2183010e6..b07d536fa 100644 --- a/distributed_game/src/world/mod.rs +++ b/distributed_game/src/world/mod.rs @@ -14,10 +14,12 @@ //! - [`allocation`]: [`ZoneAllocation`] records written by the service that //! allocates a zone, read before the zone is created //! ([`ZoneActivation`], [`ZoneAllocationStore`]). -//! - `zone_status` (feature `cell`): the read model of every zone's lease -//! and hibernation slot, projected from the `zone_lease` / `zone_state` -//! cells' outward events (active / hibernated / cold without opening a -//! cell). +//! - `zone_activity` (feature `cell`): a celld-hosted zone's own lifecycle +//! reports (active / dormant / hibernated) as a cell. +//! - `zone_status` (feature `cell`): the read model of every zone's lease, +//! hibernation slot and activity, projected from the `zone_lease` / +//! `zone_state` / `zone_activity` cells' outward events (active / dormant +//! / hibernated / cold without opening a cell). //! - [`barrier`]: content-version policy types shared by the host and the //! world barrier ([`StalePolicy`], [`BarrierMode`]). @@ -28,6 +30,9 @@ pub mod ownership; pub mod state; pub mod topology; #[cfg(feature = "cell")] +#[allow(missing_docs)] +pub mod zone_activity; +#[cfg(feature = "cell")] // The `sourced` and `Snapshot` derives generate undocumented public items. #[allow(missing_docs)] pub mod zone_lease; diff --git a/distributed_game/src/world/mutations/save_zone_activity.mutation.graphql b/distributed_game/src/world/mutations/save_zone_activity.mutation.graphql new file mode 100644 index 000000000..368e212f4 --- /dev/null +++ b/distributed_game/src/world/mutations/save_zone_activity.mutation.graphql @@ -0,0 +1,5 @@ +# Syntax-only read-model mutation → MutationProgram IR (not a public field). +# The row is the zone's post-transition `ZoneActivityStatus`. +mutation SaveZoneActivity { + upsert_zone_activities(object: $input.activity) +} diff --git a/distributed_game/src/world/zone_activity.rs b/distributed_game/src/world/zone_activity.rs new file mode 100644 index 000000000..d9fbc31a3 --- /dev/null +++ b/distributed_game/src/world/zone_activity.rs @@ -0,0 +1,288 @@ +//! What a zone running as a celld object is doing, as a cell: +//! `ZoneActivity:{zone_id}` (cell kind [`ZONE_ACTIVITY_KIND`]). +//! +//! A zone hosted by [`celld_zone`](crate::celld_zone) reports its own +//! lifecycle — woke up (`active`, `restored` when it came back from a +//! snapshot), emptied (`dormant`), slept (`hibernated`) — as facts +//! ([`RecordActivity`], command [`RECORD`]) through its fact sink, so the +//! transitions are durable and exactly once like any other zone fact. +//! +//! | Command | Effect | Outward event | +//! |---|---|---| +//! | `zone_activity.record` | the reported status, if emitted after the one stored | `zone.activity_changed` | +//! +//! Facts can settle out of order (retries, concurrent commits): a report +//! emitted before the stored one ([`FactStamp::emission_order`]) is +//! ignored. The outward event carries the post-transition +//! [`ZoneActivityStatus`]; the [`zone_status`](super::zone_status) read model +//! projects it into `zone_activity` rows (minimaps, dashboards). +//! +//! Only the zone itself may report: the fact's principal must be the +//! zone's system principal [`zone_principal`]`(zone)`, and the stamp's zone +//! is the shard. +//! +//! `node` and `isolate` are debug details (which celld node and V8 isolate +//! ran the activation, when known); nothing may route by them. + +use distributed::cell_host::AggregateCell; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, +}; +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; + +use crate::cell::session_stamp; +use crate::FactStamp; + +/// Aggregate type (cell `ZoneActivity:{zone_id}`). +pub const ZONE_ACTIVITY: &str = "ZoneActivity"; +/// Cell kind (URL segment / Durable Object routing) of zone-activity cells. +pub const ZONE_ACTIVITY_KIND: &str = "zone-activity"; +/// The one command (a zone fact). +pub const RECORD: &str = "zone_activity.record"; + +/// Status: ticking with players (or just woke up). +pub const ACTIVITY_ACTIVE: &str = "active"; +/// Status: empty, still in memory, snapshot written. +pub const ACTIVITY_DORMANT: &str = "dormant"; +/// Status: asleep; the next join restores the snapshot. +pub const ACTIVITY_HIBERNATED: &str = "hibernated"; + +/// The system principal a zone reports its own activity under. +pub fn zone_principal(zone_id: &str) -> String { + format!("zone:{zone_id}") +} + +/// One zone's activity. +#[derive(Clone, Debug, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct ZoneActivity { + entity: Entity, + status: String, + activation: String, + restored: bool, + players: u32, + node: String, + isolate: String, + activations: u64, + changed_at_ms: u64, + /// Ordering key of the report that set the current status. + last: Option, +} + +/// The emission-order key of the last applied report. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StampKey { + /// Zone. + pub zone_id: String, + /// Activation epoch. + pub activation: u64, + /// Tick. + pub tick: u64, + /// Emission wall time. + pub emitted_at_ms: u64, + /// Emission sequence. + pub seq: u64, +} + +impl StampKey { + fn of(stamp: &FactStamp) -> Self { + Self { + zone_id: stamp.zone_id.clone(), + activation: stamp.activation, + tick: stamp.tick, + emitted_at_ms: stamp.emitted_at_ms, + seq: stamp.seq, + } + } + + fn stamp(&self) -> FactStamp { + FactStamp { + zone_id: self.zone_id.clone(), + activation: self.activation, + tick: self.tick, + emitted_at_ms: self.emitted_at_ms, + seq: self.seq, + ..FactStamp::default() + } + } +} + +/// The public state carried by `zone.activity_changed`: one +/// `zone_activity` read-model row. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, distributed::DomainState)] +#[domain_state(version = 1)] +pub struct ZoneActivityStatus { + /// Zone (the cell's stream id). + pub zone_id: String, + /// [`ACTIVITY_ACTIVE`], [`ACTIVITY_DORMANT`] or [`ACTIVITY_HIBERNATED`]. + pub status: String, + /// Id of the object activation that reported it. + pub activation: String, + /// That activation restored a snapshot. + pub restored: bool, + /// Players in the zone when reported. + pub players: u32, + /// celld node running the activation (debug; empty when unknown). + pub node: String, + /// V8 isolate running the activation (debug). + pub isolate: String, + /// Activations seen so far. + pub activations: u64, + /// When the zone reported it (zone clock, Unix ms). + pub changed_at_ms: u64, +} + +impl From<&ZoneActivity> for ZoneActivityStatus { + fn from(activity: &ZoneActivity) -> Self { + Self { + zone_id: activity.entity.id().to_string(), + status: activity.status.clone(), + activation: activity.activation.clone(), + restored: activity.restored, + players: activity.players, + node: activity.node.clone(), + isolate: activity.isolate.clone(), + activations: activity.activations, + changed_at_ms: activity.changed_at_ms, + } + } +} + +#[sourced( + entity, + aggregate_type = "ZoneActivity", + events = "ZoneActivityEvent", + domain_state = ZoneActivityStatus +)] +impl ZoneActivity { + #[event("zone.activity_changed", version = 1, domain)] + fn changed(&mut self, zone_id: String, report: RecordActivity, stamp: StampKey) { + if self.entity.id().is_empty() { + self.entity.set_id(zone_id); + } + if report.activation != self.activation { + self.activations += 1; + } + self.status = report.status; + self.activation = report.activation; + self.restored = report.restored; + self.players = report.players; + self.node = report.node; + self.isolate = report.isolate; + self.changed_at_ms = stamp.emitted_at_ms; + self.last = Some(stamp); + } +} + +impl ZoneActivity { + /// Current status (empty before the first report). + pub fn status(&self) -> &str { + &self.status + } + + /// The public row. + pub fn row(&self) -> ZoneActivityStatus { + self.into() + } +} + +/// `zone_activity.record` input: what the zone reports about itself. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Default, + Serialize, + Deserialize, + distributed::CommandInput, + crate::FactCommand, +)] +#[fact(command = "zone_activity.record")] +pub struct RecordActivity { + /// [`ACTIVITY_ACTIVE`], [`ACTIVITY_DORMANT`] or [`ACTIVITY_HIBERNATED`]. + pub status: String, + /// Object activation id. + pub activation: String, + /// The activation restored a snapshot. + pub restored: bool, + /// Players present. + pub players: u32, + /// celld node (debug; empty when unknown). + pub node: String, + /// V8 isolate (debug). + pub isolate: String, +} + +/// The applied (or ignored) report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, distributed::CommandOutput)] +pub struct ActivityApplied { + /// False when a later-emitted report already set the status. + pub applied: bool, + /// The status after the command. + pub status: String, +} + +fn rejected(message: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(message.to_string()) +} + +async fn handle_record( + ctx: &CausalCommandContext<'_, ZoneActivity>, + input: RecordActivity, +) -> Result>, HandlerError> { + if ![ACTIVITY_ACTIVE, ACTIVITY_DORMANT, ACTIVITY_HIBERNATED].contains(&input.status.as_str()) { + return Err(rejected(format!("unknown activity `{}`", input.status))); + } + let stamp = session_stamp(ctx.session()) + .ok_or_else(|| rejected("activity reports are zone facts and need a fact stamp"))?; + if ctx.user_id()? != zone_principal(&stamp.zone_id) { + return Err(rejected("only the zone itself reports its activity")); + } + let repo = ctx.repo(); + let mut activity = match repo.get(&stamp.zone_id).await? { + Some(activity) => activity, + None => repo.create(), + }; + let newer = activity + .last + .as_ref() + .is_none_or(|last| last.stamp().emission_order(&stamp) == std::cmp::Ordering::Less); + if !newer { + let status = activity.status.clone(); + return repo.commit(activity)?.succeeded(ActivityApplied { + applied: false, + status, + }); + } + let status = input.status.clone(); + activity + .changed(stamp.zone_id.clone(), input, StampKey::of(&stamp)) + .map_err(rejected)?; + repo.publish_events() + .commit(activity)? + .succeeded(ActivityApplied { + applied: true, + status, + }) +} + +struct Record; +impl PortableCommand for Record +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + RECORD, + )) + .handle(handle_record) + } +} + +/// Mount the zone-activity command on a `ZoneActivity` cell. +pub fn mount_zone_activity(cell: AggregateCell) -> AggregateCell { + cell.mount(Record) +} diff --git a/distributed_game/src/world/zone_status.rs b/distributed_game/src/world/zone_status.rs index 2d029ec22..5ac3d4f12 100644 --- a/distributed_game/src/world/zone_status.rs +++ b/distributed_game/src/world/zone_status.rs @@ -8,11 +8,15 @@ //! |---|---|---| //! | [`ZoneLease`](super::zone_lease::ZoneLease) | `zone.activated`, `zone.released`, `zone.lease_expired` | [`ZoneLeases`] (`zone_leases`) | //! | [`ZoneState`](super::zone_state::ZoneState) | `zone_state.saved`, `zone_state.deleted` | [`ZoneSnapshots`] (`zone_snapshots`) | +//! | [`ZoneActivity`](super::zone_activity::ZoneActivity) (zones hosted as celld objects) | `zone.activity_changed` | [`ZoneActivities`] (`zone_activities`) | //! //! Each event carries its cell's post-transition state, so a row is an -//! upsert of the body ([`ZONE_LEASES`], [`ZONE_SNAPSHOTS`]); the projector -//! runtime orders and deduplicates per stream. [`ZoneStatus::of`] combines -//! a zone's two rows. Mount [`projectors`] on a consumer service (any read +//! upsert of the body ([`ZONE_LEASES`], [`ZONE_SNAPSHOTS`], +//! [`ZONE_ACTIVITIES`]); the projector runtime orders and deduplicates per +//! stream. [`ZoneStatus::of`] combines a zone's rows: a zone hosted as a +//! celld object ([`crate::celld_zone`]) has only an activity row (active / +//! dormant / hibernated); a zone on the native multi-host server has lease +//! and snapshot rows. Mount [`projectors`] on a consumer service (any read //! model store: SQLite, Postgres) fed by the bus the cells' outbox reaches. //! //! The read model is eventually consistent: it is for views (world maps, @@ -27,6 +31,9 @@ use distributed::{ }; use serde::{Deserialize, Serialize}; +use super::zone_activity::{ + ZoneActivityActivityChangedDomainEvent, ACTIVITY_ACTIVE, ACTIVITY_DORMANT, ACTIVITY_HIBERNATED, +}; use super::zone_lease::{ ZoneLeaseActivatedDomainEvent, ZoneLeaseLeaseExpiredDomainEvent, ZoneLeaseReleasedDomainEvent, STATUS_ACTIVE, @@ -44,6 +51,10 @@ pub const SNAPSHOTS_PROJECTOR: &str = "project_zone_snapshots"; pub const LEASES_EPOCH: &str = "zone-leases-v1"; /// Epoch of [`ZONE_SNAPSHOTS`]. pub const SNAPSHOTS_EPOCH: &str = "zone-snapshots-v1"; +/// Projector of [`ZoneActivities`]. +pub const ACTIVITIES_PROJECTOR: &str = "project_zone_activities"; +/// Epoch of [`ZONE_ACTIVITIES`]. +pub const ACTIVITIES_EPOCH: &str = "zone-activities-v1"; /// One `zone_leases` row: the last lease transition of a zone /// ([`ZoneLeaseStatus`](super::zone_lease::ZoneLeaseStatus); the type is @@ -80,6 +91,39 @@ pub struct ZoneSnapshots { pub bytes: i64, } +/// One `zone_activities` row: what a zone hosted as a celld object last +/// reported about itself +/// ([`ZoneActivityStatus`](super::zone_activity::ZoneActivityStatus)). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ReadModel)] +#[table("zone_activities")] +pub struct ZoneActivities { + /// Zone id. + #[id("zone_id")] + pub zone_id: String, + /// `active`, `dormant` or `hibernated`. + pub status: String, + /// Object activation id. + pub activation: String, + /// The activation restored a snapshot. + pub restored: bool, + /// Players when reported. + pub players: i64, + /// celld node of the activation (debug; empty when unknown). + pub node: String, + /// V8 isolate of the activation (debug). + pub isolate: String, + /// Activations seen. + pub activations: i64, + /// When the zone reported it (Unix ms). + pub changed_at_ms: i64, +} + +/// Upsert a [`ZoneActivities`] row from a `zone.activity_changed` body. +#[allow(non_snake_case)] +pub fn SaveZoneActivity() -> Mutation<()> { + mutation_file!("src/world/mutations/save_zone_activity.mutation.graphql") +} + /// Upsert a [`ZoneLeases`] from a lease event's state body. #[allow(non_snake_case)] pub fn SaveZoneLease() -> Mutation<()> { @@ -124,11 +168,52 @@ distributed::projection! { }; } -/// Physical read-model catalog (`zone_leases`, `zone_snapshots`). +distributed::projection! { + pub const ZONE_ACTIVITIES: ProjectionDescriptor = { + name: "project_zone_activities", + version: 1, + epoch: "zone-activities-v1", + model: ZoneActivities, + on { + events: [ZoneActivityActivityChangedDomainEvent], + mutation: SaveZoneActivity, + input: { activity: body }, + }, + }; +} + +/// Physical read-model catalog (`zone_leases`, `zone_snapshots`, +/// `zone_activities`). pub fn read_model_catalog() -> ReadModelCatalog { ReadModelCatalog::new(APPLICATION) .table_schema(ZoneLeases::schema().clone()) .table_schema(ZoneSnapshots::schema().clone()) + .table_schema(ZoneActivities::schema().clone()) +} + +/// The modeled projector of [`ZoneActivities`] alone (a consumer that +/// shows celld-hosted zones, e.g. a minimap service), applying +/// [`ZONE_ACTIVITIES`]. Mount it on a consumer service named +/// [`APPLICATION`] (or any name of the caller's, with its own cursor). +pub fn activity_projector() -> SurfaceProjector { + activity_projector_for(APPLICATION) +} + +/// [`activity_projector`] owned by `application` (when the consumer service +/// is not named [`APPLICATION`]). +pub fn activity_projector_for(application: &str) -> SurfaceProjector { + LocalProjectionMountsBuilder::new(application, "ordered-domain-events") + .expect("projection source") + .eventual_model::( + ACTIVITIES_PROJECTOR, + ZONE_ACTIVITIES, + ACTIVITIES_EPOCH, + ) + .expect("zone activities mount") + .build() + .expect("projection catalog") + .projector(ACTIVITIES_PROJECTOR) + .expect("zone activities projector") } /// The modeled projector mounts `(leases, snapshots)`: mount both on one @@ -157,11 +242,16 @@ pub fn projectors() -> (SurfaceProjector, SurfaceProjector) { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "state", rename_all = "camelCase")] pub enum ZoneStatus { - /// A holder has the lease. + /// Running: a holder has the lease (native multi-host server; `holder` + /// is the host) or a celld object reported `active` (`holder` is its + /// node when known, a debug detail). Active { - /// The lease holder. + /// The lease holder, or the celld node (may be empty). holder: String, }, + /// A celld-hosted zone that emptied: still in memory, snapshot written, + /// about to hibernate. + Dormant, /// No holder; a hibernation snapshot is stored. Hibernated { /// Snapshot size in bytes. @@ -172,6 +262,18 @@ pub enum ZoneStatus { } impl ZoneStatus { + /// The status of a celld-hosted zone from its activity row. + pub fn of_activity(activity: Option<&ZoneActivities>) -> Self { + match activity.map(|row| row.status.as_str()) { + Some(ACTIVITY_ACTIVE) => Self::Active { + holder: activity.map(|row| row.node.clone()).unwrap_or_default(), + }, + Some(ACTIVITY_DORMANT) => Self::Dormant, + Some(ACTIVITY_HIBERNATED) => Self::Hibernated { bytes: 0 }, + _ => Self::Cold, + } + } + /// Combine a zone's rows (either may be missing). pub fn of(lease: Option<&ZoneLeases>, snapshot: Option<&ZoneSnapshots>) -> Self { if let Some(lease) = lease.filter(|lease| lease.status == STATUS_ACTIVE) { @@ -224,7 +326,28 @@ mod tests { assert_eq!(ZoneStatus::of(None, None), ZoneStatus::Cold); // The projections compile against the cells' outward events. let (leases, snapshots) = projectors(); - drop((leases, snapshots)); - assert_eq!(read_model_catalog().tables.len(), 2); + drop((leases, snapshots, activity_projector())); + assert_eq!(read_model_catalog().tables.len(), 3); + let activity = |status: &str| ZoneActivities { + zone_id: "z".into(), + status: status.into(), + node: "node-a".into(), + ..ZoneActivities::default() + }; + assert_eq!( + ZoneStatus::of_activity(Some(&activity("active"))), + ZoneStatus::Active { + holder: "node-a".into() + } + ); + assert_eq!( + ZoneStatus::of_activity(Some(&activity("dormant"))), + ZoneStatus::Dormant + ); + assert_eq!( + ZoneStatus::of_activity(Some(&activity("hibernated"))), + ZoneStatus::Hibernated { bytes: 0 } + ); + assert_eq!(ZoneStatus::of_activity(None), ZoneStatus::Cold); } } diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index a6e2710bd..984631dbd 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -249,8 +249,10 @@ pub trait Zone: Sized + Send + 'static { /// state the zone sent on join (own entity id, hello events…). /// /// Defaults to `on_leave(Rejoined)` then `on_join`, i.e. a fresh join in - /// the same [`PlayerId`]. Override to keep in-zone state and only resend - /// what the client needs. + /// the same [`PlayerId`]: the body is respawned. Override to keep + /// in-zone state and only resend what the client needs, so a reloaded + /// page takes over the same body (what players expect; the examples' + /// Mob Pit, Arena and Endless Grid do). fn on_rejoin(&mut self, ctx: &mut Ctx<'_, Self>, player: PlayerId) { self.on_leave(ctx, player, LeaveReason::Rejoined); self.on_join(ctx, player); diff --git a/distributed_game/tests/celld_zone/main.rs b/distributed_game/tests/celld_zone/main.rs new file mode 100644 index 000000000..37b9f5017 --- /dev/null +++ b/distributed_game/tests/celld_zone/main.rs @@ -0,0 +1,420 @@ +//! Zones as celld Durable Objects (`celld_zone`), end to end against +//! `tests/celld/zone-worker` (a tally zone, its Tally cells and its +//! ZoneActivity cell): +//! +//! - `single_node`: `celld dev`. Join with a ticket, input, facts settle in +//! the player's cell, the empty zone goes dormant then hibernated, a new +//! join wakes it with its state, tickets and transfer tickets are single +//! use, a doorway hands the player to the next zone. +//! - `fleet`: three celld nodes in production mode on one RustFS bucket +//! (docker compose). A zone is reached through a node that does not own +//! it, the owner is killed while 200 facts are in flight: the socket +//! closes with 1012, the client rejoins through another node, the zone +//! restores, and every fact reaches the cell exactly once. +//! +//! ```text +//! (cd tests/celld/zone-worker && worker-build --release) +//! CELLD_BIN=… CELLD_ESBUILD=… cargo test -p distributed_game \ +//! --features celld,net --test celld_zone +//! ``` +//! +//! Skips without a built worker / celld (and, for `fleet`, docker and the +//! aws CLI) unless `CELLD_ZONE_REQUIRED=1`. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use distributed_game::protocol::{error, ClientMessage, Hello, ServerMessage, PROTOCOL_VERSION}; +use distributed_game::ticket::TicketKey; +use distributed_game::{Role, TicketClaims}; +use futures_util::{SinkExt, StreamExt}; +use serde_json::Value; +use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; +use tokio_tungstenite::tungstenite::Message; + +#[path = "../support/celld.rs"] +mod celld_support; +#[path = "../support/fleet.rs"] +mod fleet; + +const INTERNAL_SECRET: &str = "test-only-internal-secret-change-me-2026"; +const TICKET_SECRET: &str = "test-only-ticket-secret-change-me-2026"; +const REQUIRED: &str = "CELLD_ZONE_REQUIRED"; + +fn worker() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../tests/celld/zone-worker") +} + +fn ticket(principal: &str, zone: &str) -> String { + let exp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + + 120; + let nonce = format!( + "{principal}-{zone}-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + TicketKey::new(TICKET_SECRET) + .sign(&TicketClaims::new(principal, zone, exp, nonce, Role::Player).with_name(principal)) +} + +type Ws = + tokio_tungstenite::WebSocketStream>; + +#[derive(Debug)] +enum Got { + Server(ServerMessage), + Closed(Option), +} + +struct Player { + ws: Ws, + seq: u32, +} + +impl Player { + async fn connect(base: &str, zone: &str) -> Self { + Self::try_connect(base, zone).await.expect("connect") + } + + async fn try_connect(base: &str, zone: &str) -> Result { + let url = format!("{}/rooms/{zone}", base.replacen("http://", "ws://", 1)); + let (ws, _) = tokio_tungstenite::connect_async(url) + .await + .map_err(|e| e.to_string())?; + Ok(Self { ws, seq: 0 }) + } + + async fn send(&mut self, message: ClientMessage) { + self.ws + .send(Message::Binary(message.encode().into())) + .await + .expect("send"); + } + + async fn hello(&mut self, ticket: &str) { + self.send(ClientMessage::Hello(Hello { + protocol_version: PROTOCOL_VERSION, + content_version: String::new(), + capabilities: 0, + ticket: ticket.into(), + reconnect_token: String::new(), + })) + .await; + } + + async fn input(&mut self, payload: &[u8]) { + self.seq += 1; + self.send(ClientMessage::Input { + seq: self.seq, + client_tick: 0, + payload: payload.to_vec(), + }) + .await; + } + + async fn next(&mut self, within: Duration) -> Option { + let deadline = Instant::now() + within; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + let frame = tokio::time::timeout(left, self.ws.next()).await.ok()?; + match frame { + Some(Ok(Message::Binary(bytes))) => match ServerMessage::decode(&bytes, &()) { + Ok(message) => return Some(Got::Server(message)), + // Deltas need the Welcome's schema; these tests skip them. + Err(_) => continue, + }, + Some(Ok(Message::Close(frame))) => { + return Some(Got::Closed(frame.map(|f| u16::from(f.code)))) + } + Some(Ok(_)) => continue, + Some(Err(_)) | None => return Some(Got::Closed(None)), + } + } + } + + /// The first message matching `pick`, skipping others (Deltas, Pongs). + async fn expect( + &mut self, + within: Duration, + what: &str, + mut pick: impl FnMut(&Got) -> Option, + ) -> T { + let deadline = Instant::now() + within; + while let Some(got) = self + .next(deadline.saturating_duration_since(Instant::now())) + .await + { + if let Some(value) = pick(&got) { + return value; + } + if let Got::Closed(code) = got { + panic!("closed ({code:?}) while waiting for {what}"); + } + } + panic!("timed out waiting for {what}"); + } + + async fn welcome(&mut self) { + self.expect(Duration::from_secs(20), "Welcome", |got| match got { + Got::Server(ServerMessage::Welcome(_)) => Some(()), + Got::Server(ServerMessage::Error { code, message }) => { + panic!("join refused: {code} {message}") + } + _ => None, + }) + .await + } + + /// Event `type_id` carrying a u64 total. + async fn total(&mut self, type_id: u32) -> u64 { + self.expect(Duration::from_secs(20), "a total event", |got| match got { + Got::Server(ServerMessage::Event { + type_id: t, + payload, + .. + }) if *t == type_id => Some(u64::from_le_bytes(payload[..8].try_into().unwrap())), + _ => None, + }) + .await + } + + async fn refused(&mut self) -> u32 { + self.expect(Duration::from_secs(20), "an Error", |got| match got { + Got::Server(ServerMessage::Error { code, .. }) => Some(*code), + Got::Server(ServerMessage::Welcome(_)) => panic!("admitted a used ticket"), + _ => None, + }) + .await + } +} + +/// GET a JSON route with the internal secret (Null on any failure: the +/// callers poll). +async fn get(base: &str, path: &str) -> Value { + let response = reqwest::Client::new() + .get(format!("{base}{path}")) + .header("x-distributed-internal-secret", INTERNAL_SECRET) + .send() + .await + .and_then(|r| r.error_for_status()); + match response { + Ok(response) => response.json().await.unwrap_or(Value::Null), + Err(_) => Value::Null, + } +} + +/// Poll `path` until `ok` (or panic after `within`). +async fn until(base: &str, path: &str, within: Duration, ok: impl Fn(&Value) -> bool) -> Value { + let deadline = Instant::now() + within; + loop { + let value = get(base, path).await; + if ok(&value) { + return value; + } + assert!(Instant::now() < deadline, "{path}: last {value}"); + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +fn status(value: &Value) -> &str { + value["activity"]["status"].as_str().unwrap_or("") +} + +fn exactly_once(value: &Value, count: u64) -> bool { + let hits: Vec<&str> = value["hits"] + .as_array() + .map(|a| a.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + let distinct: std::collections::HashSet<_> = hits.iter().collect(); + assert_eq!( + distinct.len(), + hits.len(), + "a fact committed twice: {value}" + ); + hits.len() as u64 == count +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn single_node() { + let Some(celld) = celld_support::Celld::start_worker(&worker(), REQUIRED) else { + return; + }; + let base = celld.base(); + let long = Duration::from_secs(30); + + // Join, input, facts. + let mut ana = Player::connect(&base, "room-a").await; + let first = ticket("ana", "room-a"); + ana.hello(&first).await; + ana.welcome().await; + assert_eq!(ana.total(1).await, 0); + let active = until(&base, "/zone-activity/room-a", long, |v| { + status(v) == "active" + }) + .await; + assert_eq!(active["activity"]["restored"], false); + ana.input(&[1, 5]).await; + assert_eq!(ana.total(2).await, 5); + until(&base, "/tally/ana", long, |v| exactly_once(v, 5)).await; + + // A used ticket is refused. + let mut replay = Player::connect(&base, "room-a").await; + replay.hello(&first).await; + assert_eq!(replay.refused().await, error::UNAUTHORIZED); + + // Empty: dormant, then hibernated (dropped from memory). + ana.send(ClientMessage::Leave).await; + until(&base, "/zone-activity/room-a", long, |v| { + status(v) == "dormant" + }) + .await; + until(&base, "/zone-activity/room-a", long, |v| { + status(v) == "hibernated" + }) + .await; + let info = get(&base, "/rooms/room-a/info").await; + assert_eq!(info["awake"], false, "{info}"); + + // Wake with state. + let mut ana = Player::connect(&base, "room-a").await; + ana.hello(&ticket("ana", "room-a")).await; + ana.welcome().await; + assert_eq!(ana.total(1).await, 5, "restored from the snapshot"); + let woke = until(&base, "/zone-activity/room-a", long, |v| { + status(v) == "active" + }) + .await; + assert_eq!(woke["activity"]["restored"], true); + assert!(woke["activity"]["activations"].as_u64().unwrap() >= 2); + + // A doorway: handoff to room-b with a single-use transfer ticket. + ana.input(&[2, b'b']).await; + let transfer = ana + .expect(long, "Handoff", |got| match got { + Got::Server(ServerMessage::Handoff { + target_zone, + transfer_ticket, + }) => { + assert_eq!(target_zone, "room-b"); + Some(transfer_ticket.clone()) + } + _ => None, + }) + .await; + let mut ana_b = Player::connect(&base, "room-b").await; + ana_b.hello(&transfer).await; + ana_b.welcome().await; + let mut again = Player::connect(&base, "room-b").await; + again.hello(&transfer).await; + assert_eq!(again.refused().await, error::UNAUTHORIZED); + ana_b.input(&[1, 3]).await; + assert_eq!(ana_b.total(2).await, 3); + until(&base, "/tally/ana", long, |v| exactly_once(v, 8)).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fleet() { + let Some(mut fleet) = + fleet::Fleet::start(&worker(), 3, &[("CELLD_ZONE_NODES", "{nodes}")], REQUIRED) + else { + return; + }; + let long = Duration::from_secs(60); + // First touch through node-a places the zone (celld: first touch). + let a = fleet.base(0); + let mut probe = Player::connect(&a, "room-x").await; + probe.hello(&ticket("probe", "room-x")).await; + probe.welcome().await; + let info = until(&a, "/rooms/room-x/info", long, |v| { + v["node"].as_str().is_some_and(|n| !n.is_empty()) + }) + .await; + let owner = fleet + .index(info["node"].as_str().unwrap()) + .expect("known node"); + probe.send(ClientMessage::Leave).await; + // Empty: the zone writes its snapshot and goes dormant. + until(&a, "/zone-activity/room-x", long, |v| { + status(v) == "dormant" + }) + .await; + let via = (owner + 1) % 3; + let rejoin_via = (owner + 2) % 3; + + // Play through a node that does not own the zone (forwarded). + let mut bo = Player::connect(&fleet.base(via), "room-x").await; + bo.hello(&ticket("bo", "room-x")).await; + bo.welcome().await; + bo.input(&[1, 200]).await; + assert_eq!(bo.total(2).await, 200); + // Crash the owner once the 200 facts are in the zone's outbox (written + // right after the tick that emitted them); some may already be + // committed, most are in flight. + until(&a, "/rooms/room-x/info", long, |v| { + v["spooledFacts"].as_u64().unwrap_or(0) >= 200 + }) + .await; + fleet.kill(owner); + let closed = bo + .expect(Duration::from_secs(30), "the close", |got| match got { + Got::Closed(code) => Some(*code), + _ => None, + }) + .await; + assert_eq!( + closed, + Some(u16::from(CloseCode::Restart)), + "a forwarded socket of a crashed zone closes with 1012" + ); + + // Rejoin through the third node: celld recovers the zone elsewhere. + let base = fleet.base(rejoin_via); + let deadline = Instant::now() + long; + let (_bo, restored) = loop { + let mut bo = match Player::try_connect(&base, "room-x").await { + Ok(bo) => bo, + // celld answers 500 until the dead owner's lease expires. + Err(_) => { + assert!(Instant::now() < deadline, "the zone never came back"); + tokio::time::sleep(Duration::from_millis(500)).await; + continue; + } + }; + bo.hello(&ticket("bo", "room-x")).await; + let got = bo + .expect( + Duration::from_secs(20), + "Welcome or Error", + |got| match got { + Got::Server(ServerMessage::Welcome(_)) => Some(true), + Got::Server(ServerMessage::Error { .. }) => Some(false), + Got::Closed(_) => Some(false), + _ => None, + }, + ) + .await; + if got { + let total = bo.total(1).await; + break (bo, total); + } + assert!(Instant::now() < deadline, "the zone never came back"); + tokio::time::sleep(Duration::from_millis(500)).await; + }; + assert!(restored <= 200, "restored total {restored}"); + // Every hit the zone emitted reaches the cell exactly once. + until(&base, "/tally/bo", long, |v| exactly_once(v, 200)).await; + let activity = until(&base, "/zone-activity/room-x", long, |v| { + status(v) == "active" && v["activity"]["restored"] == true + }) + .await; + assert_ne!( + activity["activity"]["node"].as_str().unwrap_or(""), + fleet.nodes[owner].name, + "recovered on a live node" + ); +} diff --git a/distributed_game/tests/support/celld.rs b/distributed_game/tests/support/celld.rs index 1efd2fffa..03d2ce17d 100644 --- a/distributed_game/tests/support/celld.rs +++ b/distributed_game/tests/support/celld.rs @@ -26,13 +26,23 @@ pub struct Celld { impl Celld { pub fn start_new() -> Option { - let build = worker_dir().join("build"); + Self::start_worker(&worker_dir(), "CELLD_FACTS_REQUIRED") + } + + /// `celld dev` serving the built worker in `dir`; skips (None) when it + /// is not built or celld cannot start, unless env `required` is set. + pub fn start_worker(dir: &Path, required: &str) -> Option { + let build = dir.join("build"); if !build.join("worker/shim.mjs").exists() { assert!( - std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), - "CELLD_FACTS_REQUIRED is set but the fact worker is not built" + std::env::var_os(required).is_none(), + "{required} is set but {} is not built", + dir.display() + ); + eprintln!( + "skipping: build {} first (worker-build --release)", + dir.display() ); - eprintln!("skipping celld fact routing: build tests/celld/fact-worker first"); return None; } let project = std::env::temp_dir().join(format!( @@ -44,11 +54,7 @@ impl Celld { .as_nanos() )); copy_dir(&build, &project.join("build")); - std::fs::copy( - worker_dir().join("wrangler.jsonc"), - project.join("wrangler.jsonc"), - ) - .unwrap(); + std::fs::copy(dir.join("wrangler.jsonc"), project.join("wrangler.jsonc")).unwrap(); let port = TcpListener::bind("127.0.0.1:0") .unwrap() .local_addr() @@ -62,10 +68,10 @@ impl Celld { }; if let Err(reason) = celld.start() { assert!( - std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), + std::env::var_os(required).is_none(), "celld dev is required but did not start: {reason}" ); - eprintln!("skipping celld fact routing: {reason}"); + eprintln!("skipping: {reason}"); return None; } Some(celld) @@ -100,7 +106,10 @@ impl Celld { let log = std::fs::File::create(&log_path).unwrap(); let child = Command::new(&binary) .args(&args) - .env("RUST_LOG", "warn") + .env( + "RUST_LOG", + std::env::var("CELLD_RUST_LOG").unwrap_or_else(|_| "warn".into()), + ) .stdin(Stdio::null()) .stdout(log.try_clone().unwrap()) .stderr(log) @@ -200,7 +209,7 @@ impl Drop for Celld { } } -fn copy_dir(from: &Path, to: &Path) { +pub fn copy_dir(from: &Path, to: &Path) { std::fs::create_dir_all(to).unwrap(); for entry in std::fs::read_dir(from).unwrap() { let entry = entry.unwrap(); @@ -213,7 +222,7 @@ fn copy_dir(from: &Path, to: &Path) { } } -fn blocking_get(url: &str) -> Option { +pub fn blocking_get(url: &str) -> Option { let url = url.strip_prefix("http://")?; let (host, path) = url.split_once('/')?; let mut stream = std::net::TcpStream::connect(host).ok()?; diff --git a/distributed_game/tests/support/fleet.rs b/distributed_game/tests/support/fleet.rs new file mode 100644 index 000000000..9eb7231be --- /dev/null +++ b/distributed_game/tests/support/fleet.rs @@ -0,0 +1,308 @@ +//! A local multi-node celld fleet owned by one test: one RustFS container +//! (docker compose, its own project and port) holding the fleet bucket, the +//! worker deployed to it with `celld deploy`, and N `celld` nodes in +//! production mode. Kills only what it started; removes the container on +//! drop. Skips (None) without docker / aws / a built worker unless env +//! `required` is set. + +#![allow(dead_code)] + +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use super::celld_support::{blocking_get, copy_dir}; + +const ACCESS_KEY: &str = "fleettest"; +const SECRET_KEY: &str = "fleettestsecret123"; + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +pub struct Node { + pub name: String, + pub public: u16, + pub operator: u16, + child: Option, +} + +pub struct Fleet { + dir: PathBuf, + project: String, + s3_port: u16, + bucket: String, + binary: String, + pub nodes: Vec, +} + +fn skip(required: &str, reason: String) -> Option { + assert!( + std::env::var_os(required).is_none(), + "{required} is set but the fleet is unavailable: {reason}" + ); + eprintln!("skipping the celld fleet test: {reason}"); + None +} + +impl Fleet { + /// Start `nodes` nodes serving the worker in `worker` with extra + /// string `vars` (`{nodes}` in a value is replaced by the operator + /// listeners, `node-a=127.0.0.1:p,…`). + pub fn start( + worker: &Path, + nodes: usize, + vars: &[(&str, &str)], + required: &str, + ) -> Option { + if !worker.join("build/worker/shim.mjs").exists() { + return skip(required, format!("build {} first", worker.display())); + } + for tool in ["docker", "aws"] { + if Command::new(tool).arg("--version").output().is_err() { + return skip(required, format!("`{tool}` is not available")); + } + } + let binary = std::env::var("CELLD_BIN").unwrap_or_else(|_| "celld".into()); + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + let dir = std::env::temp_dir().join(format!( + "distributed-celld-fleet-{}-{stamp}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let mut fleet = Self { + project: format!("dg-fleet-{}-{}", std::process::id(), stamp % 100_000), + s3_port: free_port(), + bucket: "dg-fleet".into(), + binary, + nodes: Vec::new(), + dir, + }; + for i in 0..nodes { + fleet.nodes.push(Node { + name: format!("node-{}", (b'a' + i as u8) as char), + public: free_port(), + operator: free_port(), + child: None, + }); + } + if let Err(reason) = fleet.s3_up() { + return skip(required, reason); + } + let listeners = fleet + .nodes + .iter() + .map(|n| format!("{}=127.0.0.1:{}", n.name, n.operator)) + .collect::>() + .join(","); + let vars: Vec<(String, String)> = vars + .iter() + .map(|(k, v)| (k.to_string(), v.replace("{nodes}", &listeners))) + .collect(); + if let Err(reason) = fleet.deploy(worker, &vars) { + return skip(required, reason); + } + for i in 0..fleet.nodes.len() { + if let Err(reason) = fleet.start_node(i) { + return skip(required, reason); + } + } + Some(fleet) + } + + fn env(&self, command: &mut Command) { + command + .env("AWS_ACCESS_KEY_ID", ACCESS_KEY) + .env("AWS_SECRET_ACCESS_KEY", SECRET_KEY) + .env("AWS_REGION", "us-east-1") + .env("S3_ENDPOINT", self.endpoint()); + } + + fn endpoint(&self) -> String { + format!("http://127.0.0.1:{}", self.s3_port) + } + + fn s3_up(&self) -> Result<(), String> { + let compose = self.dir.join("s3-compose.yaml"); + std::fs::write( + &compose, + format!( + "services:\n s3:\n image: rustfs/rustfs:1.0.3\n command: [\"/data\"]\n environment:\n RUSTFS_ADDRESS: \"0.0.0.0:9000\"\n RUSTFS_ACCESS_KEY: {ACCESS_KEY}\n RUSTFS_SECRET_KEY: {SECRET_KEY}\n ports: [\"127.0.0.1:{}:9000\"]\n", + self.s3_port + ), + ) + .unwrap(); + let up = Command::new("docker") + .args(["compose", "-p", &self.project, "-f"]) + .arg(&compose) + .args(["up", "-d"]) + .output() + .map_err(|e| format!("docker compose: {e}"))?; + if !up.status.success() { + return Err(format!( + "docker compose up: {}", + String::from_utf8_lossy(&up.stderr) + )); + } + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let mut mb = Command::new("aws"); + self.env(&mut mb); + let made = mb + .args(["--endpoint-url", &self.endpoint(), "s3", "mb"]) + .arg(format!("s3://{}", self.bucket)) + .output() + .map_err(|e| format!("aws: {e}"))?; + let text = String::from_utf8_lossy(&made.stderr); + if made.status.success() || text.contains("BucketAlready") { + return Ok(()); + } + if Instant::now() > deadline { + return Err(format!("bucket: {text}")); + } + std::thread::sleep(Duration::from_millis(500)); + } + } + + fn bucket_url(&self) -> String { + format!("s3://{}/fleet", self.bucket) + } + + fn deploy(&self, worker: &Path, vars: &[(String, String)]) -> Result<(), String> { + let project = self.dir.join("worker"); + copy_dir(&worker.join("build"), &project.join("build")); + let config = std::fs::read_to_string(worker.join("wrangler.jsonc")).unwrap(); + let mut config: serde_json::Value = serde_json::from_str(&strip_comments(&config)) + .map_err(|e| format!("wrangler.jsonc: {e}"))?; + for (key, value) in vars { + config["vars"][key] = serde_json::Value::String(value.clone()); + } + std::fs::write(project.join("wrangler.json"), config.to_string()).unwrap(); + let mut deploy = Command::new(&self.binary); + self.env(&mut deploy); + let out = deploy + .arg("deploy") + .arg(&project) + .args(["--bucket", &self.bucket_url()]) + .output() + .map_err(|e| format!("celld deploy: {e}"))?; + if !out.status.success() { + return Err(format!( + "celld deploy: {}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + )); + } + Ok(()) + } + + /// (Re)start node `i`. + pub fn start_node(&mut self, i: usize) -> Result<(), String> { + let node = &self.nodes[i]; + let dir = self.dir.join(&node.name); + std::fs::create_dir_all(&dir).unwrap(); + let log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(dir.join("celld.log")) + .unwrap(); + let mut command = Command::new(&self.binary); + self.env(&mut command); + let child = command + .args(["--bucket", &self.bucket_url()]) + .args(["--listen", &format!("127.0.0.1:{}", node.public)]) + .args(["--internal-listen", &format!("127.0.0.1:{}", node.operator)]) + .args(["--advertise", &format!("127.0.0.1:{}", node.operator)]) + .env("CELLD_NODE", &node.name) + .env("CELLD_WATCH", dir.join("watch")) + .env("CELLD_READY_FLEET_GATE_MS", "0") + .env( + "RUST_LOG", + std::env::var("CELLD_RUST_LOG").unwrap_or_else(|_| "warn".into()), + ) + .stdin(Stdio::null()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .map_err(|e| format!("celld: {e}"))?; + let public = node.public; + self.nodes[i].child = Some(child); + let deadline = Instant::now() + Duration::from_secs(90); + while Instant::now() < deadline { + if blocking_get(&format!("http://127.0.0.1:{public}/health")) == Some(200) { + return Ok(()); + } + let exited = self.nodes[i] + .child + .as_mut() + .and_then(|c| c.try_wait().ok().flatten()); + if let Some(status) = exited { + // A cold node sometimes fails its first start; retry once. + self.nodes[i].child = None; + return Err(format!( + "{} exited ({status}); logs in {}", + self.nodes[i].name, + dir.display() + )); + } + std::thread::sleep(Duration::from_millis(250)); + } + Err(format!( + "{} not ready; logs in {}", + self.nodes[i].name, + dir.display() + )) + } + + /// `http://127.0.0.1:{public port}` of node `i`. + pub fn base(&self, i: usize) -> String { + format!("http://127.0.0.1:{}", self.nodes[i].public) + } + + /// Index of the node named `name`. + pub fn index(&self, name: &str) -> Option { + self.nodes.iter().position(|n| n.name == name) + } + + /// SIGKILL node `i` (a crash). + pub fn kill(&mut self, i: usize) { + if let Some(mut child) = self.nodes[i].child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +impl Drop for Fleet { + fn drop(&mut self) { + for i in 0..self.nodes.len() { + self.kill(i); + } + let _ = Command::new("docker") + .args(["compose", "-p", &self.project, "-f"]) + .arg(self.dir.join("s3-compose.yaml")) + .args(["down", "-v"]) + .output(); + if std::thread::panicking() { + eprintln!("fleet logs kept in {}", self.dir.display()); + } else { + let _ = std::fs::remove_dir_all(&self.dir); + } + } +} + +/// Drop `//` line comments (wrangler.jsonc). +fn strip_comments(text: &str) -> String { + text.lines() + .filter(|line| !line.trim_start().starts_with("//")) + .collect::>() + .join("\n") +} diff --git a/distributed_game/tests/zone_status/main.rs b/distributed_game/tests/zone_status/main.rs index 18e6ffa15..fc0b1c116 100644 --- a/distributed_game/tests/zone_status/main.rs +++ b/distributed_game/tests/zone_status/main.rs @@ -230,3 +230,155 @@ async fn cell_events_project_into_zone_status_rows() { assert_eq!(of(status("z-never").await), ZoneStatus::Cold); let _ = std::fs::remove_dir_all(dir); } + +/// A celld-hosted zone's lifecycle reports (`zone_activity.record` facts) +/// project into `zone_activities`; a report emitted earlier but settled +/// later never overwrites a newer one; only the zone itself may report. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn zone_activity_reports_project_into_zone_status() { + use distributed_game::cell::trusted_session; + use distributed_game::world::zone_activity::{ + mount_zone_activity, zone_principal, RecordActivity, ZoneActivity, RECORD, + }; + use distributed_game::world::zone_status::{ + activity_projector_for, ZoneActivities, ZONE_ACTIVITIES, + }; + use distributed_game::{Fact, FactStamp}; + + let clock = ManualClock::new(SystemTime::UNIX_EPOCH + Duration::from_millis(T0)); + let cell = mount_zone_activity(cell::("ZoneActivity", "pit.0.1", &clock)); + let report = |status: &str, seq: u64, principal: &str| { + let fact = Fact::trusted( + format!("zone_activity.record:pit.0.1:9:{seq}"), + RECORD, + serde_json::to_value(RecordActivity { + status: status.into(), + activation: "act-1".into(), + restored: status == "active" && seq > 1, + players: u32::from(status == "active"), + node: "node-b".into(), + isolate: "iso".into(), + }) + .unwrap(), + Some(principal.to_string()), + FactStamp { + zone_id: "pit.0.1".into(), + activation: 9, + seq, + emitted_at_ms: T0 + seq, + ..FactStamp::default() + }, + ); + (fact.clone(), trusted_session(principal, &fact)) + }; + let send = |status: &'static str, seq: u64, principal: &'static str| { + let cell = &cell; + async move { + let (fact, session) = report(status, seq, principal); + let identity = CellCommandIdentity::new( + "celld-zone", + principal, + command_id_for("celld-zone", fact.key()), + ) + .unwrap(); + cell.dispatch_idempotent(RECORD, &identity, fact.input().clone(), session) + .await + } + }; + let me = Box::leak(zone_principal("pit.0.1").into_boxed_str()); + assert_eq!( + send("active", 1, me).await.unwrap().payload()["applied"], + true + ); + assert_eq!( + send("hibernated", 3, me).await.unwrap().payload()["applied"], + true + ); + // Emitted before `hibernated`, settled after it: ignored. + let late = send("dormant", 2, me).await.unwrap(); + assert_eq!(late.payload()["applied"], false); + assert_eq!(late.payload()["status"], "hibernated"); + assert!( + send("active", 4, "guest-x").await.is_err(), + "only the zone reports" + ); + assert_eq!(cell.load().await.unwrap().unwrap().status(), "hibernated"); + + let dir = std::env::temp_dir().join(format!( + "zone-activity-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let bus_url = format!("sqlite:{}?mode=rwc", dir.join("bus.db").display()); + let pool = sqlx::SqlitePool::connect(&bus_url).await.unwrap(); + let bus = SqliteBus::new(pool.clone()); + bus.ensure_tables().await.unwrap(); + let publisher = BusPublisher::new(Arc::new(bus)); + let mut messages = cell.durable_outbox().unwrap(); + messages.sort_by_key(|m| m.created_at); + assert_eq!(messages.len(), 2, "one event per applied report"); + for message in messages { + publisher.publish(Message::from(message)).await.unwrap(); + } + let read_url = format!("sqlite:{}?mode=rwc", dir.join("read.db").display()); + let repo = SqliteRepository::connect_and_migrate(&read_url) + .await + .unwrap(); + repo.bootstrap_table_schema_for_dev(&read_model_catalog().table_registry().unwrap()) + .await + .unwrap(); + let read_models = repo.clone(); + async fn project( + context: CausalProjectorContext, + projection: ModeledProjection, + ) -> Result<(), HandlerError> { + projection.apply(ZONE_ACTIVITIES, &context).await + } + // A consumer of its own (a minimap service would be one too). + spawn_service_consumer_loop(move || { + Service::new() + .named("zone-minimap") + .routes( + Routes::new() + .with_read_model_store(read_models.clone()) + .modeled_projector(activity_projector_for("zone-minimap")) + .handle(project), + ) + .with_bus(SqliteBus::new(pool.clone()).with_idle_poll(CONSUMER_IDLE_POLL)) + }); + let start = Instant::now(); + let row = loop { + let row = sqlx::query_as::<_, (String, String, String, i64)>( + "SELECT zone_id, status, node, activations FROM zone_activities WHERE zone_id = ?", + ) + .bind("pit.0.1") + .fetch_optional(repo.pool()) + .await + .unwrap(); + if let Some(row) = row.filter(|row| row.1 == "hibernated") { + break row; + } + assert!( + start.elapsed() < Duration::from_secs(20), + "projection timed out" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + }; + assert_eq!(row.2, "node-b"); + assert_eq!(row.3, 1); + let activity = ZoneActivities { + zone_id: row.0, + status: row.1, + node: row.2, + ..ZoneActivities::default() + }; + assert_eq!( + ZoneStatus::of_activity(Some(&activity)), + ZoneStatus::Hibernated { bytes: 0 } + ); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/examples/game/README.md b/examples/game/README.md index bc3f1a98a..b2cbfcece 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -9,8 +9,8 @@ guests and lets any page change the network simulation. | Demo | Run | Mechanic it demonstrates | Durable state | |---|---|---|---| | **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | none (stateless: positions only live in the zone) | -| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player celld cells (`cells/`, `celld dev` started by `run.sh`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence cells fed by the runtime's player lifecycle facts (`/dev/mobpit/presence`) | celld: `crawler/{player}`, `presence/{player}`; crawler list = `player_stats` read model projected from the cells' events | -| **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | celld (the Mob Pit cells); boards are a projection (memory or Redis) | +| **Mob Pit** | `./run.sh mobpit` (`--fleet` for a local multi-node celld) | A near-infinite seeded grid of rooms joined by doorways, **each room a celld Durable Object** (celld places, forwards, hibernates and recovers rooms; the debug overlay shows the node); authoritative melee combat vs. telegraphing mobs; kills are **non-blocking durable facts** settled exactly once in per-player cells; rooms you leave go dormant then asleep (live minimap from the zone status read model) and come back with their mobs | celld: `pit..` rooms (snapshots), `crawler/{player}`, `presence/{player}`, `zone-activity/{room}`; crawler list and room status = read models projected from the cells' events | +| **Leaderboard** | `./run.sh leaderboard` | Same setup as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed from the projected crawler list with idempotent `max` updates, your rank + neighbours, rebuild from the cells' event history | celld (the Mob Pit cells); boards are a projection (memory or Redis) | | **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | none (stateless: score is per session) | | **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | celld: `gem-pouch/{player}`, `content-log/{zone}` | | **Endless Grid** | `./run.sh grid` | Seeded procedural world (terrain/biomes/props generated bit-identically in Rust and TS, nothing static replicated), 32 m cells as zones, observer connections to the 8 neighbours, handoff across borders **and hosts** (2 processes, hash ring, Redirect + transfer tickets), hibernation to celld and restore | celld: `zone-lease/{zone}`, `zone-state/{zone}` (hibernation) | @@ -243,15 +243,16 @@ REDIS_URL=redis://127.0.0.1:6379 examples/game/run.sh sveltekit # → http://localhost:5573/ — sign in with any username ``` -`sveltekit/` is Mob Pit as a SvelteKit app over two backends: the Mob Pit -zone server (gameplay) and `mobpit-service` (the business-side read model). -The script starts `celld dev` with the Mob Pit cells (`cells/`; state in -`.data/mobpit-sveltekit/celld`), then `game-demos mobpit` (no `--dev`) on -`GAME_DEMOS_PORT` (default 2967 here; data in `.data/mobpit-sveltekit`) with -a fresh ticket secret, `--redis-url "$REDIS_URL"` (its kill boards) and -`--projection-bus sqlite:…/bus.db` (celld's outbox Queue relays the Crawler -domain events to it; the board change notices go there too), then runs -`distributed dev server/`: the +`sveltekit/` is Mob Pit as a SvelteKit app over its rooms on celld +(gameplay) and `mobpit-service` (the business-side read models). The +script starts `celld dev` with the rooms and cells (`cells/`; state in +`.data/mobpit-sveltekit/celld`; `run.sh sveltekit --fleet [N]` starts a +local multi-node fleet instead, see *Mob Pit*), then `game-demos mobpit` +(the services, no `--dev`) on `GAME_DEMOS_PORT` (default 2967 here; data in +`.data/mobpit-sveltekit`) with the rooms' ticket secret, `--redis-url +"$REDIS_URL"` (its kill boards) and `--projection-bus sqlite:…/bus.db` +(celld's outbox Queue relays the cells' domain events to it; the board +change notices go there too), then runs `distributed dev server/`: the application lifecycle generates the `$distributed` GraphQL client, starts `mobpit-service` on `MOBPIT_SERVICE_PORT` (default `GAME_DEMOS_PORT + 2`) and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: @@ -262,12 +263,17 @@ and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: username sign-in (`dev-` principal, same encrypted session cookie, nothing external). `src/lib/server/game.ts` is the whole ticket path: `createGameTicketHandler({ secret, getSession: (e) => - e.locals.auth(), zone: () => 'pit-1' })`, exported as `POST` from + e.locals.auth(), zone })`, exported as `POST` from `routes/api/game/ticket/+server.ts`. The principal comes from the session, - the zone from the server; a body naming a principal is refused (400), a - foreign zone gets 409, another origin 403, no session 401. The page's - `load` uses `tickets.resolve(event)` for the same zone. There is no - `/dev/ticket` call and no guest token. + the room from the server: on a page load the room the player was last + in (its presence cell on celld, read with `CELLD_URL` / + `DISTRIBUTED_INTERNAL_SECRET`), else the entrance `pit.0.0`; or the room + a client rejoins (after a room restart closed its socket with 1012); moving + through doorways needs no ticket from here (the room signs a transfer + ticket). A body naming a principal is refused (400), something that is + not a room gets 409, another origin 403, no session 401. The page's + `load` uses `tickets.resolve(event)`. There is no `/dev/ticket` call and + no guest token. - **Code generation.** `distributed.config.js` lists both surfaces. `games`: the Mob Pit zone surface; the Vite plugin runs `game-manifest`, `game-client` and `game-wasm` on start and on Rust changes (`server/src`, @@ -313,9 +319,16 @@ and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: `GraphqlEngine::leaderboard_change_handlers` (`spawn_board_notices`). Both processes use `REDIS_URL` and the same key prefix (`MOBPIT_REDIS_PREFIX`, default `mobpit`). - The rows come from the cells (see *Mob Pit*, step 5): the zone's - `profile`/`board` pushes and the `--dev` leaderboard route are not used by - this app (they remain for the vanilla client pages). + The rows come from the cells (see *Mob Pit*, step 5). +- **Rooms.** `src/lib/pit/Rooms.graphql` (`zone_activities`, `@load + @live`) is the minimap: every room's status (active, dormant, asleep, + never ran) live as rooms wake, empty and sleep, with toasts when a room + goes to sleep or wakes from its snapshot. The debug overlay polls the + current room object's `GET /rooms/{room}/info` (node, isolate, + activation). `GAME_ZONES_URL` lists celld's entry points (comma + separated for a local fleet); the page rotates over them per connection — + entry points, not placement: celld forwards each socket to the room's + node. - **API identity.** `mobpit-service` trusts no identity header (`IdentityMode::OidcBearer`). Auth.js's `session` callback puts a Bearer in `session.accessToken`: an ES256 JWT this server signs for the session's @@ -329,12 +342,17 @@ and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: `./run.sh smoke` runs `check:games`, then `distributed build server/` (the GraphQL client, `svelte-check`, `vite build` with the game codegen and WASM) and `tests/pit.test.mjs`: the production build (`node build`) against the -real zone server and `mobpit-service` (on the Redis in `REDIS_URL`, with a -per-run key prefix) in headless Chromium — sign-in, ticket -rejection paths, an unauthenticated GraphQL request refused, join, keyboard -movement, auto-hunt until a kill settles, the killer's stats and a second -signed-in player's leaderboard updating live without a reload, the -server-rendered numbers after a reload, navigation away, no page errors. +rooms on a 3-node celld fleet (docker compose + RustFS; a single `celld +dev` when docker or the aws CLI is missing), the services and +`mobpit-service` (on the Redis in `REDIS_URL`, with a per-run key prefix) +in headless Chromium — sign-in, ticket rejection paths, an +unauthenticated GraphQL request refused, join, keyboard movement, auto-hunt +until a kill settles, the killer's stats and a second signed-in player's +leaderboard updating live without a reload, both players walking through +doorways (one into a room on another node, by the overlay), the room they +left asleep on the minimap and restored on return, another kill settled +live, the server-rendered numbers after a reload, navigation away, no page +errors. ## Architecture @@ -389,40 +407,44 @@ What the TypeScript still restates, and why it stays: ``` examples/game/ run.sh one-command dev + smoke runner - cells/ crate `game-demo-cells` (own workspace, wasm32, worker-build): the - demos' celld cells — Crawler + presence (Mob Pit), gem pouch + - content log (The Freeze), zone lease + zone state (multi-host - demos), and the `game-demo-outbox` Queue consumer (relay) + scripts/round-robin.mjs dev TCP balancer in front of a local celld fleet (`run.sh mobpit --fleet`) + cells/ crate `game-demo-cells` (own workspace, wasm32, worker-build): Mob + Pit's rooms (`PitRoom`, one Durable Object per room) and the demos' + celld cells — Crawler + presence + zone activity (Mob Pit), gem + pouch + content log (The Freeze), zone lease + zone state + (multi-host demos), and the `game-demo-outbox` Queue consumer (relay) server/ Rust crate `game_demos`, its own cargo workspace (with sim/, - crawler/, service/); also the `distributed build|dev` project + crawler/, rooms/, service/); also the `distributed build|dev` project crawler/ crate `mobpit_crawler`: Crawler aggregate, RecordKill (XP rules), `CrawlerState` domain event (shared by the zone and the service) - service/ crate `mobpit_service` (bin `mobpit-service`): player_stats read - model, modeled projector from the bus, the Redis boards as GraphQL - board roots (@live from bus notices), GraphQL @live (Bearer only), - tests/projection.rs (cells → bus → table → live push, board push - over bus notices, no mutations, 401) + rooms/ crate `mobpit_rooms` (wasm32-portable): the Mob Pit room zone + (combat, doorways, hibernation) and its `CelldZone` hooks + service/ crate `mobpit_service` (bin `mobpit-service`): player_stats and + zone_activities read models, projectors from the bus, the Redis + boards as GraphQL board roots (@live from bus notices), GraphQL + @live (Bearer only), tests/projection.rs (cells → bus → table → + live push, board push over bus notices, no mutations, 401) sim/ crate `game_sim`: the `#[predict]` steps (arena.rs movement + floor, - mobpit.rs, ripple.rs inputs + steps, grid.rs world + walker step + - `#[game_pure]` grid_cell / grid_sample); compiled natively for the + mobpit.rs (+ `#[game_pure]` pit_room, doorways), ripple.rs inputs + steps, + grid.rs world + walker step + `#[game_pure]` grid_cell / grid_sample); compiled natively for the server and to wasm32 for the browser src/main.rs `game-demos ` subcommands (+ hidden `predict-trace`) src/predict_trace.rs native traces of every step (recorded inputs) and pure function (arguments) src/common/ shared: Demo trait, run/spawn, dev routes, guests; celld.rs (CelldOptions: celld URL + internal secret), relay.rs (authenticated /internal/celld-queue/relay → SqliteBus), - zone_status.rs (zone lease/snapshot status read model) + zone_status.rs (zone lease/snapshot/activity status read model) src/devcelld.rs `celld dev` harness for tests (own project, port, process tree) src/cluster.rs multi-host demos: --host-id/--peer, directory, ownership, snapshots src/arena/ Arena zone (`movement` = `game_sim::arena`) - src/mobpit/ Mob Pit zone, fact sink to the Crawler/presence cells, crawler - directory projection, leaderboards, dev routes + src/mobpit/ Mob Pit services (the rooms run on celld): dev room tickets, relay, + crawler directory and room status projections, leaderboards src/gallery/ Shooting Gallery zone (targets, rewind hit tests) src/freeze/ The Freeze zone, content packs, two-host world + admin routes src/grid/ Endless Grid zone (`world` = `game_sim::grid`), dev map routes src/ripple/ Boss Ripple tiers and placement tests/ arena_smoke.rs (bots, dev routes), - mobpit_smoke.rs (facts, slow storage, kill -9 recovery, boards), + mobpit_smoke.rs (facts, rooms on celld, celld crash, boards, presence), gallery_smoke.rs, freeze_smoke.rs (binary content packs), grid_smoke.rs, ripple_smoke.rs (+ support/ for two in-process hosts) sveltekit/ SvelteKit app over the Mob Pit server + mobpit-service (see SvelteKit above) @@ -467,161 +489,181 @@ The server crate is excluded from the root workspace (`Cargo.toml` ## Mob Pit ```sh -examples/game/run.sh mobpit # → http://localhost:5173/mobpit/ -examples/game/run.sh leaderboard # same server → http://localhost:5173/leaderboard/ +examples/game/run.sh mobpit # → http://localhost:5173/mobpit/ (rooms on `celld dev`) +examples/game/run.sh mobpit --fleet # same on a local 3-node celld (docker compose + RustFS) +examples/game/run.sh leaderboard # same, board page → http://localhost:5173/leaderboard/ ``` -Controls: WASD / stick move (the Arena's movement, predicted by `pit_step`, same floor and -pillars), **Space** attack (120° cone, 2.2 m), **E** dash, **L** slow -storage. Mobs wander near home, aggro within 7 m, chase, stop and show a red -**telegraph ring** that fills for 0.8 s, then hit everyone inside it. Grunts -(green, 10 XP), brutes (purple, 25 XP) and the elite (red, 60 XP) respawn 4 s -after dying. Level `n` starts at `40·(n−1)²` XP. - -**One kill, end to end** (`server/src/mobpit/`): - -1. `zone.rs`, tick N: the mob's HP reaches 0. The zone builds the cell - command's own input, `RecordKillInput { mob, tier, name }` — a - `#[derive(FactCommand)]` bound to `RECORD_KILL` — as a typed fact with - `ctx.fact_with_key(&input, &[m{id}, gen])`, emits it with - `ctx.emit_fact_for(killer, fact)` and broadcasts a typed `kill` event. - `emit_fact_for` only enqueues: the host binds the killer's **verified - ticket principal**; the zone never awaits storage. Backpressure (256 - unsettled facts) parks facts in a backlog retried next tick. -2. The zone's fact worker (`GameServer::with_fact_sink`, retrying until - settled) runs `store::MobPitFactSink`: optional artificial latency, then - `distributed_game::celld::CelldFactSink` posts the fact over the - internal path (`DISTRIBUTED_INTERNAL_SECRET`, principal and `FactStamp` - as trusted headers) to the killer's own celld cell, `crawler/{principal}` - (`cells/`, a workers-rs Durable Object per player). The command id is - `sha256(service ‖ fact key)`, so the cell's command ledger turns any repeat - of the key into a replay of the stored receipt. Nothing else is written - per kill: there is no index cell every player passes through (celld - cannot enumerate cells, and such a cell would be a global hot spot); the - crawler list is a read model (step 5). celld down: the fact stays queued - in the zone (backpressure at capacity), never in a local store. -3. `crawler/src/lib.rs`: `crawler.record_kill` (a `#[command]` method) takes - the principal from the trusted session (`x-user-id`) and **decides the XP - from the tier**; the input has no identity or XP field and unknown fields - are rejected. Event, ledger receipt, snapshot and outbox row commit in one - Durable Object SQLite transaction. -4. Tick N+k: `Zone::on_fact_settled` sends the killer `saved` (with - `waitedMs`) and `profile`; the popup turns from "saving…" to - "+XP saved ✓". The sink also hands the kill to the leaderboard - (fire-and-forget). -5. The read model (`crawler/`, `service/`): `crawler.mob_killed` is a - **domain event** — the same transaction writes the post-kill - `CrawlerState` (principal, name, XP, kills, level) to the cell's outbox - (`repo.publish_events()`). celld drains the outbox to the `game-demo-outbox` - Queue; the cells worker's Queue consumer relays each message to the zone - server's `/internal/celld-queue/relay` (secret-authenticated), which - publishes it onto the `SqliteBus` at `--projection-bus` (default - `sqlite:{data-dir}/bus.db`). A crash between commit and drain, or a relay - outage, only delays it (the cell's alarm re-drains; the Queue retries). - `mobpit-service` consumes the bus with a - modeled projector (`projection!` → `upsert_player_stats`, defined in - `crawler/src/stats.rs`), checkpointed and deduplicated in its own SQLite - database, and serves `player_stats` over GraphQL with `@live`. The service - has no commands: nothing on that side can name a player or an amount. - The zone server projects the same events into its own copy - (`mobpit/directory.rs`, bus consumer `mobpit-zone`, - `{data-dir}/read-models.db`): `/dev/mobpit/crawlers` and the leaderboard - rebuild list crawlers from it (eventually consistent; the rebuild then - reads each listed crawler's kill history from its cell). - -The read path: on join the zone asks a background task to load the player's -cell and sends `profile` (`source: "cell"`) a few ticks later. - -**Slow storage.** The *Slow storage* toggle (`POST /dev/mobpit/storage -{latencyMs: 1500}`) delays every commit by 1.5 s. Facts commit in order per -zone, so pending climbs when you kill faster than one per 1.5 s; the top-right -graph shows pending (orange) rising and settling while the tick interval -(blue, `pit` entity `tickMs`/`tickMaxMs`) stays at ~33 ms and the HUD's tick -work stays in microseconds. This is the fix for a world that froze while -awaiting durable commits: the tick only enqueues. - -**Crash recovery** (`kill -9`): - -```sh -examples/game/run.sh mobpit # play, get some kills, toggle slow storage, kill more -kill -9 $(lsof -t -iTCP:2567 -sTCP:LISTEN) # in another terminal, mid-fight -examples/game/run.sh mobpit # restart (same .data/mobpit), reload the page -``` - -The ticket secret is persisted in the data dir (`dev-ticket-secret`), so the -page's guest token still maps to the same principal, and the profile comes -back from the cell. `run.sh` keeps celld's state in `.data/mobpit/celld` -(`.celld/dev`), so celld itself can be killed and restarted too. What -survives: - -- **Settled facts** (you saw "saved ✓"): durable. celld acknowledged the - commit only after the cell's SQLite transaction (event + ledger receipt) - was durable. -- **Facts in flight** at the crash: each is either fully committed (the - transaction finished; you just never saw the settlement) or not at all — - never half-applied, never twice. The zone's - in-memory state (mob HP, positions, the fact queue) dies with the process, - so an uncommitted kill is lost like any other unsaved world state; there is - no zone-side outbox (see `distributed_game` README, *Facts on dispose*). -- **Retries and replays** of the same key (the worker's retry after a - timeout, or a re-emitted fact) replay the stored receipt through the - ledger: `replayed: true`, nothing credited twice. Fact keys - (`crawler.record_kill:{zone}:{boot_epoch}:m{id}:{gen}`) contain the activation's boot - epoch (`Ctx::boot_epoch`, grows across restarts) because zone ticks and - mob generations restart at 0. - -`tests/mobpit_smoke.rs` proves it with the real binary on its own -`celld dev`: bots fight, storage is made slow, the server gets `SIGKILL` with -facts in flight, then celld is killed (`SIGKILL` of its process tree) and -restarted on its state, and the server restarts; then `settled ≤ durable -kills ≤ settled + in flight`, the board (rebuilt at boot) equals the cells, -and re-committing a pre-crash fact (same key and input) returns the original -receipt with `replayed: true` and unchanged kills. - -**celld.** Every demo with durable state runs its cells on celld: +Mob Pit is a near-infinite grid of **rooms**: room `(x, y)` is zone +`pit..`, and **every room is a celld Durable Object** (the cells +worker declares `PitRoom` with `distributed_game::celld_zone_object!`; the +zone code is `server/rooms`, `mobpit_rooms`). celld decides where a room +runs — the node that first touched it, forwarded from any other node, +evicted when idle, moved when idle, recovered elsewhere if its node +crashes. The game never names a node; the debug overlay only *shows* the +one celld chose. + +Controls: WASD / stick move (the Arena's movement, predicted by `pit_step`, +same floor and pillars), **Space** attack (120° cone, 2.2 m), **E** dash, +**O** debug overlay. Walk through a **doorway** — the glowing gap in the +middle of each wall (go beside the pillar on its axis) — into the +neighbouring room. The rooms are **one continuous world**: room `(x, y)` +sits at world `(x, y) × 40 m` (they share walls), the rooms around yours +are drawn from their seeded layout, so you see into the next room, and +going through a doorway just carries on walking — no scene swap, no jump. Mobs wander near home, aggro within 7 m, chase, stop and +show a red **telegraph ring** that fills for 0.8 s, then hit everyone +inside it. Grunts (green, 10 XP), brutes (purple, 25 XP) and elites (red, +60 XP) respawn 4 s after dying. Level `n` starts at `40·(n−1)²` XP. + +**Coming back.** The same browser is the same player (the dev guest token +is kept in `localStorage`; `?player=

Mob Pit leaderboard

- Kill facts committed to the Crawler cells are projected into two sorted-set boards (fire-and-forget, - idempotent apply keys). The top 10 is pushed over the zone connection (observer); your rank is read on each push. + Kills settle in the Crawler cells next to the rooms on celld; their events are projected into the crawler list + and from there into two sorted-set boards (idempotent `max` updates). The top 10 refreshes live; your rank is + read on each change. Play the pit ↗

@@ -24,7 +25,7 @@

Mob Pit leaderboard

You

—
-

Live pit

+

Entrance room (observer)

diff --git a/examples/game/client/src/generated/mobpit/constants.ts b/examples/game/client/src/generated/mobpit/constants.ts index 0e96cb0ab..2b523df3e 100644 --- a/examples/game/client/src/generated/mobpit/constants.ts +++ b/examples/game/client/src/generated/mobpit/constants.ts @@ -1,7 +1,7 @@ // @generated by `distributed game-client` from game surface `mobpit` (schema v3, protocol v1). // Constants: static data defined once in Rust (`GameSurface::constant`). // Do not edit: change the Rust definitions and regenerate. -import type { PitRules } from './events.ts'; +import type { PitRules, PitWorld } from './events.ts'; /** `pit_rules`, as the client decodes it. */ export const PIT_RULES: PitRules = { @@ -15,3 +15,15 @@ export const PIT_RULES: PitRules = { attackHalfAngle: 1.0471975803375244, swingFrames: 6 }; + +/** `pit_world`, as the client decodes it. */ +export const PIT_WORLD: PitWorld = { + defaultSeed: 186474221, + half: 20, + doorHalf: 2.5, + roomSize: 40, + passage: 2, + hysteresis: 0.5, + maxRoom: 1000000, + biomes: ['highlands', 'steppe', 'forest', 'meadow'] +}; diff --git a/examples/game/client/src/generated/mobpit/events.ts b/examples/game/client/src/generated/mobpit/events.ts index c748ce269..b98152277 100644 --- a/examples/game/client/src/generated/mobpit/events.ts +++ b/examples/game/client/src/generated/mobpit/events.ts @@ -3,124 +3,6 @@ // Do not edit: change the Rust definitions and regenerate. import { decodeGameEvent, decodePayload, decodeValue, encodePayload, encodeValue, onGameEvent, type DecodedGameEvent, type GameEventCodecs, type GameEventSource, type PayloadLayout, type WireType } from '@hops-ops/distributed/game'; -/** Record `BackgroundStats` (`#[derive(GameRecord)]`). */ -export interface BackgroundStats { - readonly enqueued: number; - readonly applied: number; - readonly dropped: number; - readonly failed: number; -} - -const BACKGROUND_STATS_TYPE: WireType = { - type: 'record', - fields: [ - ['enqueued', { type: 'u64' }], - ['applied', { type: 'u64' }], - ['dropped', { type: 'u64' }], - ['failed', { type: 'u64' }], - ] -}; - -/** Record `BoardRow` (`#[derive(GameRecord)]`). */ -export interface BoardRow { - readonly rank: number; - readonly member: string; - readonly name: string; - readonly score: number; -} - -const BOARD_ROW_TYPE: WireType = { - type: 'record', - fields: [ - ['rank', { type: 'u32' }], - ['member', { type: 'short_str' }], - ['name', { type: 'short_str' }], - ['score', { type: 'f64' }], - ] -}; - -/** Record `Scores` (`#[derive(GameRecord)]`). */ -export interface Scores { - readonly kills: number; - readonly xp: number; -} - -const SCORES_TYPE: WireType = { - type: 'record', - fields: [ - ['kills', { type: 'f64' }], - ['xp', { type: 'f64' }], - ] -}; - -/** Record `Mismatch` (`#[derive(GameRecord)]`). */ -export interface Mismatch { - readonly member: string; - readonly cell: Scores; - readonly board: Scores; -} - -const MISMATCH_TYPE: WireType = { - type: 'record', - fields: [ - ['member', { type: 'short_str' }], - ['cell', SCORES_TYPE], - ['board', SCORES_TYPE], - ] -}; - -/** Record `RebuildReport` (`#[derive(GameRecord)]`). */ -export interface RebuildReport { - readonly backend: string; - readonly crawlers: number; - readonly events: number; - readonly applied: number; - readonly duplicates: number; - readonly identicalTop: boolean; - readonly matchesCells: boolean; - readonly mismatches: readonly Mismatch[]; - readonly tookMs: number; -} - -const REBUILD_REPORT_TYPE: WireType = { - type: 'record', - fields: [ - ['backend', { type: 'short_str' }], - ['crawlers', { type: 'u32' }], - ['events', { type: 'u32' }], - ['applied', { type: 'u32' }], - ['duplicates', { type: 'u32' }], - ['identicalTop', { type: 'bool' }], - ['matchesCells', { type: 'bool' }], - ['mismatches', { type: 'list', item: MISMATCH_TYPE }], - ['tookMs', { type: 'f64' }], - ] -}; - -/** Record `BoardSnapshot` (`#[derive(GameRecord)]`). */ -export interface BoardSnapshot { - readonly version: number; - readonly backend: string; - readonly members: number; - readonly kills: readonly BoardRow[]; - readonly xp: readonly BoardRow[]; - readonly background: BackgroundStats; - readonly lastRebuild: RebuildReport | undefined; -} - -const BOARD_SNAPSHOT_TYPE: WireType = { - type: 'record', - fields: [ - ['version', { type: 'u64' }], - ['backend', { type: 'short_str' }], - ['members', { type: 'u64' }], - ['kills', { type: 'list', item: BOARD_ROW_TYPE }], - ['xp', { type: 'list', item: BOARD_ROW_TYPE }], - ['background', BACKGROUND_STATS_TYPE], - ['lastRebuild', { type: 'optional', item: REBUILD_REPORT_TYPE }], - ] -}; - /** Record `MobTier` (`#[derive(GameRecord)]`). */ export interface MobTier { readonly tier: number; @@ -138,6 +20,27 @@ export interface MoveState { readonly cd: number; } +/** Record `PitRoomContext` (`#[derive(GameRecord)]`). */ +export interface PitRoomContext { + readonly seed: number; + readonly x: number; + readonly y: number; +} + +/** Record `RoomMob` (`#[derive(GameRecord)]`). */ +export interface RoomMob { + readonly tier: number; + readonly x: number; + readonly y: number; +} + +/** Record `PitRoomLayout` (`#[derive(GameRecord)]`). */ +export interface PitRoomLayout { + readonly biome: number; + readonly danger: number; + readonly mobs: readonly RoomMob[]; +} + /** Record `PitRules` (`#[derive(GameRecord)]`). */ export interface PitRules { readonly tiers: readonly MobTier[]; @@ -147,6 +50,18 @@ export interface PitRules { readonly swingFrames: number; } +/** Record `PitWorld` (`#[derive(GameRecord)]`). */ +export interface PitWorld { + readonly defaultSeed: number; + readonly half: number; + readonly doorHalf: number; + readonly roomSize: number; + readonly passage: number; + readonly hysteresis: number; + readonly maxRoom: number; + readonly biomes: readonly string[]; +} + /** Record `Profile` (`#[derive(GameRecord)]`). */ export interface Profile { readonly xp: number; @@ -173,17 +88,25 @@ const PROFILE_TYPE: WireType = { export interface HelloEvent { readonly entity: number; readonly half: number; - readonly boot: string; readonly attackRange: number; readonly attackArcCos: number; + readonly roomX: number; + readonly roomY: number; + readonly seed: number; + readonly restored: boolean; + readonly roomKills: number; } const HELLO_LAYOUT: PayloadLayout = [ ['entity', { type: 'u32' }], ['half', { type: 'f32' }], - ['boot', { type: 'short_str' }], ['attackRange', { type: 'f32' }], ['attackArcCos', { type: 'f32' }], + ['roomX', { type: 'i32' }], + ['roomY', { type: 'i32' }], + ['seed', { type: 'u32' }], + ['restored', { type: 'bool' }], + ['roomKills', { type: 'u32' }], ]; /** Event `profile` (id 2). */ @@ -243,11 +166,6 @@ const SAVED_LAYOUT: PayloadLayout = [ ['attempts', { type: 'u32' }], ]; -/** Event `board` (id 5). */ -export type BoardEvent = BoardSnapshot; - -const BOARD_PAYLOAD: WireType = BOARD_SNAPSHOT_TYPE; - /** Event `died` (id 6). */ export interface DiedEvent { readonly respawnMs: number; @@ -257,13 +175,27 @@ const DIED_LAYOUT: PayloadLayout = [ ['respawnMs', { type: 'u32' }], ]; +/** Message `loaded` (id 1). */ +export interface LoadedMessage { + readonly principal: string; + readonly xp: number; + readonly level: number; + readonly kills: number; +} + +const MESSAGE_LOADED_LAYOUT: PayloadLayout = [ + ['principal', { type: 'short_str' }], + ['xp', { type: 'u64' }], + ['level', { type: 'u32' }], + ['kills', { type: 'u32' }], +]; + /** Event type ids by name. */ export const EVENT = { hello: 1, profile: 2, kill: 3, saved: 4, - board: 5, died: 6, } as const; @@ -273,7 +205,6 @@ export interface Events { profile: ProfileEvent; kill: KillEvent; saved: SavedEvent; - board: BoardEvent; died: DiedEvent; } @@ -303,12 +234,6 @@ export const EVENTS: GameEventCodecs = { encode: (value) => encodePayload(SAVED_LAYOUT, value), decode: (payload) => decodePayload(SAVED_LAYOUT, payload) as unknown as SavedEvent }, - board: { - id: 5, - name: 'board', - encode: (value) => encodeValue(BOARD_PAYLOAD, value), - decode: (payload) => decodeValue(BOARD_PAYLOAD, payload) as BoardEvent - }, died: { id: 6, name: 'died', @@ -333,3 +258,32 @@ export function onEvent( ): () => void { return onGameEvent(client, EVENTS, name, handler); } + +/** Zone inbox message type ids by name. */ +export const MESSAGE = { + loaded: 1, +} as const; + +/** Zone inbox message payload types by name. */ +export interface Messages { + loaded: LoadedMessage; +} + +/** Codecs of every zone inbox message. */ +export const MESSAGES: GameEventCodecs = { + loaded: { + id: 1, + name: 'loaded', + encode: (value) => encodePayload(MESSAGE_LOADED_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_LOADED_LAYOUT, payload) as unknown as LoadedMessage + }, +}; + +/** Encode a zone inbox message: `typeId` and `payload` of a `/admin/zone-message` body (base64url the payload). */ +export function encodeMessage( + name: K, + value: Messages[K] +): { readonly typeId: number; readonly payload: Uint8Array } { + const codec = MESSAGES[name]; + return { typeId: codec.id, payload: codec.encode(value) }; +} diff --git a/examples/game/client/src/generated/mobpit/index.ts b/examples/game/client/src/generated/mobpit/index.ts index 481102c6c..1bb1759f5 100644 --- a/examples/game/client/src/generated/mobpit/index.ts +++ b/examples/game/client/src/generated/mobpit/index.ts @@ -6,4 +6,5 @@ export * from './constants.ts'; export * from './events.ts'; export * from './input.ts'; export * from './predict.ts'; +export * from './pures.ts'; export * from './schema.ts'; diff --git a/examples/game/client/src/generated/mobpit/pures.ts b/examples/game/client/src/generated/mobpit/pures.ts new file mode 100644 index 000000000..e2d25d5c2 --- /dev/null +++ b/examples/game/client/src/generated/mobpit/pures.ts @@ -0,0 +1,60 @@ +// @generated by `distributed game-client` from game surface `mobpit` (schema v3, protocol v1). +// Pure functions: the Rust `#[game_pure]` fns compiled to WebAssembly (binary ABI). +// Do not edit: change the Rust definitions and regenerate. +import { createWasmPures, type WasmPures, type WasmSource, type WireType } from '@hops-ops/distributed/game'; + +import type { PitRoomContext, PitRoomLayout } from './events.ts'; + +const PIT_ROOM_CONTEXT_TYPE: WireType = { + type: 'record', + fields: [ + ['seed', { type: 'u32' }], + ['x', { type: 'i32' }], + ['y', { type: 'i32' }], + ] +}; + +const ROOM_MOB_TYPE: WireType = { + type: 'record', + fields: [ + ['tier', { type: 'u8' }], + ['x', { type: 'f32' }], + ['y', { type: 'f32' }], + ] +}; + +const PIT_ROOM_LAYOUT_TYPE: WireType = { + type: 'record', + fields: [ + ['biome', { type: 'u8' }], + ['danger', { type: 'u8' }], + ['mobs', { type: 'list', item: ROOM_MOB_TYPE }], + ] +}; + +/** The pure functions (`#[game_pure]`) of Cargo package `game_sim`: wrapper name → module export. */ +export const PURES = { + pitRoom: { export: 'pit_room', abi: 1 }, +} as const; + +/** The compiled module (`distributed game-wasm`; shared with the predicted step); bundlers emit it as an asset. */ +export const PURES_WASM_URL: URL = new URL('./wasm/mobpit.wasm', import.meta.url); + +/** Typed calls of the compiled functions (synchronous; they throw until `ready` resolves). */ +export interface MobpitPureFns { + /** `pit_room` (`dg_pure_pit_room`). */ + pitRoom(arg: PitRoomContext): PitRoomLayout; +} + +/** The `mobpit` pure functions plus `ready` / `loaded()` / `module()`. */ +export type MobpitPures = WasmPures; + +/** + * Load the compiled pure functions (default `PURES_WASM_URL`; a URL loaded by + * the predicted step too is instantiated once). + */ +export function createMobpitPures(source: WasmSource = PURES_WASM_URL): MobpitPures { + return createWasmPures(source, { + pitRoom: { exportName: 'pit_room', arg: PIT_ROOM_CONTEXT_TYPE, output: PIT_ROOM_LAYOUT_TYPE }, + }); +} diff --git a/examples/game/client/src/generated/mobpit/schema.ts b/examples/game/client/src/generated/mobpit/schema.ts index 3d9f5f004..cf619af1f 100644 --- a/examples/game/client/src/generated/mobpit/schema.ts +++ b/examples/game/client/src/generated/mobpit/schema.ts @@ -15,9 +15,9 @@ export type KindName = keyof typeof KIND; /** `player` (kind 1, priority 2). */ export interface PlayerFields { - /** `quant [-20, 19.999847] / 18 bits` */ + /** `quant [-22, 21.999832] / 18 bits` */ readonly x: number; - /** `quant [-20, 19.999847] / 18 bits` */ + /** `quant [-22, 21.999832] / 18 bits` */ readonly y: number; /** `quant [-16, 15.9921875] / 12 bits` */ readonly vx: number; @@ -131,18 +131,16 @@ export function pillarFields(entity: { readonly fields: EntityFields }): PillarF export interface PitFields { /** `u16` */ readonly pending: number; + /** `u32` */ + readonly committed: number; /** `u16` */ - readonly slowMs: number; + readonly backlog: number; /** `f32` */ readonly tickMs: number; /** `f32` */ readonly tickMaxMs: number; /** `u32` */ - readonly workUs: number; - /** `u32` */ - readonly committed: number; - /** `u16` */ - readonly backlog: number; + readonly kills: number; } /** True for a `pit` entity. */ @@ -155,20 +153,19 @@ export function pitFields(entity: { readonly fields: EntityFields }): PitFields const f = entity.fields; return { pending: fieldNumber(f, 'pending'), - slowMs: fieldNumber(f, 'slow_ms'), - tickMs: fieldNumber(f, 'tick_ms'), - tickMaxMs: fieldNumber(f, 'tick_max_ms'), - workUs: fieldNumber(f, 'work_us'), committed: fieldNumber(f, 'committed'), backlog: fieldNumber(f, 'backlog'), + tickMs: fieldNumber(f, 'tick_ms'), + tickMaxMs: fieldNumber(f, 'tick_max_ms'), + kills: fieldNumber(f, 'kills'), }; } /** Wire type of every replicated field, by kind and wire field name. */ export const FIELD_TYPES = { player: { - x: { type: 'quant', min: -20, max: 19.999847412109375, bits: 18 }, - y: { type: 'quant', min: -20, max: 19.999847412109375, bits: 18 }, + x: { type: 'quant', min: -22, max: 21.999832153320313, bits: 18 }, + y: { type: 'quant', min: -22, max: 21.999832153320313, bits: 18 }, vx: { type: 'quant', min: -16, max: 15.9921875, bits: 12 }, vy: { type: 'quant', min: -16, max: 15.9921875, bits: 12 }, dash: { type: 'u8' }, @@ -196,12 +193,11 @@ export const FIELD_TYPES = { }, pit: { pending: { type: 'u16' }, - slow_ms: { type: 'u16' }, - tick_ms: { type: 'f32' }, - tick_max_ms: { type: 'f32' }, - work_us: { type: 'u32' }, committed: { type: 'u32' }, backlog: { type: 'u16' }, + tick_ms: { type: 'f32' }, + tick_max_ms: { type: 'f32' }, + kills: { type: 'u32' }, }, } as const satisfies Record>; diff --git a/examples/game/client/src/generated/mobpit/surface.json b/examples/game/client/src/generated/mobpit/surface.json index 05ed196c4..4299b39e5 100644 --- a/examples/game/client/src/generated/mobpit/surface.json +++ b/examples/game/client/src/generated/mobpit/surface.json @@ -36,6 +36,28 @@ } ] } + }, + { + "name": "pit_world", + "type": { + "name": "PitWorld", + "type": "record" + }, + "value": { + "biomes": [ + "highlands", + "steppe", + "forest", + "meadow" + ], + "default_seed": 186474221, + "door_half": 2.5, + "half": 20.0, + "hysteresis": 0.5, + "max_room": 1000000, + "passage": 2.0, + "room_size": 40.0 + } } ], "events": [ @@ -57,21 +79,45 @@ } }, { - "name": "boot", + "name": "attack_range", "type": { - "type": "short_str" + "type": "f32" } }, { - "name": "attack_range", + "name": "attack_arc_cos", "type": { "type": "f32" } }, { - "name": "attack_arc_cos", + "name": "room_x", "type": { - "type": "f32" + "type": "i32" + } + }, + { + "name": "room_y", + "type": { + "type": "i32" + } + }, + { + "name": "seed", + "type": { + "type": "u32" + } + }, + { + "name": "restored", + "type": { + "type": "bool" + } + }, + { + "name": "room_kills", + "type": { + "type": "u32" } } ], @@ -224,17 +270,6 @@ "kind": "fields" } }, - { - "id": 5, - "name": "board", - "payload": { - "kind": "value", - "type": { - "name": "BoardSnapshot", - "type": "record" - } - } - }, { "id": 6, "name": "died", @@ -298,8 +333,8 @@ "priority": 1.0, "type": { "bits": 18, - "max": 19.999847, - "min": -20.0, + "max": 21.999832, + "min": -22.0, "type": "quant" } }, @@ -312,8 +347,8 @@ "priority": 1.0, "type": { "bits": 18, - "max": 19.999847, - "min": -20.0, + "max": 21.999832, + "min": -22.0, "type": "quant" } }, @@ -542,23 +577,23 @@ }, { "correction": null, - "name": "slow_ms", + "name": "committed", "priority": 1.0, "type": { - "type": "u16" + "type": "u32" } }, { "correction": null, - "name": "tick_ms", + "name": "backlog", "priority": 1.0, "type": { - "type": "f32" + "type": "u16" } }, { "correction": null, - "name": "tick_max_ms", + "name": "tick_ms", "priority": 1.0, "type": { "type": "f32" @@ -566,27 +601,19 @@ }, { "correction": null, - "name": "work_us", + "name": "tick_max_ms", "priority": 1.0, "type": { - "type": "u32" + "type": "f32" } }, { "correction": null, - "name": "committed", + "name": "kills", "priority": 1.0, "type": { "type": "u32" } - }, - { - "correction": null, - "name": "backlog", - "priority": 1.0, - "type": { - "type": "u16" - } } ], "id": 4, @@ -594,6 +621,41 @@ "priority": 1.0 } ], + "messages": [ + { + "id": 1, + "name": "loaded", + "payload": { + "fields": [ + { + "name": "principal", + "type": { + "type": "short_str" + } + }, + { + "name": "xp", + "type": { + "type": "u64" + } + }, + { + "name": "level", + "type": { + "type": "u32" + } + }, + { + "name": "kills", + "type": { + "type": "u32" + } + } + ], + "kind": "fields" + } + } + ], "name": "mobpit", "predict": { "abi": 1, @@ -603,213 +665,135 @@ "state": "MoveState" }, "protocolVersion": 1, - "records": [ + "pures": [ { - "fields": [ - { - "name": "enqueued", - "type": { - "type": "u64" - } - }, - { - "name": "applied", - "type": { - "type": "u64" - } - }, - { - "name": "dropped", - "type": { - "type": "u64" - } - }, - { - "name": "failed", - "type": { - "type": "u64" - } - } - ], - "name": "BackgroundStats" - }, + "abi": 1, + "arg": { + "name": "PitRoomContext", + "type": "record" + }, + "export": "pit_room", + "output": { + "name": "PitRoomLayout", + "type": "record" + }, + "package": "game_sim" + } + ], + "records": [ { "fields": [ { - "name": "rank", - "type": { - "type": "u32" - } - }, - { - "name": "member", + "name": "tier", "type": { - "type": "short_str" + "type": "u8" } }, { - "name": "name", + "name": "radius", "type": { - "type": "short_str" + "type": "f32" } }, { - "name": "score", + "name": "tele", "type": { - "type": "f64" + "type": "f32" } } ], - "name": "BoardRow" + "name": "MobTier" }, { "fields": [ { - "name": "version", - "type": { - "type": "u64" - } - }, - { - "name": "backend", + "name": "x", "type": { - "type": "short_str" + "type": "f32" } }, { - "name": "members", + "name": "y", "type": { - "type": "u64" + "type": "f32" } }, { - "name": "kills", + "name": "vx", "type": { - "item": { - "name": "BoardRow", - "type": "record" - }, - "type": "list" + "type": "f32" } }, { - "name": "xp", + "name": "vy", "type": { - "item": { - "name": "BoardRow", - "type": "record" - }, - "type": "list" + "type": "f32" } }, { - "name": "background", + "name": "dash", "type": { - "name": "BackgroundStats", - "type": "record" + "type": "u8" } }, { - "name": "last_rebuild", + "name": "cd", "type": { - "item": { - "name": "RebuildReport", - "type": "record" - }, - "type": "optional" + "type": "u8" } } ], - "name": "BoardSnapshot" + "name": "MoveState" }, { "fields": [ { - "name": "member", + "name": "seed", "type": { - "type": "short_str" + "type": "u32" } }, { - "name": "cell", + "name": "x", "type": { - "name": "Scores", - "type": "record" + "type": "i32" } }, { - "name": "board", + "name": "y", "type": { - "name": "Scores", - "type": "record" + "type": "i32" } } ], - "name": "Mismatch" + "name": "PitRoomContext" }, { "fields": [ { - "name": "tier", + "name": "biome", "type": { "type": "u8" } }, { - "name": "radius", - "type": { - "type": "f32" - } - }, - { - "name": "tele", - "type": { - "type": "f32" - } - } - ], - "name": "MobTier" - }, - { - "fields": [ - { - "name": "x", - "type": { - "type": "f32" - } - }, - { - "name": "y", - "type": { - "type": "f32" - } - }, - { - "name": "vx", - "type": { - "type": "f32" - } - }, - { - "name": "vy", - "type": { - "type": "f32" - } - }, - { - "name": "dash", + "name": "danger", "type": { "type": "u8" } }, { - "name": "cd", + "name": "mobs", "type": { - "type": "u8" + "item": { + "name": "RoomMob", + "type": "record" + }, + "type": "list" } } ], - "name": "MoveState" + "name": "PitRoomLayout" }, { "fields": [ @@ -856,123 +840,122 @@ { "fields": [ { - "name": "xp", - "type": { - "type": "u64" - } - }, - { - "name": "level", + "name": "default_seed", "type": { "type": "u32" } }, { - "name": "kills", + "name": "half", "type": { - "type": "u32" + "type": "f32" } }, { - "name": "level_xp", + "name": "door_half", "type": { - "type": "u64" + "type": "f32" } }, { - "name": "next_level_xp", + "name": "room_size", "type": { - "type": "u64" + "type": "f32" } }, { - "name": "source", + "name": "passage", "type": { - "type": "short_str" + "type": "f32" } - } - ], - "name": "Profile" - }, - { - "fields": [ + }, { - "name": "backend", + "name": "hysteresis", "type": { - "type": "short_str" + "type": "f32" } }, { - "name": "crawlers", + "name": "max_room", "type": { - "type": "u32" + "type": "i32" } }, { - "name": "events", + "name": "biomes", "type": { - "type": "u32" + "item": { + "type": "short_str" + }, + "type": "list" } - }, + } + ], + "name": "PitWorld" + }, + { + "fields": [ { - "name": "applied", + "name": "xp", "type": { - "type": "u32" + "type": "u64" } }, { - "name": "duplicates", + "name": "level", "type": { "type": "u32" } }, { - "name": "identical_top", + "name": "kills", "type": { - "type": "bool" + "type": "u32" } }, { - "name": "matches_cells", + "name": "level_xp", "type": { - "type": "bool" + "type": "u64" } }, { - "name": "mismatches", + "name": "next_level_xp", "type": { - "item": { - "name": "Mismatch", - "type": "record" - }, - "type": "list" + "type": "u64" } }, { - "name": "took_ms", + "name": "source", "type": { - "type": "f64" + "type": "short_str" } } ], - "name": "RebuildReport" + "name": "Profile" }, { "fields": [ { - "name": "kills", + "name": "tier", + "type": { + "type": "u8" + } + }, + { + "name": "x", "type": { - "type": "f64" + "type": "f32" } }, { - "name": "xp", + "name": "y", "type": { - "type": "f64" + "type": "f32" } } ], - "name": "Scores" + "name": "RoomMob" } ], "schemaVersion": 3 diff --git a/examples/game/client/src/leaderboard/main.ts b/examples/game/client/src/leaderboard/main.ts index 07e28d326..bf8be5d22 100644 --- a/examples/game/client/src/leaderboard/main.ts +++ b/examples/game/client/src/leaderboard/main.ts @@ -1,20 +1,29 @@ /** - * Leaderboard page: observes the pit zone (no avatar), renders the pushed - * top-10 snapshots, reads your rank and neighbours on every push, and - * triggers a rebuild from the cells' event history. + * Leaderboard page: renders the services' top-10 snapshots (a projection of + * the Crawler cells, polled), reads your rank and neighbours on every + * change, triggers a rebuild from the cells' event history, and observes + * the entrance room (no avatar) for a live mini map. */ import { ARENA_FLOOR } from '../generated/arena/constants.ts'; import { + createMobpitClient, isMob, isPillar, isPlayer, mobFields, pillarFields, - playerFields, + playerFields +} from '../generated/mobpit/index.ts'; +import { + createBoardWatcher, + DEFAULT_ZONE, + MobPitApi, + roomUrl, + TIERS, + type BoardPosition, type BoardRow, type BoardSnapshot -} from '../generated/mobpit/index.ts'; -import { createBoardWatcher, DEFAULT_ZONE, MobPitApi, TIERS, type BoardPosition } from '../mobpit/client.ts'; +} from '../mobpit/client.ts'; import { DevApi } from '../shared/dev-api.ts'; /** The Arena floor, generated from the simulation's constants (`game_sim::arena::arena_floor`). */ @@ -24,8 +33,13 @@ const params = new URLSearchParams(location.search); const zone = params.get('zone') ?? DEFAULT_ZONE; const api = new DevApi(); const pitApi = new MobPitApi(); -const watcher = createBoardWatcher({ api }); -const { client } = watcher; +const watcher = createBoardWatcher(pitApi); +const { client } = createMobpitClient({ + url: (room) => roomUrl(room), + ticket: api.ticketSource(), + pingIntervalMs: 1000, + prediction: false +}); const $ = (k: string) => document.querySelector(`[data-k="${k}"]`) as HTMLElement; const status = $('status'); @@ -74,7 +88,7 @@ async function render(board: BoardSnapshot): Promise { table($('xp'), board.xp, 'XP', me); const bg = board.background; status.textContent = - `push #${pushes} · version ${board.version} · store ${board.backend} · ${board.members} ranked` + + `update #${pushes} · version ${board.version} · store ${board.backend} · ${board.members} ranked` + ` · background updates: ${bg.applied} applied, ${bg.dropped} dropped, ${bg.failed} failed`; status.classList.add('flash'); setTimeout(() => status.classList.remove('flash'), 120); diff --git a/examples/game/client/src/mobpit/client.ts b/examples/game/client/src/mobpit/client.ts index 543ddfed3..91155023f 100644 --- a/examples/game/client/src/mobpit/client.ts +++ b/examples/game/client/src/mobpit/client.ts @@ -1,20 +1,81 @@ /** * Mob Pit networking without rendering (Node-testable): the generated * `createMobpitClient` (the own capsule predicted by `pit_step`, the Rust - * Arena movement compiled to WebAssembly), the zone's typed events folded into a small state object, and - * the `/dev/mobpit/*` routes. + * Arena movement compiled to WebAssembly), the room's typed events folded + * into a small state object, the rooms' generated contents + * (`createMobpitPures().pitRoom`, the Rust room generator as WebAssembly), + * and the `/dev/mobpit/*` routes. + * + * Every room (`pit..`) is a celld Durable Object reached at + * `/rooms/{room}`; celld forwards the socket to whichever node runs it. + * Walking through a doorway, the room hands the player to the next room + * (`Handoff`), and the SDK reconnects there with the transfer ticket. + * + * One continuous world: room `(x, y)` sits at world `(x, y) × ROOM_SIZE` + * and entity positions are room-local, so `worldOf` places anything from + * any room (the doorway arrival is the same world point, see + * `game_sim::mobpit`). * * Event flow of one kill: `kill` (broadcast, the tick the mob dies — the * fact is only *queued*) → `saved` (to the killer, when the durable cell - * commit settled; `waitedMs` shows the storage latency) → `profile`. + * commit settled) → `profile`. */ -import { GameClient, type GameSocketFactory } from '@hops-ops/distributed/game'; +import type { EntityFields, GameSocketFactory, RenderedEntity } from '@hops-ops/distributed/game'; import * as pit from '../generated/mobpit/index.ts'; -import { zoneUrl, type DevApi } from '../shared/dev-api.ts'; +import type { DevApi } from '../shared/dev-api.ts'; import { latest } from '../shared/latest.ts'; -export const DEFAULT_ZONE = 'pit-1'; +export const DEFAULT_ZONE = 'pit.0.0'; + +/** `pit..` → `[x, y]` (undefined for anything else). */ +export function parseRoom(zone: string): [number, number] | undefined { + const m = /^pit\.(-?\d+)\.(-?\d+)$/.exec(zone); + return m === null ? undefined : [Number(m[1]), Number(m[2])]; +} + +/** Room edge length (m): room `(x, y)` is centred at world `(x, y) × ROOM_SIZE`. */ +export const ROOM_SIZE = pit.PIT_WORLD.roomSize; + +/** World position (m) of a room-local point of `zone` (undefined for a non-room zone). */ +export function toWorldXY(zone: string, x: number, y: number): { x: number; y: number } | undefined { + const room = parseRoom(zone); + return room === undefined ? undefined : { x: room[0] * ROOM_SIZE + x, y: room[1] * ROOM_SIZE + y }; +} + +/** World position (m) of a rendered entity (every Mob Pit kind has local `x` / `y`). */ +export function worldOf(entity: RenderedEntity): { x: number; y: number } | undefined { + const { x, y } = entity.fields as { x?: unknown; y?: unknown }; + return typeof x === 'number' && typeof y === 'number' ? toWorldXY(entity.zoneId, x, y) : undefined; +} + +/** + * The own player's predicted fields re-expressed in the next room's frame + * (the same world point): lets the SDK keep predicting through a doorway + * handoff instead of holding the player still (`handoffFrame`). + */ +export function pitHandoffFrame(fields: EntityFields, from: string, to: string): EntityFields | undefined { + const a = parseRoom(from); + const b = parseRoom(to); + const { x, y } = fields as { x?: unknown; y?: unknown }; + if (a === undefined || b === undefined || typeof x !== 'number' || typeof y !== 'number') return undefined; + return { ...fields, x: x + (a[0] - b[0]) * ROOM_SIZE, y: y + (a[1] - b[1]) * ROOM_SIZE }; +} + +/** `[x, y]` → `pit..`. */ +export function roomZone(x: number, y: number): string { + return `pit.${x}.${y}`; +} + +/** + * WebSocket URL of a room. `base` is the http(s) origin rooms are reached + * at (celld, in Node tests); in the browser the page's own origin (Vite + * proxies `/rooms` to celld). + */ +export function roomUrl(zone: string, base?: string): string { + const origin = base ?? globalThis.location.origin; + return `${origin.replace(/^http/, 'ws')}/rooms/${encodeURIComponent(zone)}`; +} /** `state` names of the `mob` kind, by value (generated from the Rust `MOB_STATES`). */ export const MOB_STATE = pit.PIT_RULES.states; /** Presentation of each tier (client only). */ @@ -31,88 +92,206 @@ export const TIERS: Readonly [t.tier, { ...(TIER_LOOK[t.tier] ?? { name: `Tier ${t.tier}`, color: '#9ad16b' }), radius: t.radius, tele: t.tele }]) ); -/** `/dev/mobpit/leaderboard` neighbourhood (HTTP only, not a zone event). */ -export type BoardPosition = Readonly<{ rank: number | null; score: number | null; around: pit.BoardRow[] }>; +/** One leaderboard row (`/dev/mobpit/leaderboard`). */ +export type BoardRow = Readonly<{ rank: number; member: string; name: string; score: number }>; +/** The leaderboard snapshot the services keep (top-N, store, rebuilds). */ +export type BoardSnapshot = Readonly<{ + version: number; + backend: string; + members: number; + kills: BoardRow[]; + xp: BoardRow[]; + background: { enqueued: number; applied: number; dropped: number; failed: number }; + lastRebuild: RebuildReport | null; +}>; +export type RebuildReport = Readonly<{ + backend: string; + crawlers: number; + events: number; + applied: number; + duplicates: number; + identicalTop: boolean; + matchesCells: boolean; + mismatches: { member: string; cell: { kills: number; xp: number }; board: { kills: number; xp: number } }[]; + tookMs: number; +}>; +/** `/dev/mobpit/leaderboard` neighbourhood. */ +export type BoardPosition = Readonly<{ rank: number | null; score: number | null; around: BoardRow[] }>; + +/** A room as the `zone_status` read model knows it (`/dev/mobpit/rooms`). */ +export type RoomStatus = Readonly<{ + zone: string; + x: number; + y: number; + /** `active` · `dormant` · `hibernated` (unknown rooms are cold). */ + status: 'active' | 'dormant' | 'hibernated'; + players: number; + /** The current activation restored a snapshot. */ + restored: boolean; + activations: number; + /** celld node of the activation — a debug detail (empty when unknown). */ + node: string; + changedAtMs: number; +}>; + +/** A room object's debug JSON (`/rooms/{room}/info`). */ +export type RoomInfo = Readonly<{ + zone: string; + status: string; + awake: boolean; + activation: string; + restored: boolean; + wakes: number; + node: string; + isolate: string; + players: number; + tick: number; + hz: number; + lateMs: { p50: number; p99: number }; + droppedTicks: number; +}>; export type MobPitClient = pit.MobpitClient & Readonly<{ hello(): pit.HelloEvent | undefined; profile(): pit.Profile | undefined; - board(): pit.BoardSnapshot | undefined; + /** The room the player is in (authority zone). */ + room(): string | undefined; + /** The own player as rendered now (predicted; held through a handoff). */ + me(): RenderedEntity | undefined; + /** World position (m) of the own player as rendered now. */ + position(): { x: number; y: number } | undefined; /** Kills by me whose fact has not settled yet (key → event). */ unsaved: ReadonlyMap; + /** The rooms' generated contents (the Rust generator as WASM). */ + pures: ReturnType; /** Resolves on the first hello, once the predicted step is loaded. */ ready: Promise; }>; export type MobPitClientOptions = Readonly<{ api: DevApi; - base?: string; + /** http(s) origin of the rooms (celld); default the page's origin. */ + roomsBase?: string; socket?: GameSocketFactory; }>; export function createMobPitClient(options: MobPitClientOptions): MobPitClient { const typed = pit.createMobpitClient({ - url: (zone) => zoneUrl(zone, options.base), + url: (zone) => roomUrl(zone, options.roomsBase), ticket: options.api.ticketSource(), socket: options.socket, pingIntervalMs: 500, - inputDtMs: 1000 / 30 - // Prediction: the generated default, `pit_step` compiled to WASM. + inputDtMs: 1000 / 30, + // Prediction: the generated default, `pit_step` compiled to WASM, + // carried on through doorways in the next room's frame. + prediction: { handoffFrame: pitHandoffFrame } }); const hello = latest( (h) => typed.on('hello', h), (value) => typed.client.setControlledEntity(value.entity) ); const profile = latest((h) => typed.on('profile', h)); - const board = latest((h) => typed.on('board', h)); const unsaved = new Map(); typed.on('kill', (kill) => { if (kill.killer === hello.get()?.entity) unsaved.set(kill.key, kill); }); typed.on('saved', (saved) => unsaved.delete(saved.key)); + const pures = pit.createMobpitPures(); + // The own player: the predicted entity, or (mid-handoff) the SDK's + // bridge, the last predicted state in the room just left. + const me = () => { + const id = hello.get()?.entity; + const rendered = typed.client.render().filter((e) => pit.isPlayer(e)); + return rendered.find((e) => e.predicted) ?? rendered.find((e) => e.id === id && e.zoneId === typed.client.authority?.zoneId) ?? rendered.find((e) => e.id === id); + }; return { ...typed, hello: hello.get, profile: profile.get, - board: board.get, + room: () => typed.client.authority?.zoneId, + me, + position: () => { + const entity = me(); + return entity === undefined ? undefined : worldOf(entity); + }, unsaved, - ready: Promise.all([hello.ready, typed.prediction?.ready]).then(([h]) => h) + pures, + ready: Promise.all([hello.ready, typed.prediction?.ready, pures.ready]).then(([h]) => h) }; } -/** Observer-only client for the leaderboard page (no prediction, no inputs). */ -export function createBoardWatcher(options: MobPitClientOptions): Readonly<{ - client: GameClient; - board(): pit.BoardSnapshot | undefined; - on(handler: (value: pit.BoardSnapshot) => void): () => void; +/** Polls the services' leaderboard snapshot (the board is a projection of the cells). */ +export function createBoardWatcher(pitApi: MobPitApi, everyMs = 500): Readonly<{ + board(): BoardSnapshot | undefined; + on(handler: (value: BoardSnapshot) => void): () => void; + stop(): void; }> { - const typed = pit.createMobpitClient({ - url: (zone) => zoneUrl(zone, options.base), - ticket: options.api.ticketSource(), - socket: options.socket, - pingIntervalMs: 1000, - prediction: false - }); - const board = latest((h) => typed.on('board', h)); - return { client: typed.client, board: board.get, on: (handler) => typed.on('board', handler) }; + let current: BoardSnapshot | undefined; + const handlers = new Set<(value: BoardSnapshot) => void>(); + let stopped = false; + const poll = async () => { + while (!stopped) { + try { + const { snapshot } = await pitApi.leaderboard(); + if (current === undefined || snapshot.version !== current.version) { + current = snapshot; + for (const h of handlers) h(snapshot); + } + } catch { + // Services restarting: try again. + } + await new Promise((r) => setTimeout(r, everyMs)); + } + }; + void poll(); + return { + board: () => current, + on: (handler) => { + handlers.add(handler); + return () => handlers.delete(handler); + }, + stop: () => { + stopped = true; + } + }; } /** `/dev/mobpit/*` (dev only; none of these take an identity or a reward). */ export class MobPitApi { readonly base: string; + readonly roomsBase: string; - constructor(base = '') { + /** `base`: the services' origin; `roomsBase`: celld's (both default to the page's). */ + constructor(base = '', roomsBase = base) { this.base = base; + this.roomsBase = roomsBase; + } + + /** + * The room a returning guest starts in: its last room (presence cell, + * verified on the server from the guest token), else the entrance. + */ + async startRoom(guest: string | undefined): Promise { + const { zone } = await this.#json<{ zone: string; resumed: boolean }>('/dev/mobpit/start', guest === undefined ? {} : { guest }); + return parseRoom(zone) === undefined ? DEFAULT_ZONE : zone; + } + + /** Every room the zone status read model knows. */ + rooms(): Promise<{ rooms: RoomStatus[] }> { + return this.#json('/dev/mobpit/rooms'); } - storage(): Promise<{ latencyMs: number }> { - return this.#json('/dev/mobpit/storage'); + /** A room object's debug JSON, straight from the room on celld (does not wake it). */ + async roomInfo(zone: string): Promise { + const response = await fetch(`${this.roomsBase}/rooms/${encodeURIComponent(zone)}/info`); + if (!response.ok) throw new Error(`room info ${zone}: HTTP ${response.status}`); + return (await response.json()) as RoomInfo; } - /** Artificial latency in front of every durable commit (server-wide). */ - setStorage(latencyMs: number): Promise<{ latencyMs: number }> { - return this.#json('/dev/mobpit/storage', { latencyMs }); + /** The player's presence cell. */ + presence(principal: string): Promise<{ presence: { zone: string; status: string } | null }> { + return this.#json(`/dev/mobpit/presence?principal=${encodeURIComponent(principal)}`); } stats(): Promise> { @@ -123,12 +302,12 @@ export class MobPitApi { return this.#json('/dev/mobpit/crawlers'); } - leaderboard(member?: string): Promise<{ snapshot: pit.BoardSnapshot; me: { kills: BoardPosition; xp: BoardPosition } | null }> { + leaderboard(member?: string): Promise<{ snapshot: BoardSnapshot; me: { kills: BoardPosition; xp: BoardPosition } | null }> { const query = member === undefined ? '' : `?member=${encodeURIComponent(member)}`; return this.#json(`/dev/mobpit/leaderboard${query}`); } - rebuild(): Promise { + rebuild(): Promise { return this.#json('/dev/mobpit/leaderboard/rebuild', {}); } diff --git a/examples/game/client/src/mobpit/main.ts b/examples/game/client/src/mobpit/main.ts index 661a50e94..b8e8c8ecc 100644 --- a/examples/game/client/src/mobpit/main.ts +++ b/examples/game/client/src/mobpit/main.ts @@ -1,49 +1,76 @@ /** * Mob Pit page: Babylon scene over `createMobPitClient`. * - * What to look at: kill a mob (Space) and watch the grey "saving…" popup - * turn into "+XP saved ✓" when the durable cell commit settles. Toggle - * *Slow storage* (L): every commit now takes 1.5 s — the pending-facts graph - * climbs while the tick graph stays flat at ~33 ms, because the zone never - * waits for storage. Kill the server (`kill -9`), restart it, reload: your - * XP and kills come back from the cell. + * What to look at: rooms are celld Durable Objects on a near-infinite + * seeded grid, laid out as one continuous world (room `(x, y)` at + * `(x, y) × ROOM_SIZE`): the rooms around yours are drawn from the seeded + * layout (`pitRoom`, Rust as WASM: biome floor, walls, doorways, pillars), + * so you see into the next room before you go. Walk through a doorway (the + * gaps in the middle of each wall) and you are simply in the next room — + * the room hands you over at the same world point; nothing moves. A + * floating origin at your room keeps coordinates small. The minimap (bottom right) shows every known room + * live — active, dormant (just left), hibernated (asleep in storage) — from + * the `zone_status` read model. Come back to a room after it slept: it + * wakes from its snapshot with the mobs you left (toast). Kill a mob + * (Space): the grey "saving…" popup turns into "+XP saved ✓" when the kill + * settled in your Crawler cell. The debug overlay (O) shows which celld + * node and isolate run the room — celld decides, the game never does. */ import type { StandardMaterial } from '@babylonjs/core/Materials/standardMaterial'; +import { DynamicTexture } from '@babylonjs/core/Materials/Textures/dynamicTexture'; import { Color3 } from '@babylonjs/core/Maths/math.color'; +import { Vector3 } from '@babylonjs/core/Maths/math.vector'; import { Mesh } from '@babylonjs/core/Meshes/mesh'; import { MeshBuilder } from '@babylonjs/core/Meshes/meshBuilder'; +import { TransformNode } from '@babylonjs/core/Meshes/transformNode'; import { InputPacer, type RenderedEntity } from '@hops-ops/distributed/game'; import { ARENA_FLOOR } from '../generated/arena/constants.ts'; import { BUTTON, PIT_RULES, + PIT_WORLD, isMob, isPillar, isPit, mobFields, - pillarFields, pitFields, playerFields, - type BoardSnapshot, type PitFields, type PitInput } from '../generated/mobpit/index.ts'; import { DevApi } from '../shared/dev-api.ts'; import { Hud } from '../shared/hud.ts'; import { cameraRelative, InputSampler, toAxis } from '../shared/input.ts'; -import { createStage, material, toWorld } from '../shared/scene.ts'; -import { createMobPitClient, DEFAULT_ZONE, MobPitApi, TIERS } from './client.ts'; +import { createStage, material } from '../shared/scene.ts'; +import { + createBoardWatcher, + createMobPitClient, + DEFAULT_ZONE, + MobPitApi, + parseRoom, + ROOM_SIZE, + roomZone, + TIERS, + type BoardSnapshot, + type RoomInfo, + type RoomStatus +} from './client.ts'; /** The Arena floor, generated from the simulation's constants (`game_sim::arena::arena_floor`). */ -const { half: HALF, playerRadius: PLAYER_RADIUS } = ARENA_FLOOR; +const { half: HALF, playerRadius: PLAYER_RADIUS, pillars: PILLARS } = ARENA_FLOOR; +/** Rooms drawn around yours (2 → 5×5). */ +const ROOM_VIEW = 2; +/** Wall thickness and height (m), drawn just outside each room's floor. */ +const WALL_T = 0.3; +const WALL_H = 0.6; /** Swing cone, from the server's rules: reach to a mob's edge, plus the body. */ const { attackRange, attackHalfAngle: SWING_HALF, swingFrames: SWING_FRAMES } = PIT_RULES; const ATTACK_REACH = attackRange + PLAYER_RADIUS; -const SLOW_MS = 1500; const params = new URLSearchParams(location.search); -const zone = params.get('zone') ?? DEFAULT_ZONE; +/** `?zone=` forces a room; otherwise you start where you last were (`/dev/mobpit/start`). */ +const forcedZone = params.get('zone') ?? undefined; const storedName = (() => { try { return localStorage.getItem('game-demos.name') ?? undefined; @@ -64,19 +91,12 @@ const { scene } = stage; const hud = new Hud({ title: 'Mob Pit', - hint: 'WASD move · Space attack · E dash · L slow storage. Kills are durable facts; the tick never waits for them.', + hint: 'WASD move · Space attack · E dash · walk through a doorway to the next room · O debug overlay. Rooms run on celld; kills are durable facts.', client, devApi: api }); -let slow = false; -hud.toggle(`Slow storage (${SLOW_MS} ms per commit)`, false, (on) => { - slow = on; - pitApi.setStorage(on ? SLOW_MS : 0).catch((e: unknown) => hud.row('Storage', `failed: ${String(e)}`)); -}, 'l'); -pitApi - .storage() - .then((s) => (slow = s.latencyMs > 0)) - .catch(() => undefined); +let debug = params.get('debug') !== '0'; +hud.toggle('Debug overlay (celld node per room)', debug, (on) => (debug = on), 'o'); // ---- side panels: XP bar, durable facts graph, mini leaderboard ---- const xpPanel = div('pit-xp', document.body); @@ -103,6 +123,109 @@ function div(className: string, parent: HTMLElement): HTMLDivElement { return d; } +// ---- rooms: minimap from the zone status read model, toasts, debug ---- +const roomsPanel = div('pit-rooms', document.body); +roomsPanel.innerHTML = `

Rooms

+ +
■ active ■ dormant ■ asleep ■ never ran
+
`; +const roomsMap = roomsPanel.querySelector('canvas') as HTMLCanvasElement; +const roomsField = (k: string) => roomsPanel.querySelector(`[data-k="${k}"]`) as HTMLElement; +const toasts = div('pit-toasts', document.body); +const rooms = new Map(); +let roomInfo: RoomInfo | undefined; +const STATUS_COLOR: Record = { active: '#4caf50', dormant: '#f2c94c', hibernated: '#6c7aa0' }; +const BIOME_COLOR = ['#8a8f99', '#c9a86a', '#3f7d4e', '#7fbf6a']; + +function toast(text: string): void { + const t = div('pit-toast', toasts); + t.textContent = text; + setTimeout(() => t.remove(), 5000); +} + +async function pollRooms(): Promise { + for (;;) { + try { + const { rooms: rows } = await pitApi.rooms(); + for (const row of rows) { + const before = rooms.get(row.zone); + if (before !== undefined && before.status !== row.status) { + if (row.status === 'active' && row.restored && row.activations > before.activations) { + toast(`${row.zone} woke up from hibernation — its mobs are back`); + } else if (row.status === 'hibernated') { + toast(`${row.zone} went to sleep (snapshot in celld storage)`); + } + } + rooms.set(row.zone, row); + } + } catch { + // The services restarting: keep the last view. + } + await new Promise((r) => setTimeout(r, 500)); + } +} +void pollRooms(); + +async function pollRoomInfo(): Promise { + for (;;) { + const here = pit.room(); + if (debug && here !== undefined) roomInfo = await pitApi.roomInfo(here).catch(() => roomInfo); + await new Promise((r) => setTimeout(r, 1000)); + } +} +void pollRoomInfo(); + +function drawRooms(): void { + const ctx = roomsMap.getContext('2d'); + const here = parseRoom(pit.room() ?? DEFAULT_ZONE); + if (ctx === null || here === undefined) return; + const { width } = roomsMap; + const span = 7; + const cell = width / span; + ctx.clearRect(0, 0, width, width); + const seed = pit.hello()?.seed ?? PIT_WORLD.defaultSeed; + for (let j = 0; j < span; j++) { + for (let i = 0; i < span; i++) { + const x = here[0] + i - 3; + const y = here[1] + (span - 1 - j) - 3; + const zone = roomZone(x, y); + const row = rooms.get(zone); + let color = '#20252e'; + if (row !== undefined) color = STATUS_COLOR[row.status] ?? color; + else if (pit.pures.loaded()) color = `${BIOME_COLOR[pit.pures.pitRoom({ seed, x, y }).biome] ?? '#333'}55`; + ctx.fillStyle = color; + ctx.fillRect(i * cell + 2, j * cell + 2, cell - 4, cell - 4); + if (x === here[0] && y === here[1]) { + ctx.strokeStyle = '#ffffff'; + ctx.lineWidth = 2; + ctx.strokeRect(i * cell + 1, j * cell + 1, cell - 2, cell - 2); + } + ctx.fillStyle = '#0e1116'; + ctx.font = '10px monospace'; + if (row !== undefined && row.players > 0) ctx.fillText(`${row.players}p`, i * cell + 5, j * cell + 13); + if (debug && row !== undefined && row.node !== '') ctx.fillText(row.node.replace(/^node-/, ''), i * cell + 5, j * cell + cell - 6); + } + } +} + +function renderDebug(): void { + const el = roomsField('debug'); + roomsField('here').textContent = pit.room() ?? ''; + if (!debug) { + el.textContent = ''; + return; + } + const info = roomInfo; + if (info === undefined || info.zone !== pit.room()) { + el.textContent = 'room object: …'; + return; + } + const node = info.node === '' ? 'unknown (celld does not tell code its node; set CELLD_ZONE_NODES)' : info.node; + el.innerHTML = `celld object ${info.zone}
node ${node}
isolate ${info.isolate} · activation ${info.activation}` + + `
wakes ${info.wakes} · ${info.restored ? 'restored from snapshot' : 'fresh'} · ${info.players} players` + + `
tick ${info.hz} Hz · late p50 ${info.lateMs.p50.toFixed(1)} / p99 ${info.lateMs.p99.toFixed(1)} ms`; +} + // ---- materials and templates ---- const mats = { me: material(scene, 'me', '#6cc4ff'), @@ -121,10 +244,181 @@ mats.blade.emissiveColor = new Color3(0.6, 0.6, 0.7); const tierMats = new Map(); for (const [tier, t] of Object.entries(TIERS)) tierMats.set(Number(tier), material(scene, `mob-${tier}`, t.color)); +// ---- the world: rooms at fixed places, floating origin at your room ---- +// The stage's single floor and walls give way to one chunk per room. +for (const mesh of [...scene.meshes]) if (mesh.name === 'floor' || mesh.name === 'wall') mesh.dispose(); +type Room = readonly [number, number]; +let origin: Room = parseRoom(forcedZone ?? DEFAULT_ZONE) ?? [0, 0]; + +/** Scene position of a room-local point of room `room`. */ +function place(room: Room, x: number, y: number, height = 0): Vector3 { + return new Vector3((room[0] - origin[0]) * ROOM_SIZE + x, height, (room[1] - origin[1]) * ROOM_SIZE + y); +} + +/** Scene position of a room-local point of zone `zoneId` (the origin room if it is not a room). */ +function placeIn(zoneId: string, x: number, y: number, height = 0): Vector3 { + return place(parseRoom(zoneId) ?? origin, x, y, height); +} + +/** Room templates, instanced per room: floor per biome, wall segment, doorway gate, pillar. */ +const templates = (() => { + const px = 1024; + const texture = new DynamicTexture('room-grid', { width: px, height: px }, scene, true); + const ctx = texture.getContext() as CanvasRenderingContext2D; + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, px, px); + const lines = HALF * 2; + for (let i = 0; i <= lines; i += 1) { + const p = (i / lines) * px; + ctx.strokeStyle = i % 5 === 0 ? '#9aa0aa' : '#c8ccd2'; + ctx.lineWidth = i % 5 === 0 ? 3 : 1.5; + ctx.beginPath(); + ctx.moveTo(p, 0); + ctx.lineTo(p, px); + ctx.moveTo(0, p); + ctx.lineTo(px, p); + ctx.stroke(); + } + texture.update(); + const floors = BIOME_COLOR.map((color, biome) => { + const m = material(scene, `floor-${biome}`, color); + m.diffuseTexture = texture; + m.diffuseColor = Color3.FromHexString(color).scale(0.42); + m.specularColor = Color3.Black(); + const ground = MeshBuilder.CreateGround(`floor-${biome}`, { width: HALF * 2, height: HALF * 2 }, scene); + ground.material = m; + ground.isPickable = false; + ground.setEnabled(false); + return ground; + }); + const wallMat = material(scene, 'room-wall', '#4b5668'); + const segment = HALF - PIT_WORLD.doorHalf; + const wall = MeshBuilder.CreateBox('room-wall', { width: segment, depth: WALL_T, height: WALL_H }, scene); + wall.material = wallMat; + wall.isPickable = false; + wall.setEnabled(false); + const doorMat = material(scene, 'door', '#4dd0e1', 0.55); + doorMat.emissiveColor = Color3.FromHexString('#4dd0e1'); + const gate = MeshBuilder.CreateBox('door', { width: PIT_WORLD.doorHalf * 2, depth: 0.3, height: 0.08 }, scene); + gate.material = doorMat; + gate.isPickable = false; + gate.setEnabled(false); + const pillar = MeshBuilder.CreateCylinder('room-pillar', { diameter: 1, height: 2.4, tessellation: 24 }, scene); + pillar.material = mats.pillar; + pillar.isPickable = false; + pillar.setEnabled(false); + return { floors, wall, segment, gate, pillar }; +})(); + +type Chunk = { room: Room; node: TransformNode; biome: number | undefined; seed: number }; +const chunks = new Map(); + +/** One room's static geometry under a node at the room's centre. */ +function createChunk(room: Room): Chunk { + const key = roomZone(room[0], room[1]); + const node = new TransformNode(`room-${key}`, scene); + const seed = pit.hello()?.seed ?? PIT_WORLD.defaultSeed; + const biome = pit.pures.loaded() ? pit.pures.pitRoom({ seed, x: room[0], y: room[1] }).biome : undefined; + const add = (mesh: Mesh, name: string, x: number, y: number, height: number, turn = 0) => { + const instance = mesh.createInstance(`${key}-${name}`); + instance.parent = node; + instance.position.set(x, height, y); + instance.rotation.y = turn; + instance.isPickable = false; + return instance; + }; + add(templates.floors[biome ?? 0]!, 'floor', 0, 0, 0); + // Each wall: two segments either side of its doorway gap, just outside + // the floor (the neighbour's wall stands right behind it). + const off = PIT_WORLD.doorHalf + templates.segment / 2; + const out = HALF + WALL_T / 2; + for (const side of [-1, 1]) { + for (const along of [-1, 1]) { + add(templates.wall, `wall-h${side}${along}`, along * off, side * out, WALL_H / 2); + add(templates.wall, `wall-v${side}${along}`, side * out, along * off, WALL_H / 2, Math.PI / 2); + } + add(templates.gate, `door-h${side}`, 0, side * HALF, 0.04); + add(templates.gate, `door-v${side}`, side * HALF, 0, 0.04, Math.PI / 2); + } + PILLARS.forEach(({ x, y, r }, i) => { + const instance = add(templates.pillar, `pillar-${i}`, x, y, 1.2); + instance.scaling.set(r * 2, 1, r * 2); + }); + return { room, node, biome, seed }; +} + +/** Keep the rooms within `ROOM_VIEW` of the origin built and placed. */ +function syncChunks(): void { + const wanted = new Set(); + for (let dy = -ROOM_VIEW; dy <= ROOM_VIEW; dy += 1) { + for (let dx = -ROOM_VIEW; dx <= ROOM_VIEW; dx += 1) { + const room: Room = [origin[0] + dx, origin[1] + dy]; + const key = roomZone(room[0], room[1]); + wanted.add(key); + const chunk = chunks.get(key); + // Built before the world module loaded or the seed was known: rebuild. + const seed = pit.hello()?.seed ?? PIT_WORLD.defaultSeed; + if (chunk !== undefined && chunk.seed === seed && (chunk.biome !== undefined || !pit.pures.loaded())) continue; + chunk?.node.dispose(); + chunks.set(key, createChunk(room)); + } + } + for (const [key, chunk] of chunks) { + if (!wanted.has(key)) { + chunk.node.dispose(); + chunks.delete(key); + continue; + } + chunk.node.position.copyFrom(place(chunk.room, 0, 0)); + } +} + +/** Re-centre on `to` without a visible move: the camera shifts with the world. */ +function moveOrigin(to: Room): void { + if (to[0] === origin[0] && to[1] === origin[1]) return; + const shift = new Vector3((to[0] - origin[0]) * ROOM_SIZE, 0, (to[1] - origin[1]) * ROOM_SIZE); + origin = to; + stage.camera.setTarget(stage.camera.target.subtract(shift), false, false, true); + syncChunks(); +} +syncChunks(); +pit.pures.ready.then(syncChunks, () => undefined); +pit.on('hello', () => syncChunks()); + +// Doorway labels of your room: anchors at its doors (the origin room, so +// they never move), named after the room behind each. +const DOORS: { dx: number; dy: number }[] = [ + { dx: 1, dy: 0 }, + { dx: -1, dy: 0 }, + { dx: 0, dy: 1 }, + { dx: 0, dy: -1 } +]; +const doorAnchors = DOORS.map(({ dx, dy }, i) => { + const anchor = MeshBuilder.CreateBox(`door-anchor-${i}`, { size: 0.01 }, scene); + anchor.isVisible = false; + anchor.isPickable = false; + anchor.position.set(dx * HALF, 0.1, dy * HALF); + return anchor; +}); +let shownRoom = ''; + +function showRoom(zone: string): void { + const at = parseRoom(zone); + if (at === undefined || zone === shownRoom) return; + shownRoom = zone; + DOORS.forEach(({ dx, dy }, i) => { + stage.labels.set(`door-${i}`, doorAnchors[i]!, `→ ${roomZone(at[0] + dx, at[1] + dy)}`, 0.6, 'door'); + }); + const seed = pit.hello()?.seed ?? PIT_WORLD.defaultSeed; + if (pit.pures.loaded()) { + const room = pit.pures.pitRoom({ seed, x: at[0], y: at[1] }); + hud.row('Room', `${zone} · ${PIT_WORLD.biomes[room.biome] ?? '?'} · danger ${room.danger} · ${room.mobs.length} mobs`); + } +} + type PlayerVisual = { kind: 'player'; body: Mesh; blade: Mesh; arc: Mesh; nose: Mesh }; type MobVisual = { kind: 'mob'; body: Mesh; hpBack: Mesh; hpFill: Mesh; ring: Mesh; fill: Mesh; tier: number }; -type PillarVisual = { kind: 'pillar'; body: Mesh }; -type Visual = PlayerVisual | MobVisual | PillarVisual; +type Visual = PlayerVisual | MobVisual; const visuals = new Map(); function flatDisc(id: string, radius: number, mat: StandardMaterial, arc = 1): Mesh { @@ -171,11 +465,6 @@ function visualFor(entity: RenderedEntity, mine: boolean): Visual { if (visual !== undefined) return visual; if (isMob(entity)) { visual = createMob(entity); - } else if (isPillar(entity)) { - const { r } = pillarFields(entity); - const body = MeshBuilder.CreateCylinder(entity.key, { diameter: r * 2, height: 2.4, tessellation: 24 }, scene); - body.material = mats.pillar; - visual = { kind: 'pillar', body }; } else { visual = createPlayer(entity, mine); } @@ -188,37 +477,42 @@ function dispose(visual: Visual): void { } // ---- kill / saved popups (DOM labels anchored to invisible meshes) ---- -type Popup = { mesh: Mesh; until: number }; +type Popup = { mesh: Mesh; until: number; zone: string; x: number; y: number }; const popups = new Map(); let popupSeq = 0; -function popup(key: string, x: number, y: number, text: string, className: string, ttlMs: number): void { +/** The room a kill happened in, from its fact key (`…:pit..:…`). */ +function killRoom(key: string, fallback: string): string { + return /:(pit\.-?\d+\.-?\d+):/.exec(key)?.[1] ?? fallback; +} + +function popup(key: string, zoneId: string, x: number, y: number, text: string, className: string, ttlMs: number): void { let p = popups.get(key); if (p === undefined) { const mesh = MeshBuilder.CreateBox(`popup-${popupSeq++}`, { size: 0.01 }, scene); mesh.isVisible = false; - mesh.position.copyFrom(toWorld(x, y, 1.2)); - p = { mesh, until: 0 }; + p = { mesh, until: 0, zone: killRoom(key, zoneId), x, y }; + mesh.position.copyFrom(placeIn(p.zone, x, y, 1.2)); popups.set(key, p); } p.until = performance.now() + ttlMs; stage.labels.set(`popup:${key}`, p.mesh, text, 0.6, `pit-pop ${className}`); } -pit.on('kill', (kill) => { +pit.on('kill', (kill, zoneId) => { const mine = kill.killer === pit.hello()?.entity; const tier = TIERS[kill.tier]?.name ?? 'Mob'; - if (mine) popup(kill.key, kill.x, kill.y, `${tier} slain · saving…`, 'pending', 60_000); - else popup(kill.key, kill.x, kill.y, `${kill.killerName} slew a ${tier}`, 'other', 2000); + if (mine) popup(kill.key, zoneId, kill.x, kill.y, `${tier} slain · saving…`, 'pending', 60_000); + else popup(kill.key, zoneId, kill.x, kill.y, `${kill.killerName} slew a ${tier}`, 'other', 2000); }); let savedCount = 0; -pit.on('saved', (saved) => { +pit.on('saved', (saved, zoneId) => { if (saved.ok) { savedCount += 1; const waited = saved.waitedMs >= 200 ? ` (${(saved.waitedMs / 1000).toFixed(1)} s)` : ''; - popup(saved.key, saved.x, saved.y, `+${saved.xp} XP saved ✓${waited}`, 'saved', 2500); + popup(saved.key, zoneId, saved.x, saved.y, `+${saved.xp} XP saved ✓${waited}`, 'saved', 2500); } else { - popup(saved.key, saved.x, saved.y, `save failed: ${saved.error ?? '?'}`, 'failed', 4000); + popup(saved.key, zoneId, saved.x, saved.y, `save failed: ${saved.error ?? '?'}`, 'failed', 4000); } }); pit.on('died', (d) => { @@ -226,7 +520,11 @@ pit.on('died', (d) => { deathBanner.style.display = ''; setTimeout(() => (deathBanner.style.display = 'none'), d.respawnMs); }); -pit.on('board', (board) => renderBoard(board)); +pit.on('hello', (hello) => { + if (hello.restored) toast(`You entered ${roomZone(hello.roomX, hello.roomY)}: restored from hibernation (${hello.roomKills} kills here so far)`); +}); +const boards = createBoardWatcher(pitApi, 1000); +boards.on((board) => renderBoard(board)); function renderBoard(board: BoardSnapshot): void { const me = api.last?.principal; @@ -295,42 +593,47 @@ function sendCommandFrame(): void { // ---- frame ---- stage.onFrame((now, dtMs) => { commandFrames.update(now, sendCommandFrame); + const here = pit.room(); + const hereRoom = here === undefined ? undefined : parseRoom(here); + if (hereRoom !== undefined) moveOrigin(hereRoom); const hello = pit.hello(); const authority = client.authority; const rendered = client.render(); const seen = new Set(); - let me: { x: number; y: number; hp: number; dead: boolean } | undefined; + let me: { at: Vector3; hp: number; dead: boolean } | undefined; let stats: PitFields | undefined; for (const entity of rendered) { if (isPit(entity)) { - stats = pitFields(entity); + if (entity.zoneId === here) stats = pitFields(entity); continue; } - const mine = authority !== undefined && hello !== undefined && entity.zoneId === authority.zoneId && entity.id === hello.entity; + // Pillars are static: every room's are drawn with its chunk. + if (isPillar(entity)) continue; + // Yours: predicted, or the SDK's hold of it during a doorway handoff. + const mine = + entity.predicted || (authority !== undefined && hello !== undefined && entity.zoneId === authority.zoneId && entity.id === hello.entity); + const at = (x: number, y: number, height: number) => placeIn(entity.zoneId, x, y, height); const visual = visualFor(entity, mine); seen.add(entity.key); - const visible = !entity.stale || visual.kind === 'pillar'; - if (visual.kind === 'pillar') { - const { x, y } = pillarFields(entity); - visual.body.position.copyFrom(toWorld(x, y, 1.2)); - } else if (visual.kind === 'mob') { + const visible = !entity.stale; + if (visual.kind === 'mob') { const mob = mobFields(entity); const { x, y } = mob; const height = visual.tier === 3 ? 1 : visual.tier === 2 ? 0.75 : 0.45; - visual.body.position.copyFrom(toWorld(x, y, height)); + visual.body.position.copyFrom(at(x, y, height)); visual.body.rotation.y = -mob.face; const hp = mob.hp / 100; const top = height * 2 + 0.35; - visual.hpBack.position.copyFrom(toWorld(x, y, top)); - visual.hpFill.position.copyFrom(toWorld(x, y, top)); + visual.hpBack.position.copyFrom(at(x, y, top)); + visual.hpFill.position.copyFrom(at(x, y, top)); visual.hpFill.scaling.x = Math.max(0.001, hp); const tele = mob.tele / 255; const winding = mob.state === 2; visual.ring.setEnabled(visible && winding); visual.fill.setEnabled(visible && winding); if (winding) { - visual.ring.position.copyFrom(toWorld(x, y, 0.03)); - visual.fill.position.copyFrom(toWorld(x, y, 0.04)); + visual.ring.position.copyFrom(at(x, y, 0.03)); + visual.fill.position.copyFrom(at(x, y, 0.04)); visual.fill.scaling.setAll(Math.max(0.05, tele)); } visual.body.scaling.setAll(winding ? 1 + 0.15 * tele : 1); @@ -338,19 +641,19 @@ stage.onFrame((now, dtMs) => { } else { const player = playerFields(entity); const { x, y, dead, face, swing } = player; - visual.body.position.copyFrom(toWorld(x, y, 0.9)); + visual.body.position.copyFrom(at(x, y, 0.9)); visual.body.material = dead ? mats.dead : mine ? mats.me : mats.other; - visual.nose.position.copyFrom(toWorld(x + Math.cos(face) * 0.5, y + Math.sin(face) * 0.5, 1.3)); + visual.nose.position.copyFrom(at(x + Math.cos(face) * 0.5, y + Math.sin(face) * 0.5, 1.3)); visual.nose.rotation.y = -face; // Blade sweeps through the swing cone while `swing` counts down to 0. const sweep = swing > 0 ? ((SWING_FRAMES - swing) / SWING_FRAMES) * 2 * SWING_HALF - SWING_HALF : -Math.PI / 2.5; const a = face + sweep; - visual.blade.position.copyFrom(toWorld(x + Math.cos(a) * 0.8, y + Math.sin(a) * 0.8, 1.0)); + visual.blade.position.copyFrom(at(x + Math.cos(a) * 0.8, y + Math.sin(a) * 0.8, 1.0)); visual.blade.rotation.y = -a; visual.blade.setEnabled(visible && !dead); visual.arc.setEnabled(visible && swing > 0); if (swing > 0) { - visual.arc.position.copyFrom(toWorld(x, y, 0.05)); + visual.arc.position.copyFrom(at(x, y, 0.05)); // Disc arcs start at +x and run counter-clockwise; centre it on `face`. visual.arc.rotation.y = -(face - SWING_HALF); } @@ -358,7 +661,7 @@ stage.onFrame((now, dtMs) => { stage.labels.set(entity.key, visual.body, mine ? `${label} (you)` : label, 1.4, mine ? 'me' : ''); visual.body.setEnabled(visible); visual.nose.setEnabled(visible && !dead); - if (mine) me = { x, y, hp: player.hp, dead }; + if (mine) me = { at: visual.body.position, hp: player.hp, dead }; } } for (const [key, visual] of visuals) { @@ -368,27 +671,26 @@ stage.onFrame((now, dtMs) => { visuals.delete(key); } for (const [key, p] of popups) { + p.mesh.position.copyFrom(placeIn(p.zone, p.x, p.y, 1.2)); if (now < p.until) continue; p.mesh.dispose(); stage.labels.delete(`popup:${key}`); popups.delete(key); } - if (me !== undefined) stage.follow(me.x, me.y, dtMs); + if (me !== undefined) stage.follow(me.at.x, me.at.z, dtMs); // ---- HUD ---- hud.frame({ entities: rendered.length }); const pending = stats?.pending ?? 0; const tickMs = stats?.tickMs ?? 0; const tickMax = stats?.tickMaxMs ?? 0; - const slowMs = stats?.slowMs ?? 0; hud.row('You', api.last === undefined ? '—' : `${api.last.name} · ${api.last.principal}`); hud.row('HP', me === undefined ? '—' : me.dead ? 'dead' : String(me.hp)); hud.row('Facts pending (zone)', String(pending)); hud.row('Facts saved (zone)', String(stats?.committed ?? 0)); hud.row('My unsaved kills', String(pit.unsaved.size)); - hud.row('Storage latency', slowMs > 0 ? `${slowMs} ms (slow)` : 'normal'); - hud.row('Server tick interval', `${tickMs.toFixed(1)} ms avg · ${tickMax.toFixed(1)} max`); - hud.row('Tick work', `${stats?.workUs ?? 0} µs`); + hud.row('Kills in this room', String(stats?.kills ?? 0)); + hud.row('Room tick interval', `${tickMs.toFixed(1)} ms avg · ${tickMax.toFixed(1)} max`); const profile = pit.profile(); if (profile !== undefined) { const span = Math.max(1, profile.nextLevelXp - profile.levelXp); @@ -399,10 +701,11 @@ stage.onFrame((now, dtMs) => { } field('pending').textContent = String(pending); field('saved').textContent = String(savedCount); - field('note').textContent = slow || slowMs > 0 - ? `Storage is slow: each commit waits ${slowMs} ms. Pending climbs, the tick does not.` - : 'Storage is fast: facts settle within a tick or two.'; + field('note').textContent = 'Kills settle in your Crawler cell next to the room on celld; the tick never waits for them.'; + if (here !== undefined) showRoom(here); if (now - lastSampleAt > 250) { + drawRooms(); + renderDebug(); lastSampleAt = now; samples.push({ pending, tick: tickMs }); if (samples.length > 120) samples.shift(); @@ -415,6 +718,11 @@ declare global { mobpit?: unknown; } } -window.mobpit = { client, api, pitApi, pit, scene }; +window.mobpit = { client, api, pitApi, pit, scene, rooms, boards, camera: stage.camera, origin: () => origin }; -client.connect(zone).catch((error: unknown) => hud.row('Status', `failed: ${String(error)}`)); +// The same browser is the same player: back in the room you were last in. +(async () => { + const zone = forcedZone ?? (await pitApi.startRoom(api.guest).catch(() => DEFAULT_ZONE)); + origin = parseRoom(zone) ?? origin; + await client.connect(zone); +})().catch((error: unknown) => hud.row('Status', `failed: ${String(error)}`)); diff --git a/examples/game/client/src/mobpit/style.css b/examples/game/client/src/mobpit/style.css index 31836839c..bdec1b330 100644 --- a/examples/game/client/src/mobpit/style.css +++ b/examples/game/client/src/mobpit/style.css @@ -118,3 +118,61 @@ width: 220px; } } +/* Rooms minimap (zone status read model) and the debug overlay. */ +.pit-rooms { + position: fixed; + right: 12px; + bottom: 16px; + width: 234px; + padding: 10px 12px; + border-radius: 8px; + background: var(--panel); + box-sizing: border-box; +} +.pit-rooms h2 { + margin: 0 0 6px; + font-size: 13px; +} +.pit-rooms h2 small { + color: var(--muted); + font-weight: normal; +} +.pit-rooms .c-active { + color: #4caf50; +} +.pit-rooms .c-dormant { + color: #f2c94c; +} +.pit-rooms .c-hibernated { + color: #6c7aa0; +} +.pit-rooms .c-cold { + color: #556070; +} +.pit-debug { + margin-top: 6px; + font: 11px/1.4 monospace; + color: var(--muted); + word-break: break-word; +} +.pit-toasts { + position: fixed; + top: 12px; + left: 50%; + transform: translateX(-50%); + display: flex; + flex-direction: column; + gap: 6px; + pointer-events: none; + z-index: 10; +} +.pit-toast { + padding: 8px 14px; + border-radius: 6px; + background: #1f3a4d; + color: #e8f4ff; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.4); +} +.label.door { + color: #4dd0e1; +} diff --git a/examples/game/client/src/shared/dev-api.ts b/examples/game/client/src/shared/dev-api.ts index 64ed40f5f..283d903d2 100644 --- a/examples/game/client/src/shared/dev-api.ts +++ b/examples/game/client/src/shared/dev-api.ts @@ -4,8 +4,11 @@ * * A real game gets admission tickets from its authenticated lobby; the dev * route mints one for a server-generated guest. We keep the server-signed - * guest token per tab (sessionStorage when available): a reload stays the - * same player, and each tab is its own player. Works in browsers (relative URLs) and in Node tests (`base`). + * guest token per browser (localStorage when available): a reload, a new + * tab or a reopened browser is the same player (a second tab takes the + * player over). `?player=