feat(game)!: zones as celld objects, Mob Pit room grid

.github/workflows/integration-celld.yaml

@@ -42,6 +42,7 @@ jobs:
             tests/celld/worker -> tests/celld/worker/target
             tests/celld/alarm-worker -> tests/celld/alarm-worker/target
             tests/celld/fact-worker -> tests/celld/fact-worker/target
+            tests/celld/zone-worker -> tests/celld/zone-worker/target
           shared-key: celld-storage
       - uses: actions/setup-node@v4
         with:
@@ -82,6 +83,13 @@ jobs:
         run: |
           cargo test -p distributed_game --features celld,runtime --test celld_zone_state
           cargo test -p distributed_game --features celld,net,lease --test world
+      - name: Build the celld zone host test Worker
+        working-directory: tests/celld/zone-worker
+        run: worker-build --release
+      - name: Zones as celld objects (celld dev; 3-node fleet on RustFS, crash, facts exactly once)
+        env:
+          CELLD_ZONE_REQUIRED: "1"
+        run: cargo test -p distributed_game --features celld,net --test celld_zone
       - name: Retain storage proof results and logs
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/integration-game-demos.yaml

@@ -7,9 +7,11 @@ name: Game demos smoke
 # headless tests, and the SvelteKit demo end to end: `distributed build
 # examples/game/server` (GraphQL client, svelte-check, vite build), then
 # Playwright against the production build, a real Mob Pit server and the real
-# mobpit-service read model. Mob Pit's durable cells run on celld: the tests
-# start their own `celld dev` (CELLD_BIN, pinned in /.celld-version; no Docker) serving
-# examples/game/cells, built with worker-build.
+# mobpit-service read model. Mob Pit's rooms and durable cells run on celld:
+# the tests start their own `celld dev` (CELLD_BIN, pinned in
+# /.celld-version) serving examples/game/cells, built with worker-build; the
+# SvelteKit Playwright test runs the rooms on a 3-node fleet (RustFS via
+# docker compose and the aws CLI, both on the runner; required here).
 on:
   workflow_call:
 
@@ -64,6 +66,7 @@ jobs:
         env:
           # The SvelteKit leaderboard is the Redis board (GraphQL board roots).
           REDIS_URL: redis://127.0.0.1:6379
+          MOBPIT_FLEET_REQUIRED: "1"
         run: examples/game/run.sh smoke
       - name: Leaderboard conformance and GraphQL boards against real Redis
         env:

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 members = ["distributed_macros", "distributed_cli", "distributed_game", "distributed_game_macros"]
-exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/celld/fact-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"]
+exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/celld/fact-worker", "tests/celld/zone-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"]
 resolver = "2"
 
 [workspace.package]

distributed_game/Cargo.toml

@@ -16,22 +16,28 @@ default = []
 runtime = ["dep:tokio"]
 # Adapter that commits facts into a `distributed` AggregateCell.
 cell = ["dep:distributed", "dep:sha2", "dep:base64"]
-# `command_sink::CommandFactSink<H: CommandHost>`: facts dispatched as
-# commands through any command host (a local service over any repository,
-# or celld cells).
+# `command_sink::CommandFactSink` over `CelldCommandHost` /
+# `LocalCommandHost` (a local service over any repository, or celld cells
+# over HTTP). The sink itself is portable (`cell`).
 command-host = ["cell", "distributed/graphql"]
 # celld clients: `CelldFactSink` (the command sink over a remote-only
 # `CelldCommandHost`), `CellZoneOwnership`, `CellZoneStateStore`.
 celld = ["cell", "command-host", "dep:reqwest", "dep:uuid"]
 # celld worker glue (wasm32 Durable Objects): `world::zone_state::celld_worker`,
 # the zone-state HTTP surface over a cell and an R2 bucket binding.
 workers-rs = ["cell", "dep:worker", "distributed/workers-rs"]
+# Zones as celld Durable Objects (wasm32): `celld_zone::ZoneObject`,
+# `CelldObjectHost` (facts object to object), `celld_zone_object!`.
+celld-zone = ["cell", "ticket", "workers-rs", "dep:js-sys", "dep:wasm-bindgen", "dep:wasm-bindgen-futures"]
 # Per-zone Prometheus metrics (dependency-free text exposition).
 metrics = []
 # Load-test bots (library + `game-bots` binary).
 bots = ["net", "dep:tokio-tungstenite", "dep:futures-util", "tokio/rt-multi-thread"]
 # axum WebSocket zone server with signed-ticket admission (+ /metrics).
-net = ["runtime", "metrics", "dep:axum", "dep:hmac", "dep:sha2", "dep:base64", "tokio/net", "tokio/macros", "tokio/io-util"]
+# HMAC-signed admission tickets (`ticket`) without the axum server (the
+# celld zone host admits players with it).
+ticket = ["dep:hmac", "dep:sha2", "dep:base64"]
+net = ["runtime", "metrics", "ticket", "dep:axum", "dep:hmac", "dep:sha2", "dep:base64", "tokio/net", "tokio/macros", "tokio/io-util"]
 # `world::FencedLeaseOwnership`: zone ownership over `distributed::lease`.
 lease = ["dep:distributed"]
 
@@ -50,6 +56,9 @@ futures-util = { version = "0.3", default-features = false, features = ["sink",
 reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true }
 uuid = { version = "1", features = ["v7"], optional = true }
 worker = { version = "0.8", optional = true }
+js-sys = { version = "0.3", optional = true }
+wasm-bindgen = { version = "0.2", optional = true }
+wasm-bindgen-futures = { version = "0.4", optional = true }
 
 [dev-dependencies]
 base64 = "0.23"
@@ -118,6 +127,11 @@ name = "celld_facts"
 path = "tests/celld_facts/main.rs"
 required-features = ["celld", "runtime"]
 
+[[test]]
+name = "celld_zone"
+path = "tests/celld_zone/main.rs"
+required-features = ["celld", "net"]
+
 [[test]]
 name = "inbox"
 path = "tests/inbox/main.rs"

distributed_game/README.md

@@ -26,7 +26,8 @@ the browser, but the client SDK has no rendering dependency.
 - [Build a zone](#build-a-zone)
 - [Guides](#guides): game loop · input · what clients see · the client SDK ·
   generated clients · SvelteKit apps · client prediction (Rust → WebAssembly) · shared
-  logic and data (no TypeScript twins) · saving progress · scaling out ·
+  logic and data (no TypeScript twins) · saving progress · scaling out
+  (zones as celld objects, the native multi-host server) ·
   live updates · lag compensation · testing and operations · security
 - [Design decisions](#design-decisions)
 - [Status and limits](#status-and-limits)
@@ -100,7 +101,7 @@ each built to show one part of the engine:
 | Demo | Run | Shows |
 |---|---|---|
 | Arena | `examples/game/run.sh arena` | ticks, deltas, prediction on/off, interest radius vs. global entities, simulated latency |
-| Mob Pit | `run.sh mobpit` | combat whose rewards are saved without stalling; "slow storage"; `kill -9` crash recovery |
+| Mob Pit | `run.sh mobpit` (`--fleet` for several celld nodes) | rooms as celld objects joined by doorways, sleeping when empty and restored on return; combat whose rewards are saved exactly once without stalling |
 | Leaderboard | `run.sh leaderboard` | a live ranking fed from saved kills, rebuilt from history |
 | Shooting Gallery | `run.sh gallery` | lag compensation on/off under latency |
 | The Freeze | `run.sh freeze` | pausing every zone on two servers, updating content, resuming; safe-room reloads |
@@ -291,7 +292,7 @@ A zone is in one of three **tiers**:
   Idle forever (a city clock, say) or from `has_pending_work` to stay Hot.
 - **Dormant**: idle for `dormant_after` (30 s); no ticks. With
   hibernation on, a Dormant zone is saved and unloaded (see
-  [Scaling out](#scaling-out-many-zones-many-servers)).
+  [Scaling out](#scaling-out-zones-as-celld-objects)).
 
 `on_tier_changed` tells you about transitions. Timers
 (`ctx.set_timeout(after, value)`, `ctx.set_interval(every, value)`,
@@ -758,7 +759,9 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"),
 - Keys must be unique across restarts; `ctx.fact_key` includes the
   activation's boot epoch for you (see [Durable facts](#durable-facts)).
 - If the process crashes, each in-flight fact is either fully saved or
-  not saved at all. Mob Pit shows this with `kill -9`.
+  not saved at all. A zone hosted on celld (`celld_zone`) also keeps its
+  emitted facts in an outbox in object storage, so they are resent after
+  a crash: exactly once (Mob Pit's rooms).
 - **Player lifecycle facts.** With `HostConfig::lifecycle_facts = true`
   the host itself emits `lifecycle::PlayerJoined` (ticket, rejoin or
   handoff), `ConnectionLost` (grace started; dropped or told to reload),
@@ -786,7 +789,66 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"),
   B" sorts after "left A" even when it commits first or the two hosts'
   clocks disagree.
 
-### Scaling out: many zones, many servers
+### Scaling out: zones as celld objects
+
+On celld every zone is a Durable Object (feature `celld-zone`,
+[`celld_zone`](src/celld_zone/mod.rs)): one object per zone id, so a room
+*is* a cell. Game code implements `CelldZone` for its `Zone` and declares
+the class; nothing in the game or its API names a server.
+
+```rust,ignore
+impl CelldZone for MobPit {
+    fn object_config(zone: &str, env: &ObjectEnv) -> Result<PitConfig, String> { /* … */ }
+    fn fact_routes(sink: ObjectFactSink) -> ObjectFactSink {
+        sink.route(FactRoute::by_principal::<RecordKillInput>("crawler"))
+    }
+}
+distributed_game::celld_zone_object!(PitRoom, MobPit);
+
+// worker entry: /rooms/{zone} → the zone's object (WebSocket or …/info)
+if let Some(response) = forward_to_zone(req, &env, "PIT_ROOM", "/rooms/").await? { … }
+```
+
+**What celld handles:** where a zone runs (first touch; overflow to the
+least-loaded node with `CELLD_MAX_RESIDENT_CELLS`), forwarding a WebSocket
+from any node to the owner, one live instance per zone (fenced by its own
+epochs), evicting idle objects, moving idle ones to balance load, draining
+a stopping node and recovering a crashed node's objects elsewhere.
+
+**What the framework handles:** the WebSocket join (signed ticket, transfer
+ticket of a doorway handoff — single use across activations — or reconnect
+token), a self-correcting tick loop that runs only while someone is
+connected, the zone's snapshot in object storage (every
+`ZONE_PERSIST_MS` while occupied, and when it empties), dormancy and
+hibernation (`ZONE_HIBERNATE_AFTER_MS` later, by object alarm, so it also
+happens after an eviction), restore on the next join, facts to the cells
+of the same worker object to object (`CelldObjectHost` under the usual
+`CommandFactSink` routes; a durable outbox in the object's storage makes
+them exactly once across crashes), and the zone's lifecycle reports
+(`active` / `dormant` / `hibernated`) into its
+[`zone_activity`](src/world/zone_activity.rs) cell, which the
+`zone_status` read model projects for minimaps.
+
+**Limits.** A live zone never moves (only idle objects are rebalanced), so
+load spreads by first touch and node caps. Objects of one class share V8
+isolates: a slow zone delays its neighbours. A node crash takes about the
+node lease TTL (`CELLD_TTL_MS`, 10 s) to recover and loses at most the
+snapshot interval of zone state; facts survive in the outbox. A socket
+that outlives its activation closes with 1012 and the SDK rejoins with a
+fresh ticket. celld does not tell code which node it runs on: the node
+shown in debug views comes from an opt-in operator lookup
+(`CELLD_ZONE_NODES`), never used for routing.
+
+Measured on celld 0.6.2 (gr-p42 spike, a loaded 18-core laptop): about 150
+one-player rooms or 4–5 fifty-player rooms per core at 30 Hz; forwarding
+through another node adds under a millisecond; a hibernated room wakes in
+35–130 ms (p50).
+
+### Scaling out: the native multi-host server
+
+The `net` server can also spread zones over several processes itself
+(Endless Grid and Boss Ripple use it): placement, ownership and
+hibernation are then the framework's job.
 
 - **Placement.** A `ZoneDirectory` (`HashRing`) maps each zone id to a
   host. A client that connects to the wrong host is told where to go
@@ -942,7 +1004,10 @@ trait and its ABI (`run_abi`, `call_bytes`).
 | *(default)* | Zone/host runtime, facts queue, protocol, schema, replication, authoring derives, game surface, predicted steps (compiles to `wasm32-unknown-unknown`, where `#[predict]` exports its steps) | `serde`, `serde_json`, `distributed_game_macros` |
 | `runtime` | `spawn_fact_worker` on Tokio | `tokio` (rt, time, sync) |
 | `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native, unit tests); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record`; the `world::zone_lease` and `world::zone_state` cell aggregates | `distributed`, `sha2`, `base64` |
-| `command-host` | `command_sink::CommandFactSink<H: CommandHost>`: facts → commands through any command host, routed by command (`FactRoute`) | `cell` + `distributed/graphql` |
+| `cell` (also) | `command_sink::CommandFactSink<H: FactHost>`: facts → commands through any fact host, routed by command (`FactRoute`); `world::zone_activity` | |
+| `command-host` | `FactHost` for `CelldCommandHost` / `LocalCommandHost` (`CelldFactSink`) | `cell` + `distributed/graphql` |
+| `celld-zone` | `celld_zone`: any `Zone` as a celld Durable Object (`ZoneObject`, `celld_zone_object!`, `forward_to_zone`), facts object to object (`CelldObjectHost`), `serve_cell` / `ZoneActivityCell` for the worker's cells (wasm32) | `cell`, `ticket`, `workers-rs` + `js-sys`, `wasm-bindgen(-futures)` |
+| `ticket` | `ticket`: HS256 admission and transfer tickets without the axum server | `hmac`, `sha2`, `base64` |
 | `celld` | `celld::CelldFactSink` (the command sink over a remote-only `CelldCommandHost`): facts → celld cells over the internal HTTP path; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells; `celld::CellZoneStateStore`: hibernation snapshots in celld `ZoneState` cells | `command-host` + `reqwest`, `uuid` |
 | `workers-rs` | `world::zone_state::celld_worker`: the `zone-state/{zone}` route of a celld worker (cell + R2 bucket binding for large snapshots) | `cell` + `worker` |
 | `metrics` | `metrics::GameMetrics` per-zone Prometheus text | – |

distributed_game/src/bots.rs

@@ -113,8 +113,13 @@ pub fn encode_i8_pair(dir: [f32; 2]) -> Vec<u8> {
 /// Bot run settings.
 #[derive(Clone)]
 pub struct BotConfig {
-    /// Server base URL, e.g. `ws://127.0.0.1:2567` (`/zones/{zone}` is appended).
+    /// Server base URL, e.g. `ws://127.0.0.1:2567` (`{path}/{zone}` is
+    /// appended).
     pub url: String,
+    /// Path the zone id is appended to: `/zones` on the `net` server,
+    /// whatever the worker routes to its zone objects on celld (e.g.
+    /// `/rooms`).
+    pub path: String,
     /// Zone to join.
     pub zone: String,
     /// Number of bots.
@@ -145,6 +150,7 @@ impl BotConfig {
     pub fn new(url: impl Into<String>, zone: impl Into<String>, ticket_key: TicketKey) -> Self {
         Self {
             url: url.into(),
+            path: "/zones".into(),
             zone: zone.into(),
             bots: 10,
             input_rate: 20.0,
@@ -364,7 +370,12 @@ async fn drive(
         name: None,
         extensions: Default::default(),
     });
-    let url = format!("{}/zones/{}", config.url.trim_end_matches('/'), config.zone);
+    let url = format!(
+        "{}/{}/{}",
+        config.url.trim_end_matches('/'),
+        config.path.trim_matches('/'),
+        config.zone
+    );
     let (mut ws, _) = tokio_tungstenite::connect_async(url.as_str())
         .await
         .map_err(|e| format!("connect: {e}"))?;