diff --git a/.celld-version b/.celld-version new file mode 100644 index 000000000..b61604874 --- /dev/null +++ b/.celld-version @@ -0,0 +1 @@ +0.6.2 diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3c7c411de..d2136caf6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,7 +13,8 @@ existing unbounded-tech quality provider plus the integration/* jobs below — | [`test-all-features.yaml`](./test-all-features.yaml) | yes | **This repo:** workspace `--all-features` | | [`integration-*.yaml`](./) | yes | **This repo:** broker / DB / CLI / observability / GraphQL identity+OIDC | | [`integration-e2e-ui.yaml`](./integration-e2e-ui.yaml) | yes | **This repo:** `tests/e2e-ui` offline suite, compose Playwright e2e, and kind local-GitOps Playwright e2e | -| [`integration-celld.yaml`](./integration-celld.yaml) | yes | **This repo:** `tests/e2e-celld` workspace tests + live Azurite+celld+NATS | +| [`integration-celld.yaml`](./integration-celld.yaml) | yes | **This repo:** `tests/e2e-celld` workspace tests, celld storage and cell-alarm proofs (`celld dev`), live celld+NATS | +| [`integration-wasm32-domains.yaml`](./integration-wasm32-domains.yaml) | yes | **This repo:** every domain crate builds for `wasm32-unknown-unknown` (`scripts/check-wasm32-domains.sh`), and a native-only fixture must fail | | [`integration-js.yaml`](./integration-js.yaml) | yes | **This repo:** install, typecheck, test, build, and packed-consumer smoke test for `js/` | | [`on-pr-quality.yaml`](./on-pr-quality.yaml) | entry | **This repo** PR gate (not the consumer quality contract) | | [`on-push-main-version-and-tag.yaml`](./on-push-main-version-and-tag.yaml) | entry | **This repo** main → **vnext** tag | diff --git a/.github/workflows/integration-celld.yaml b/.github/workflows/integration-celld.yaml index 2d4f00953..5f196a208 100644 --- a/.github/workflows/integration-celld.yaml +++ b/.github/workflows/integration-celld.yaml @@ -7,8 +7,7 @@ name: celld (live + e2e-celld) # make -C tests/e2e-celld test # make -C tests/e2e-ui up && make -C tests/e2e-ui up-celld-nats # WATCH=0 WATCH_WORKER=0 make -C tests/e2e-celld run -# E2E_UI_ORIGIN=http://localhost:8791 npx --prefix tests/e2e-ui playwright test \ -# todos.user.spec.ts chat.user.spec.ts --project chromium-user +# make -C tests/e2e-celld test-browser # the full Playwright suite # # Default `cargo test` (quality) still runs fixture-only celld checks and # skips live HTTP unless CELLD_URL is set. This job sets CELLD_URL / NATS_URL. @@ -39,7 +38,10 @@ jobs: targets: wasm32-unknown-unknown - uses: Swatinem/rust-cache@v2 with: - workspaces: tests/celld/worker -> tests/celld/worker/target + workspaces: | + tests/celld/worker -> tests/celld/worker/target + tests/celld/alarm-worker -> tests/celld/alarm-worker/target + tests/celld/fact-worker -> tests/celld/fact-worker/target shared-key: celld-storage - uses: actions/setup-node@v4 with: @@ -49,13 +51,37 @@ jobs: sudo apt-get update && sudo apt-get install -y sqlite3 npm install -g esbuild cargo install worker-build --locked - curl -fsSL https://celld.dev/install.sh | sh + # The pinned celld (/.celld-version) is the one the zone-state R2 + # path, fact routing and every profile below are proven on. + scripts/celld.sh install echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Build isolated fault-probe Worker working-directory: tests/celld/worker run: worker-build --release --features storage-conformance - name: Prove SQL atomicity, fencing, autonomous recovery and growth run: node tests/celld/storage-conformance.mjs + - name: Build the cell-alarm Ticker Worker + working-directory: tests/celld/alarm-worker + run: worker-build --release + - name: Prove re-armable cell alarms (atomic arm, cadence, watchdog, restart) + run: node tests/celld/alarms.mjs + - name: Build the zone-fact cells Worker + working-directory: tests/celld/fact-worker + run: worker-build --release + - name: Route zone facts to celld cells (CelldFactSink, outage and restart) + env: + CELLD_FACTS_REQUIRED: "1" + run: cargo test -p distributed_game --features celld,runtime --test celld_facts + - name: Zone leases in celld cells (CellZoneOwnership, alarm expiry, GameServer) + env: + CELLD_FACTS_REQUIRED: "1" + run: cargo test -p distributed_game --features celld,runtime,net --test celld_zone_lease + - name: Zone hibernation state in celld cells (inline + R2 snapshots, crash, GameServer) + env: + CELLD_FACTS_REQUIRED: "1" + run: | + cargo test -p distributed_game --features celld,runtime --test celld_zone_state + cargo test -p distributed_game --features celld,net,lease --test world - name: Retain storage proof results and logs if: always() uses: actions/upload-artifact@v4 @@ -64,6 +90,8 @@ jobs: path: | /tmp/distributed-cell-storage-*/results.json /tmp/distributed-cell-storage-*/celld-*.log + /tmp/distributed-cell-alarms-*/results.json + /tmp/distributed-cell-alarms-*/celld-*.log if-no-files-found: ignore e2e-celld: @@ -87,7 +115,7 @@ jobs: run: cargo test --workspace --verbose live: - name: celld 0.4 live (local store + Queue + NATS) + name: celld live (local store + Queue + NATS) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -126,9 +154,9 @@ jobs: - name: Install worker-build run: cargo install worker-build --locked || cargo install worker-build - - name: Install celld CLI + - name: Install celld CLI (pinned in /.celld-version) run: | - curl -fsSL https://celld.dev/install.sh | sh + scripts/celld.sh install echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Bring up Postgres + Zitadel and bootstrap OIDC @@ -155,7 +183,7 @@ jobs: working-directory: tests/e2e-ui run: npx playwright install chromium --with-deps - - name: Bring up celld 0.4 + Queue relay + NATS + - name: Bring up celld + Queue relay + NATS run: | command -v celld command -v worker-build @@ -266,11 +294,9 @@ jobs: -H "x-distributed-internal-secret: $DISTRIBUTED_INTERNAL_SECRET" \ -d '{"ids":["forged"]}' - - name: Todo + Chat browser lifecycle through celld + - name: Full Playwright suite through celld (user, admin, anon) working-directory: tests/e2e-ui - run: >- - npx playwright test todos.user.spec.ts chat.user.spec.ts - --project chromium-user + run: npx playwright test env: E2E_UI_ORIGIN: http://localhost:8791 E2E_API_ORIGIN: http://127.0.0.1:8791 diff --git a/.github/workflows/integration-game-demos.yaml b/.github/workflows/integration-game-demos.yaml index 316b4d6ef..2c1304964 100644 --- a/.github/workflows/integration-game-demos.yaml +++ b/.github/workflows/integration-game-demos.yaml @@ -7,7 +7,9 @@ name: Game demos smoke # headless tests, and the SvelteKit demo end to end: `distributed build # examples/game/server` (GraphQL client, svelte-check, vite build), then # Playwright against the production build, a real Mob Pit server and the real -# mobpit-service read model (SQLite only; no external services). +# mobpit-service read model. Mob Pit's durable cells run on celld: the tests +# start their own `celld dev` (CELLD_BIN, pinned in /.celld-version; no Docker) serving +# examples/game/cells, built with worker-build. on: workflow_call: @@ -17,7 +19,8 @@ jobs: runs-on: ubuntu-latest env: CARGO_TERM_COLOR: always - # The shared ticket rate limiter's Lua script runs against a real Redis. + # The shared ticket rate limiter's Lua script and the leaderboard store + # (conformance suite, projector, GraphQL boards) run against a real Redis. services: redis: image: redis:7-alpine @@ -43,11 +46,29 @@ jobs: js/package-lock.json examples/game/client/package-lock.json examples/game/sveltekit/package-lock.json + - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + examples/game/server -> target + examples/game/cells -> target + shared-key: game-demos + - name: Install celld (pinned) and worker-build + run: | + scripts/celld.sh install + echo "CELLD_BIN=$HOME/.local/bin/celld" >> "$GITHUB_ENV" + cargo install worker-build --locked - name: Install SvelteKit demo dependencies and Chromium working-directory: examples/game/sveltekit run: npm ci --no-audit --no-fund && npx playwright install --with-deps chromium - name: Run examples/game smoke + env: + # The SvelteKit leaderboard is the Redis board (GraphQL board roots). + REDIS_URL: redis://127.0.0.1:6379 run: examples/game/run.sh smoke + - name: Leaderboard conformance and GraphQL boards against real Redis + env: + REDIS_URL: redis://127.0.0.1:6379 + run: cargo test --features redis,graphql,sqlite --test redis_leaderboard -- --nocapture - name: Ticket rate limiter against real Redis working-directory: js env: diff --git a/.github/workflows/integration-postgres.yaml b/.github/workflows/integration-postgres.yaml index 465eac066..43dae8fe4 100644 --- a/.github/workflows/integration-postgres.yaml +++ b/.github/workflows/integration-postgres.yaml @@ -45,6 +45,9 @@ jobs: cargo test --test distributed_read_model --no-default-features --features postgres --verbose cargo test --test sql_lock_manager --no-default-features --features postgres --verbose cargo test --test graphql_literal_text --no-default-features --features graphql,postgres --verbose + # Schedule worker: claim with FOR UPDATE SKIP LOCKED, crash/restart and + # concurrent workers fire each (owner, name, generation) once. + cargo test --test schedule_worker --no-default-features --features graphql,postgres --verbose - name: Verify source fencing and snapshot rebuild against PostgreSQL env: DISTRIBUTED_SNAPSHOT_TEST_POSTGRES_URL: postgres://postgres:postgres@localhost:5432/distributed_snapshots diff --git a/.github/workflows/integration-wasm32-domains.yaml b/.github/workflows/integration-wasm32-domains.yaml new file mode 100644 index 000000000..22659e636 --- /dev/null +++ b/.github/workflows/integration-wasm32-domains.yaml @@ -0,0 +1,33 @@ +name: wasm32 domain crates + +# Reusable workflow: referenced via `uses: ./.github/workflows/integration-wasm32-domains.yaml`. +# Domain crates run inside celld cells (wasm32 Durable Objects), so each must +# build for wasm32-unknown-unknown without Tokio sockets, native SQLite or +# other native-only dependencies (game-services decision D4). The script also +# proves the check bites: tests/fixtures/wasm32-native-domain must fail. +# +# Local parity: scripts/check-wasm32-domains.sh +on: + workflow_call: + +jobs: + wasm32-domains: + name: Domain crates build for wasm32-unknown-unknown + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + with: + shared-key: wasm32-domains + cache-directories: target/wasm32-domains + - name: Build every domain crate for wasm32 (and reject the native fixture) + run: scripts/check-wasm32-domains.sh diff --git a/.github/workflows/on-pr-quality.yaml b/.github/workflows/on-pr-quality.yaml index 5d3906655..3a67d2e62 100644 --- a/.github/workflows/on-pr-quality.yaml +++ b/.github/workflows/on-pr-quality.yaml @@ -25,12 +25,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable - name: Run contracts check - run: | - if [ -f contracts/catalog.json ]; then - cargo run -p distributed_cli --quiet -- contracts check --root . --catalog contracts/catalog.json --output json - else - echo "No contracts/catalog.json yet; gate is present and ready for producers." - fi + run: cargo run -p distributed_cli --quiet -- contracts check --root . --catalog distributed.contracts.json --output json # Framework monorepo keeps its existing quality provider. The reusable # quality.yaml in this directory is the contract for *consumers* (domain @@ -82,3 +77,6 @@ jobs: celld: uses: ./.github/workflows/integration-celld.yaml + + wasm32-domains: + uses: ./.github/workflows/integration-wasm32-domains.yaml diff --git a/.github/workflows/on-push-main-version-and-tag.yaml b/.github/workflows/on-push-main-version-and-tag.yaml index a04f3a99c..303b92ed4 100644 --- a/.github/workflows/on-push-main-version-and-tag.yaml +++ b/.github/workflows/on-push-main-version-and-tag.yaml @@ -52,11 +52,14 @@ jobs: celld: uses: ./.github/workflows/integration-celld.yaml + wasm32-domains: + uses: ./.github/workflows/integration-wasm32-domains.yaml + # This uses commit logs and tags from git to determine the next version number and create a tag for the release. # Some commits such as chore: will not trigger a version bump and tag; this is by design. version-and-tag: name: Version and Tag - needs: [quality, all-features, postgres, nats, rabbitmq, kafka, distributed-cli, observability, graphql, e2e-ui, js-client, celld] + needs: [quality, all-features, postgres, nats, rabbitmq, kafka, distributed-cli, observability, graphql, e2e-ui, js-client, celld, wasm32-domains] uses: unbounded-tech/workflow-vnext-tag/.github/workflows/workflow.yaml@v1.22.2 secrets: DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} diff --git a/.gitignore b/.gitignore index 28784b1d2..aa754a628 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,12 @@ tests/celld/worker/build/ tests/celld/worker/.celld/ tests/celld/relay-worker/target/ tests/celld/relay-worker/build/ +tests/celld/alarm-worker/target/ +tests/celld/alarm-worker/build/ +tests/celld/alarm-worker/.celld/ +tests/celld/fact-worker/target/ +tests/celld/fact-worker/build/ +tests/celld/fact-worker/.celld/ # Build output of CLI test fixture crates /distributed_cli/tests/fixtures/*/target/ diff --git a/Cargo.toml b/Cargo.toml index d6793212e..9b341917f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = ["distributed_macros", "distributed_cli", "distributed_game", "distributed_game_macros"] -exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "examples/game/server"] +exclude = ["tests/e2e-ui", "tests/celld/worker", "tests/celld/relay-worker", "tests/celld/alarm-worker", "tests/celld/fact-worker", "tests/fixtures/wasm32-native-domain", "examples/game/server"] resolver = "2" [workspace.package] @@ -62,10 +62,6 @@ redis = ["dep:redis", "dep:tokio", "leaderboard-background"] # task) without Redis, e.g. over `InMemoryLeaderboard`. leaderboard-background = ["dep:tokio"] workers-rs = ["dep:worker"] -# Native durable aggregate cells: one synchronous SQLite file per cell, same -# SQL store and one-transaction-per-command contract as Durable Object cells. -# Separate from `sqlite` (async SQLx pool + tokio); no-op on wasm32. -cell-sqlite = ["dep:rusqlite"] otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing", "dep:tracing-opentelemetry"] graphql = ["dep:async-graphql", "dep:async-graphql-axum", "dep:hmac", "dep:jsonwebtoken", "http", "axum?/ws"] @@ -109,11 +105,6 @@ uuid = { version = "1", features = ["v7"] } url = { version = "2", optional = true } worker = { version = "0.8", features = ["queue"], optional = true } -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -# libsqlite3-sys 0.37 stays inside SQLx 0.9's accepted range, so `sqlite` and -# `cell-sqlite` link one bundled SQLite when enabled together. -rusqlite = { version = "0.39", features = ["bundled"], optional = true } - [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] js-sys = "0.3" uuid = { version = "1", features = ["v7", "js"] } diff --git a/Makefile b/Makefile index 947bd294d..1bf424707 100644 --- a/Makefile +++ b/Makefile @@ -55,4 +55,4 @@ compose-down: ## Read-only aggregate contract lifecycle check (never writes tracked files). contracts-check: - $(CARGO) run -p distributed_cli --quiet -- contracts check --root . --catalog contracts/catalog.json --output human + $(CARGO) run -p distributed_cli --quiet -- contracts check --root . --catalog distributed.contracts.json --output human diff --git a/README.md b/README.md index ca528c63c..713fa5cd2 100644 --- a/README.md +++ b/README.md @@ -373,9 +373,28 @@ perform online shadow swaps, or make several independent projections one transaction. Keep services stopped if a multi-projection maintenance run fails; capture fresh inventories and rerun before resuming them. -Handlers stay thin: most Todo commands are `portable_command!` — shard, -invoke one domain method, commit Eventual. `todo.create` keeps a `handle:` -escape hatch when the body needs extra checks. +Handlers stay thin. The shortest form is the command on the aggregate +method itself: `#[command]` inside the `#[sourced]` impl generates the +handler and the portable mount (see [Command contracts](#command-contracts)): + +```rust,ignore +#[command( + "todo.complete", + shard = todo_id, + load = required, + input = TodoCompleteInput, + outcome = Eventual, + payload = TodoStatusPayload::from_todo, + roles = ["user", "admin"], + field = "todos_complete", +)] +pub fn complete(&mut self, #[principal] owner_id: &str) -> Result<(), TodoError> { .. } +``` + +The Todo playground spells the same registration out with +`portable_command!` — shard, invoke one domain method, commit Eventual. +`todo.create` keeps a `handle:` escape hatch when the body needs extra +checks. ```rust,ignore portable_command! { @@ -552,7 +571,7 @@ with interest management, client prediction (`@hops-ops/distributed/game`), outcomes saved as idempotent commands on cells without ever pausing the game, and zones spread across servers with handoff, hibernation and live content updates. Start with **[`distributed_game/README.md`](distributed_game/README.md)**; -seven runnable demos are in [`examples/game`](examples/game/README.md). +eight runnable demos are in [`examples/game`](examples/game/README.md). --- @@ -605,7 +624,7 @@ Start here in the code: | [`ui/src/lib/components/blob/SelectedBlobGame.graphql`](tests/e2e-ui/ui/src/lib/components/blob/SelectedBlobGame.graphql) | Reusable component island promoted to the nearest static route boundary; one route-param binding drives SSR and component reads. | | [`ui/src/routes/blob/[[gameId]]/+page.svelte`](tests/e2e-ui/ui/src/routes/blob/[[gameId]]/+page.svelte) | Arrow keys → `commands.blob.move`; board from `BlobGames.use()`. | | [`crates/service/src/modules/compose.rs`](tests/e2e-ui/crates/service/src/modules/compose.rs) | One `Service` lists modules. No `Runtime::role`. | -| [`crates/service/src/handlers/commands/blob_move.rs`](tests/e2e-ui/crates/service/src/handlers/commands/blob_move.rs) | `PreparedCommand>` — map/score written with the event. | +| [`crates/blob-domain/src/commands/move_dir.rs`](tests/e2e-ui/crates/blob-domain/src/commands/move_dir.rs) | `portable_command!` returning `Atomic`: the map/score row commits with the event, and a WASM pure previews the move. | | [`crates/todo-domain/src/models/todo.rs`](tests/e2e-ui/crates/todo-domain/src/models/todo.rs) | Plain aggregate — no GraphQL in the domain. | | [`ui/src/auth.ts`](tests/e2e-ui/ui/src/auth.ts) | Auth.js + Zitadel scopes/groups → engine roles. | @@ -718,6 +737,16 @@ network servers. | `kafka` | No | `KafkaBus` (Kafka source/publisher). | | `redis` | No | `leaderboard::RedisLeaderboard` (sorted-set leaderboards) and `bus::RedisStreamPublisher` (Redis Streams `XADD`, id dedupe); implies `leaderboard-background`. The `leaderboard` module and `InMemoryLeaderboard` are always available. | | `leaderboard-background` | No | `leaderboard::BackgroundLeaderboard` (fire-and-forget updates on a Tokio task) over any `LeaderboardStore`, e.g. `InMemoryLeaderboard`, without pulling in Redis. | +| `metrics` | No | `distributed::metrics`: framework counters and gauges with Prometheus text exposition (no SDK dependency); with `http`, a scrape handler. Bus receive loops record into it. | +| `otel` | No | OpenTelemetry tracing spans for bus receive loops and `microsvc` dispatch/handlers (`opentelemetry`, `tracing-opentelemetry`). | +| `workers-rs` | No | Cloudflare `worker` crate bindings for celld cells: the aggregate cell host and the celld Queue publisher/relay (`bus::CelldQueue*`) on `wasm32`. | +| `application-runtime` (alias `runtime`) | No | Checked by `#[command]`/`portable_command!` output in the **declaring** crate: enable it in that crate's features to emit the runtime mounts (`*_mount`, `register_*`, `CommandDefinition`). Contract-only packages leave it off. | +| `gateway` | No | `distributed::gateway`: portable gateway declarations (bindings, routes, dispatch). Starts no listeners. | +| `gateway-native` | No | Native HTTP gateway adapter (hyper/tower), independent of GraphQL and SQL. | +| `gateway-graphql` | No | Whole-operation GraphQL selection for the gateway (parser only). | +| `gateway-delivery` | No | Portable identity, freshness and delivery contracts (snapshots, coalescing, live sharing). | +| `gateway-graphql-native` | No | Native GraphQL execution behind the gateway, including WebSocket `@live` (implies `gateway-native`, `gateway-graphql`, `graphql`). | +| `gateway-worker` | No | workers-rs gateway ingress and delivery contracts, without native executors. | > The `InMemoryBus`, `PostgresBus`, and `SqliteBus` need no separate broker > feature. SQL-backed bus support comes from the matching `postgres` or `sqlite` @@ -797,7 +826,7 @@ Every infrastructure concern in `distributed` follows the same pattern: a **trai | Read model rows | `ReadModelWritePlanStore` + `RelationalReadModelQueryStore` | `InMemoryReadModelStore` | Postgres, SQLite | | Snapshot store | `SnapshotStore` | `InMemorySnapshotStore` | Postgres, SQLite, … | | Outbox publishing | `OutboxStore` + async `MessagePublisher` | `InMemoryRepository` outbox store (dev/test) | Any `MessagePublisher` (e.g. `BusPublisher` over a real `Bus`) | -| Locking | `Lock` + `LockManager` | `InMemoryLockManager` | `PostgresLockManager`, `SqliteLockManager` (durable leases), Redis, … | +| Locking | `Lock` + `LockManager` | `InMemoryLockManager` | `PostgresLockManager`, `SqliteLockManager` (durable leases); implement the traits for anything else | All in-memory defaults are `Clone` and `Send + Sync`, so they work in single-task tests and multi-task servers alike. When you're ready for production, implement the trait for your infrastructure and plug it in — handler code does not change. @@ -1603,7 +1632,64 @@ run_source(service, source, RunOptions::idempotent()).await?; ## Command contracts -Command semantics live in `distributed::command`, independently of the query +**Commands on the aggregate.** Most commands are one aggregate method: +take the actor from the session, load (or create) the aggregate, call the +method, reject on its error, commit with a payload. Write the method and +mark it `#[command]` inside the `#[sourced]` impl; the macro generates the +handler and the portable mount (`pub struct RecordKill`, mounted with +`routes.mount(RecordKill)` or `AggregateCell::mount(RecordKill)`): + +```rust,ignore +#[sourced(entity, aggregate_type = "Crawler", events = "CrawlerEvent", domain_state = CrawlerState)] +impl Crawler { + #[command("crawler.record_kill", shard = principal, load = or_create, outcome = succeeded)] + pub fn record_kill( + &mut self, + #[principal] principal: &str, + input: RecordKillInput, + ) -> Result { + let xp = xp_for_tier(input.tier).ok_or("unknown tier")?; + self.mob_killed(principal.into(), input.name, input.mob, input.tier, xp) + .map_err(|e| e.to_string())?; + // Receipt helper: the recorded event's stream position and time. + let recorded = self.entity.last_recorded().ok_or("nothing recorded")?; + Ok(KillReceipt { sequence: recorded.sequence, at_ms: recorded.unix_ms(), .. }) + } + + #[event("crawler.mob_killed", version = 1, domain)] + fn mob_killed(&mut self, id: String, name: String, mob: String, tier: u8, xp: u32) { .. } +} +``` + +- **Principal.** Exactly one parameter is `#[principal]` (`&str` or + `String`); the generated handler fills it from the verified session + (`x-user-id`, else `Unauthorized`), never from the input. It is + mandatory, and an input whose field (Rust or serialized name) is named + like it does not compile. +- **Input.** The other parameter is the `CommandInput` type; or, with + `input = Type`, the other parameters are that input's fields by name + (`&str` parameters borrow), so a domain method keeps plain arguments. +- **`shard`** is an input field (also exposed as `RecordKill::shard`) or + the principal parameter; **`load`** is `required` (`NotFound` when + missing), `or_create` or `create`. +- **`outcome`** is `succeeded` / `eventual` (the method's `Ok` value is + the payload) or `Succeeded

` / `Eventual

` with `payload = f` + (`fn(&Aggregate) -> P`). The method's error (`Display`) becomes + `Rejected`. Captured domain events are always published with the commit. +- Optional: `roles = ["player"]`, `guard = f`, `field = "graphql_field"`, + `transition = path` (the emitted-event witness; by default the + `domain_commands::*` type `#[sourced]` derives for the method), + `mount = Name` (the struct name, default from the command name's last + segment). + +The generated registration is exactly the hand-written one +(`command_transition::(name)` + `field_name`, +`roles`, `guarded`/`handle`): the command specs, GraphQL SDL and client +manifest are identical (`tests/aggregate_command`). For unusual bodies write +the handler and register it with `portable_command! { .., handle: f }` or +the builder below. + +**Contracts.** Command semantics live in `distributed::command`, independently of the query transport. Use `CommandInput` and `CommandOutput` to describe Serde input/output shapes, and `command::{typed_command, Succeeded, Eventual, Atomic, PreparedCommand}` to declare and prepare commands. The GraphQL adapter derives its surface from @@ -2704,30 +2790,59 @@ reference: [`distributed_cli/README.md`](distributed_cli/README.md). ```text src/ - aggregate/ # Aggregate trait, hydration, async aggregate repository helpers - bus/ # Bus facade + adapters (in-memory, sqlite, postgres, nats, rabbitmq, kafka, knative) - commit_builder/ # Transactional batches for aggregates, outbox, and read models - emitter/ # In-process event emitter helpers (feature = "emitter") - entity/ # Entity, event records, metadata, upcasting codecs - graphql/ # Query service: engine, permissions, identity, SDL, HTTP/WS (feature = "graphql") - in_memory_repo/ # In-memory repository (implements every async trait) - lock/ # Lock + lock manager traits, in-memory locks - microsvc/ # Command/event handler framework: service, context, session - outbox/ # Durable outbox message + commit extension - outbox_worker/ # Outbox claiming, publishing, workers - postgres_repo/ # Postgres async SQL repository (feature = "postgres") - queued_repo/ # Queue-based locking repository wrapper - read_model/ # Read model store traits, in-memory store, schema metadata - snapshot/ # Snapshot store traits, in-memory store, snapshot repository - sqlite_repo/ # SQLite async SQL repository (feature = "sqlite") - table/ # Neutral table/row primitives shared by read models and ops tables - lib.rs # Public exports -distributed_macros/ - src/ # Proc macros: sourced, digest, aggregate, enqueue, ReadModel, Snapshot -js/ # @hops-ops/distributed JS/TS client, command runtime, and SvelteKit adapter -tests/e2e-ui/ # Full-stack CQRS + GraphQL + SvelteKit template (nested workspace) -migrations/ # Explicit SQLite and Postgres migrations -compose.yaml # Local postgres / rabbitmq / kafka / nats for integration tests + aggregate/ # Aggregate trait, hydration, async aggregate repository helpers + application/ # Placement-independent application composition (modules, mounts, plans) + bus/ # Bus facade + adapters (in-memory, sqlite, postgres, nats, rabbitmq, + # kafka, knative, redis streams, celld Queue relay) + clock.rs # Injectable wall clock for coordination primitives + command/ # Typed commands: Eventual/Atomic/Succeeded, prepared completions, client effects + command_dispatch/ # CommandHost: local, HTTP and remote dispatch boundary + command_ledger/ # Durable command identity and fenced commit primitives (crate-private) + commit_builder/ # Transactional batches for aggregates, outbox, and read models + domain_event/ # Typed outward domain events, separate from replay bytes + emitter/ # In-process event emitter helpers (feature = "emitter") + entity/ # Entity, event records, metadata, upcasting codecs + gateway/ # Portable application gateway contracts (feature = "gateway*") + graphql/ # Query service: engine, permissions, identity, SDL, HTTP/WS (feature = "graphql") + in_memory_repo/ # In-memory repository (implements every async trait) + leaderboard/ # Sorted-set leaderboards as a rebuildable projection target (memory, Redis) + lease/ # TTL leases with monotonic fencing tokens + lock/ # Lock + lock manager traits, in-memory locks + metrics/ # Prometheus-compatible framework metrics (feature = "metrics") + microsvc/ # Command/event handler framework: service, context, session, cell_host (celld) + mutation/ # Event-independent read-model mutation IR and interpreters + outbox/ # Durable outbox message + commit extension + outbox_worker/ # Outbox claiming, publishing, workers + postgres_repo/ # Postgres async SQL repository (feature = "postgres") + projection/ # Deterministic domain-event projection programs and runtime + projection_protocol/ # Projection identities, ordering vocabulary, codecs (+ projection_protocol.rs) + queued_repo/ # Queue-based locking repository wrapper + read_model/ # Read model store traits, in-memory store, schema metadata, keyset scans + repository/ # Repository traits (GetStream, TransactionalCommit, …) + saga/ # Event-sourced process managers for cross-cell workflows (incl. fan-out) + schedule.rs # Portable aggregate schedules, host clock, backend capabilities + snapshot/ # Snapshot store traits, in-memory store, snapshot repository + sqlite_repo/ # SQLite async SQL repository (feature = "sqlite") + sqlx_repo/ # SQL code shared by the Postgres and SQLite repositories + table/ # Neutral table/row primitives shared by read models and ops tables + telemetry.rs # Telemetry vocabulary shared by metrics and tracing + time.rs # Wall clock that also works on wasm32 + trace_context.rs # W3C trace-context metadata helpers + lib.rs # Public exports +distributed_macros/ # Proc macros: sourced, digest, aggregate, enqueue, command, portable_command, + # module, application, mutation!, ReadModel, Snapshot, DomainState, + # DomainEvent, CommandInput, CommandOutput (projection! is in src/lib.rs) +distributed_cli/ # The `distributed` CLI (build/dev lifecycle, client codegen, contracts + # check/accept, game-manifest/game-client/game-wasm); also a library +distributed_game/ # Authoritative multiplayer zone runtime: fixed tick, facts, replication, + # WebSocket net, celld cells (zone lease/state, fact sink) +distributed_game_macros/ # Derives for distributed_game: Replicated, GameInput, GameEvent, … +js/ # @hops-ops/distributed JS/TS client, command runtime, SvelteKit adapter, game SDK +examples/ # graphiql.rs, and examples/game (game demos + SvelteKit Mob Pit; own workspace) +tests/e2e-ui/ # Full-stack CQRS + GraphQL + SvelteKit template (nested workspace) +tests/e2e-celld/ # The same UI with commands on celld cells and NATS (nested workspace) +migrations/ # Explicit SQLite and Postgres migrations +compose.yaml # Local postgres / rabbitmq / kafka / nats / redis for integration tests ``` ## Running Tests @@ -2792,7 +2907,7 @@ e2e-celld (`tests/e2e-celld`), Blob game, live chat, GraphiQL. | `tests/graphql_*` | Engine, HTTP/WS, harden, identity, multi-IdP OIDC | | `tests/typed_commands/` | Eventual / `Atomic` / `Succeeded` command registration | | `tests/microsvc/` | Handlers on HTTP, gRPC, bus, session | -| `tests/read_models/`, `tests/distributed_read_model/` | Atomic vs eventual projections | +| `tests/read_model_*/`, `tests/distributed_read_model/` | Read models: commit bridge, keyset scans, relationships, sessions, Atomic vs eventual projections | | `tests/sourced*` / `tests/snapshots/` / `tests/upcasting/` | Macros, snapshots, event versioning | | `tests/sagas/` | Orchestration + choreography with the outbox | | `tests/*_transport/`, `tests/knative_cloudevents/` | Broker adapters + conformance | diff --git a/build.rs b/build.rs index 56da375ba..c406446f3 100644 --- a/build.rs +++ b/build.rs @@ -162,7 +162,7 @@ fn emit_migration_inventory() { fn emit_dialect(generated: &mut String, name: &str, dialect: Dialect, inventory: &Inventory) { match dialect { - Dialect::Sqlite => generated.push_str("#[cfg(any(feature = \"sqlite\", all(feature = \"workers-rs\", target_arch = \"wasm32\"), all(feature = \"cell-sqlite\", not(target_arch = \"wasm32\"))))]\n"), + Dialect::Sqlite => generated.push_str("#[cfg(any(feature = \"sqlite\", all(feature = \"workers-rs\", target_arch = \"wasm32\")))]\n"), Dialect::Postgres => generated.push_str("#[cfg(feature = \"postgres\")]\n"), } generated.push_str("pub(crate) const "); diff --git a/distributed_cli/README.md b/distributed_cli/README.md index f53a96995..9b75010b0 100644 --- a/distributed_cli/README.md +++ b/distributed_cli/README.md @@ -333,6 +333,29 @@ keep `wasm/` out of version control and run `game-wasm` before bundling. Unlike browser WASM pures it needs no wasm-pack, wasm-bindgen glue or JSON bridge; the SDK instantiates the module directly over a binary ABI. +## `distributed contracts check` / `accept` — the contract catalog + +```bash +distributed contracts check # read-only; never writes tracked files +distributed contracts check --output json +distributed contracts accept --scope --staged staged.json +``` + +`check` loads the catalog (default `distributed.contracts.json` under +`--root`, the current directory by default) and verifies that every entry's +recorded identity still matches its sources (for this repository, the +migration inventory). It exits non-zero on drift and +prints what changed. `make contracts-check` and the PR quality workflow run it +on this repository. + +`accept` takes one exact scope (no wildcards: `catalog`, +`migration_inventory`, `surface_client_manifest`, `generated_client_tree`, +`application_manifest`, `deployment_plan`, or `program:`) and a staged JSON object mapping portable relative paths to UTF-8 contents. +It writes those files through a staging directory with an atomic replace and +rolls back on failure. An unchanged payload is a no-op. + +`distributed build` reads the same catalog. + ## `distributed describe` — manifest as JSON ```bash diff --git a/distributed_cli/src/cli.rs b/distributed_cli/src/cli.rs index fff086802..33d8ee402 100644 --- a/distributed_cli/src/cli.rs +++ b/distributed_cli/src/cli.rs @@ -119,6 +119,8 @@ pub enum ServiceCommands { Build(BuildArgs), /// Run one coherent local application supervisor Dev(DevArgs), + /// Aggregate contract lifecycle check and accept + Contracts(ContractsArgs), /// Scaffold a new Distributed microservice crate #[command(alias = "create")] Scaffold(ScaffoldArgs), @@ -222,7 +224,7 @@ pub struct ContractsCheckArgs { #[arg(long, default_value = ".")] pub root: PathBuf, /// Path to the contract catalog JSON (relative to root or absolute) - #[arg(long, default_value = "contracts/catalog.json")] + #[arg(long, default_value = "distributed.contracts.json")] pub catalog: PathBuf, /// Output format #[arg(long, value_enum, default_value = "human")] @@ -762,6 +764,7 @@ pub fn run(args: &ServiceArgs) -> Result<(), Box> { match &args.command { ServiceCommands::Build(build) => run_build(build, &["service"]), ServiceCommands::Dev(dev) => run_dev(dev, &["service"]), + ServiceCommands::Contracts(contracts) => run_contracts(contracts), ServiceCommands::Scaffold(scaffold) => run_scaffold(scaffold), ServiceCommands::Describe(describe) => run_describe(describe), ServiceCommands::Client(client) => run_client(client), @@ -3226,4 +3229,26 @@ mod tests { both.db_url = Some("postgres://localhost/orders".to_string()); assert!(atlas_spec_from_flags(&both, "sql".into()).is_err()); } + + #[test] + fn hosted_service_commands_cover_every_public_distributed_command() { + use clap::Subcommand as _; + let names = |command: clap::Command| -> Vec { + command + .get_subcommands() + .filter(|sub| !sub.is_hide_set()) + .map(|sub| sub.get_name().to_string()) + .collect() + }; + let standalone = names(DistributedCommands::augment_subcommands( + clap::Command::new("distributed"), + )); + let hosted = names(ServiceCommands::augment_subcommands(clap::Command::new( + "service", + ))); + assert_eq!( + standalone, hosted, + "`hops service` (ServiceArgs) must expose every public `distributed` command" + ); + } } diff --git a/distributed_cli/src/client_compiler/manifest/leaderboards.rs b/distributed_cli/src/client_compiler/manifest/leaderboards.rs new file mode 100644 index 000000000..2591ddd6d --- /dev/null +++ b/distributed_cli/src/client_compiler/manifest/leaderboards.rs @@ -0,0 +1,147 @@ +//! Leaderboard section of the client manifest: server-ranked board queries +//! (`top`, `around_me`, `my_rank`, `rank`) generated into `leaderboards.ts`. +//! They are not read-model roots and never enter the replica cache. + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +use super::*; + +/// The only row type the framework emits for leaderboards. +pub(crate) const LEADERBOARD_ENTRY_TYPENAME: &str = "LeaderboardEntry"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManifestLeaderboard { + pub(crate) name: String, + pub(crate) entry_typename: String, + pub(crate) order: String, + pub(crate) boards: String, + pub(crate) default_first: u32, + pub(crate) max_first: u32, + pub(crate) max_radius: u32, + pub(crate) roots: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManifestLeaderboardRoot { + pub(crate) name: String, + pub(crate) operation: String, + pub(crate) list: bool, + pub(crate) live: bool, + pub(crate) arguments: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManifestLeaderboardArgument { + pub(crate) name: String, + pub(crate) type_name: String, + pub(crate) nullable: bool, +} + +fn invalid(message: impl Into) -> ClientCompileError { + ClientCompileError::manifest("client.manifest.leaderboard", message) +} + +/// Validate the leaderboard section against itself and the read-model roots. +pub(crate) fn validate_leaderboards( + leaderboards: &[ManifestLeaderboard], + roots: &std::collections::BTreeMap<(RootOperation, String), ManifestRoot>, + typenames: &BTreeSet, +) -> Result<(), ClientCompileError> { + if !leaderboards.is_empty() && typenames.contains(LEADERBOARD_ENTRY_TYPENAME) { + return Err(invalid(format!( + "model typename `{LEADERBOARD_ENTRY_TYPENAME}` collides with the leaderboard entry type" + ))); + } + let mut names = BTreeSet::new(); + let mut root_names = roots + .keys() + .map(|(_, name)| name.clone()) + .collect::>(); + for leaderboard in leaderboards { + if !is_graphql_name(&leaderboard.name) || !names.insert(leaderboard.name.clone()) { + return Err(invalid(format!( + "invalid or duplicate leaderboard `{}`", + leaderboard.name + ))); + } + if leaderboard.entry_typename != LEADERBOARD_ENTRY_TYPENAME { + return Err(invalid(format!( + "leaderboard `{}` entry type must be `{LEADERBOARD_ENTRY_TYPENAME}`", + leaderboard.name + ))); + } + if !matches!(leaderboard.order.as_str(), "high_first" | "low_first") + || !matches!(leaderboard.boards.as_str(), "single" | "family") + { + return Err(invalid(format!( + "leaderboard `{}` has an unknown order or board kind", + leaderboard.name + ))); + } + if leaderboard.default_first == 0 + || leaderboard.default_first > leaderboard.max_first + || leaderboard.max_radius == 0 + { + return Err(invalid(format!( + "leaderboard `{}` has inconsistent limits", + leaderboard.name + ))); + } + if leaderboard.roots.is_empty() { + return Err(invalid(format!( + "leaderboard `{}` exposes no roots", + leaderboard.name + ))); + } + let family = leaderboard.boards == "family"; + for root in &leaderboard.roots { + if !is_graphql_name(&root.name) || !root_names.insert(root.name.clone()) { + return Err(invalid(format!( + "leaderboard root `{}` is invalid or collides with another root", + root.name + ))); + } + let (list, live, extra): (bool, bool, Option<(&str, &str, bool)>) = + match root.operation.as_str() { + "top" => (true, root.live, Some(("first", "Int", true))), + "around_me" => (true, root.live, Some(("radius", "Int", true))), + "my_rank" => (false, root.live, None), + "rank" => (false, false, Some(("member", "String", false))), + other => { + return Err(invalid(format!( + "leaderboard root `{}` has unknown operation `{other}`", + root.name + ))) + } + }; + let mut expected = Vec::new(); + if family { + expected.push(("board", "String", false)); + } + expected.extend(extra); + let actual = root + .arguments + .iter() + .map(|argument| { + ( + argument.name.as_str(), + argument.type_name.as_str(), + argument.nullable, + ) + }) + .collect::>(); + if root.list != list || root.live != live || actual != expected { + return Err(invalid(format!( + "leaderboard root `{}` does not match its `{}` contract", + root.name, root.operation + ))); + } + } + } + Ok(()) +} diff --git a/distributed_cli/src/client_compiler/manifest/mod.rs b/distributed_cli/src/client_compiler/manifest/mod.rs index 5eac2d648..c29095b25 100644 --- a/distributed_cli/src/client_compiler/manifest/mod.rs +++ b/distributed_cli/src/client_compiler/manifest/mod.rs @@ -1,4 +1,5 @@ mod commands; +mod leaderboards; mod model_validation; mod parse; mod projections; @@ -9,6 +10,7 @@ mod util; pub(crate) use super::{is_graphql_name, ClientCompileError, ClientSurfaceSelector}; pub(crate) use commands::*; +pub(crate) use leaderboards::*; pub(crate) use model_validation::*; pub(crate) use projections::*; pub(crate) use projectors::*; diff --git a/distributed_cli/src/client_compiler/manifest/model_validation.rs b/distributed_cli/src/client_compiler/manifest/model_validation.rs index 7c5a58f97..4c34f4847 100644 --- a/distributed_cli/src/client_compiler/manifest/model_validation.rs +++ b/distributed_cli/src/client_compiler/manifest/model_validation.rs @@ -273,10 +273,12 @@ pub(crate) fn validate_derived_capabilities( capabilities: &ManifestCapabilities, roots: &BTreeMap<(RootOperation, String), ManifestRoot>, commands: &[ManifestCommand], + live_leaderboards: bool, ) -> Result<(), ClientCompileError> { - let has_live_roots = roots - .keys() - .any(|(operation, _)| *operation == RootOperation::Subscription); + let has_live_roots = live_leaderboards + || roots + .keys() + .any(|(operation, _)| *operation == RootOperation::Subscription); if capabilities.live_queries != has_live_roots { return Err(ClientCompileError::manifest( "client.manifest.live_capability", diff --git a/distributed_cli/src/client_compiler/manifest/parse.rs b/distributed_cli/src/client_compiler/manifest/parse.rs index c9651b271..ba0c5ad7a 100644 --- a/distributed_cli/src/client_compiler/manifest/parse.rs +++ b/distributed_cli/src/client_compiler/manifest/parse.rs @@ -27,6 +27,9 @@ struct ManifestWire { projectors: Vec, projection_programs: Vec, projection_bindings: Vec, + /// Absent on manifests without leaderboards (fingerprints unchanged). + #[serde(default)] + leaderboards: Vec, } #[derive(Serialize)] @@ -45,6 +48,8 @@ struct ManifestSchemaMaterial<'a> { projectors: &'a [ManifestProjector], projection_programs: &'a [ManifestProjectionProgram], projection_bindings: &'a [ManifestProjectionBinding], + #[serde(skip_serializing_if = "<[ManifestLeaderboard]>::is_empty")] + leaderboards: &'a [ManifestLeaderboard], } impl ClientManifest { @@ -205,7 +210,12 @@ impl ClientManifest { &projectors, )?); let trusted_presets = derive_trusted_preset_descriptors(&models, &commands)?; - validate_derived_capabilities(&wire.capabilities, &roots, &commands)?; + validate_leaderboards(&wire.leaderboards, &roots, &typenames)?; + let live_leaderboards = wire + .leaderboards + .iter() + .any(|leaderboard| leaderboard.roots.iter().any(|root| root.live)); + validate_derived_capabilities(&wire.capabilities, &roots, &commands, live_leaderboards)?; if wire.schema_fingerprint != computed_schema_fingerprint { return Err(ClientCompileError::manifest( "client.manifest.schema_fingerprint", @@ -233,6 +243,7 @@ impl ClientManifest { projectors, projection_programs, projection_bindings, + leaderboards: wire.leaderboards, }) } @@ -257,6 +268,7 @@ fn schema_fingerprint(wire: &ManifestWire) -> Result projectors: &wire.projectors, projection_programs: &wire.projection_programs, projection_bindings: &wire.projection_bindings, + leaderboards: &wire.leaderboards, }; serde_json::to_vec(&material) .map(|bytes| hash_bytes(&bytes)) diff --git a/distributed_cli/src/client_compiler/manifest/types.rs b/distributed_cli/src/client_compiler/manifest/types.rs index a8c9b64b3..1238bf828 100644 --- a/distributed_cli/src/client_compiler/manifest/types.rs +++ b/distributed_cli/src/client_compiler/manifest/types.rs @@ -25,6 +25,7 @@ pub(crate) struct ClientManifest { pub(crate) projectors: Vec, pub(crate) projection_programs: Vec, pub(crate) projection_bindings: Vec, + pub(crate) leaderboards: Vec, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/distributed_cli/src/client_compiler/render/leaderboards.rs b/distributed_cli/src/client_compiler/render/leaderboards.rs new file mode 100644 index 000000000..86950c5d1 --- /dev/null +++ b/distributed_cli/src/client_compiler/render/leaderboards.rs @@ -0,0 +1,147 @@ +//! `leaderboards.ts`: typed documents for the manifest's leaderboard roots. +//! +//! Leaderboard results are ranked by the server (Redis sorted sets) and are +//! not normalized read-model rows, so they are plain GraphQL operations with +//! exact documents, variable and data types, and an `@live` subscription +//! document where the surface has one. They never enter the replica cache. + +use super::common::json_string; +use crate::client_compiler::manifest::{ + ClientManifest, ManifestLeaderboard, ManifestLeaderboardRoot, LEADERBOARD_ENTRY_TYPENAME, +}; +use crate::client_compiler::ClientCompileError; + +/// `None` when the manifest has no leaderboards. +pub(super) fn render_leaderboards( + manifest: &ClientManifest, +) -> Result, ClientCompileError> { + if manifest.leaderboards.is_empty() { + return Ok(None); + } + let mut out = vec![ + "/** GENERATED by distributed client. Server-ranked leaderboard queries; not replica-cached. */".to_string(), + format!( + "export interface {LEADERBOARD_ENTRY_TYPENAME} {{\n\ + \t/** Zero-based rank in the board's order. */\n\ + \treadonly rank: number;\n\ + \treadonly member: string;\n\ + \treadonly score: number;\n\ + }}" + ), + ]; + let mut index = Vec::new(); + for leaderboard in &manifest.leaderboards { + for root in &leaderboard.roots { + let (block, export) = render_root(manifest, leaderboard, root)?; + out.push(block); + index.push(format!("\t{}: {export},", json_string(&root.name)?)); + } + } + out.push(format!( + "export const LEADERBOARD_OPERATIONS = {{\n{}\n}} as const;", + index.join("\n") + )); + Ok(Some(format!("{}\n", out.join("\n\n")))) +} + +fn pascal(name: &str) -> String { + let mut chars = name.chars(); + match chars.next() { + Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), + None => String::new(), + } +} + +fn render_root( + manifest: &ClientManifest, + leaderboard: &ManifestLeaderboard, + root: &ManifestLeaderboardRoot, +) -> Result<(String, String), ClientCompileError> { + let operation = pascal(&root.name); + let variables = root + .arguments + .iter() + .map(|argument| { + let ty = if argument.type_name == "Int" { + "number" + } else { + "string" + }; + if argument.nullable { + format!("readonly {}?: {ty} | null;", argument.name) + } else { + format!("readonly {}: {ty};", argument.name) + } + }) + .collect::>(); + let definitions = root + .arguments + .iter() + .map(|argument| { + format!( + "${}: {}{}", + argument.name, + argument.type_name, + if argument.nullable { "" } else { "!" } + ) + }) + .collect::>(); + let (definitions, call) = if root.arguments.is_empty() { + (String::new(), String::new()) + } else { + ( + format!("({})", definitions.join(", ")), + format!( + "({})", + root.arguments + .iter() + .map(|argument| format!("{0}: ${0}", argument.name)) + .collect::>() + .join(", ") + ), + ) + }; + let selection = format!("{}{call} {{ rank member score }}", root.name); + let document = format!("query {operation}{definitions} {{ {selection} }}"); + let live = root + .live + .then(|| format!("subscription {operation}_Live{definitions} {{ {selection} }}")); + let data_type = if root.list { + format!("readonly {LEADERBOARD_ENTRY_TYPENAME}[]") + } else { + format!("{LEADERBOARD_ENTRY_TYPENAME} | null") + }; + let export = format!("{operation}Leaderboard"); + let block = format!( + "export type {operation}Variables = {{{}}};\n\ + export type {operation}Data = {{ readonly {}: {data_type} }};\n\ + export const {export} = {{\n\ + \tleaderboard: {},\n\ + \toperation: {},\n\ + \torder: {},\n\ + \tlimits: {{ defaultFirst: {}, maxFirst: {}, maxRadius: {} }},\n\ + \tdocument: {},\n\ + \tliveDocument: {},\n\ + \tschemaHash: {},\n\ + }} as const;", + if variables.is_empty() { + String::new() + } else { + format!(" {} ", variables.join(" ")) + }, + root.name, + json_string(&leaderboard.name)?, + json_string(&root.operation)?, + json_string(&leaderboard.order)?, + leaderboard.default_first, + leaderboard.max_first, + leaderboard.max_radius, + json_string(&document)?, + match &live { + Some(live) => json_string(live)?, + None => "null".to_string(), + }, + json_string(&manifest.schema_fingerprint)?, + ); + Ok((block, export)) +} diff --git a/distributed_cli/src/client_compiler/render/mod.rs b/distributed_cli/src/client_compiler/render/mod.rs index 598f61b70..bacfa4e9b 100644 --- a/distributed_cli/src/client_compiler/render/mod.rs +++ b/distributed_cli/src/client_compiler/render/mod.rs @@ -1,6 +1,7 @@ mod artifact; mod commands; mod common; +mod leaderboards; mod operation; mod project; mod typescript; diff --git a/distributed_cli/src/client_compiler/render/project.rs b/distributed_cli/src/client_compiler/render/project.rs index a070313e4..94f110325 100644 --- a/distributed_cli/src/client_compiler/render/project.rs +++ b/distributed_cli/src/client_compiler/render/project.rs @@ -54,6 +54,14 @@ pub(crate) fn render_project( contents: pures, }); } + let leaderboards = super::leaderboards::render_leaderboards(manifest)?; + let has_leaderboards = leaderboards.is_some(); + if let Some(leaderboards) = leaderboards { + files.push(GeneratedClientFile { + path: "leaderboards.ts".into(), + contents: leaderboards, + }); + } files.push(GeneratedClientFile { path: "protocol.ts".into(), contents: render_protocol(manifest)?, @@ -72,7 +80,7 @@ pub(crate) fn render_project( }); files.push(GeneratedClientFile { path: "index.ts".into(), - contents: render_index(&operations, has_pures), + contents: render_index(&operations, has_pures, has_leaderboards), }); files.push(GeneratedClientFile { path: "manifest.json".into(), @@ -285,7 +293,11 @@ fn render_islands( )) } -fn render_index(operations: &[CompiledOperation], has_pures: bool) -> String { +fn render_index( + operations: &[CompiledOperation], + has_pures: bool, + has_leaderboards: bool, +) -> String { let mut lines = vec![ "/** GENERATED public entrypoint. */".to_string(), "export * from './commands.js';".into(), @@ -295,6 +307,9 @@ fn render_index(operations: &[CompiledOperation], has_pures: bool) -> String { if has_pures { lines.push("export * from './pures.js';".into()); } + if has_leaderboards { + lines.push("export * from './leaderboards.js';".into()); + } for operation in operations { let module = operation .module_path diff --git a/distributed_cli/src/client_compiler/tests.rs b/distributed_cli/src/client_compiler/tests.rs index 407f832f5..ed51fa260 100644 --- a/distributed_cli/src/client_compiler/tests.rs +++ b/distributed_cli/src/client_compiler/tests.rs @@ -4482,3 +4482,148 @@ fn generated_lazy_commands_defer_definitions_and_preserve_command_signatures() { include_str!("../../tests/fixtures/generated-lazy-commands.ts") ); } + +fn leaderboard_manifest() -> JsonValue { + let mut value = manifest(); + value["leaderboards"] = json!([ + { + "name": "xp", + "entry_typename": "LeaderboardEntry", + "order": "high_first", + "boards": "single", + "default_first": 3, + "max_first": 5, + "max_radius": 2, + "roots": [ + {"name": "xpLeaderboard", "operation": "top", "list": true, "live": true, + "arguments": [{"name": "first", "type_name": "Int", "nullable": true}]}, + {"name": "xpMyRank", "operation": "my_rank", "list": false, "live": true, + "arguments": []} + ] + }, + { + "name": "season", + "entry_typename": "LeaderboardEntry", + "order": "low_first", + "boards": "family", + "default_first": 10, + "max_first": 100, + "max_radius": 25, + "roots": [ + {"name": "seasonRank", "operation": "rank", "list": false, "live": false, + "arguments": [ + {"name": "board", "type_name": "String", "nullable": false}, + {"name": "member", "type_name": "String", "nullable": false} + ]} + ] + } + ]); + super::manifest::refresh_schema_fingerprint(&mut value); + value +} + +const LEADERBOARD_PAGE: &str = r#" + query Todos($limit: Int!) { + todos(limit: $limit) { title } + } +"#; + +#[test] +fn leaderboard_roots_generate_typed_documents() { + let project = compile_client(input_with_manifest( + leaderboard_manifest(), + LEADERBOARD_PAGE, + )) + .unwrap(); + let generated = file(&project, "leaderboards.ts"); + assert!(generated.contains("export interface LeaderboardEntry {")); + assert!(generated + .contains("export type XpLeaderboardVariables = { readonly first?: number | null; };")); + assert!(generated.contains( + "export type XpLeaderboardData = { readonly xpLeaderboard: readonly LeaderboardEntry[] };" + )); + assert!(generated.contains( + r#"document: "query XpLeaderboard($first: Int) { xpLeaderboard(first: $first) { rank member score } }","# + )); + assert!(generated.contains( + r#"liveDocument: "subscription XpLeaderboard_Live($first: Int) { xpLeaderboard(first: $first) { rank member score } }","# + )); + // The viewer's rank takes no member argument: the server binds it from + // the verified session. + assert!(generated.contains("export type XpMyRankVariables = {};")); + assert!(generated.contains(r#"document: "query XpMyRank { xpMyRank { rank member score } }","#)); + assert!(generated.contains( + "export type SeasonRankVariables = { readonly board: string; readonly member: string; };" + )); + assert!(generated.contains("liveDocument: null,")); + assert!(generated.contains("\"xpLeaderboard\": XpLeaderboardLeaderboard,")); + assert!(file(&project, "index.ts").contains("export * from './leaderboards.js';")); + assert_eq!( + generated, + include_str!("../../tests/fixtures/generated-leaderboards.ts"), + "the checked-in TypeScript consumer fixture must remain byte-exact" + ); +} + +#[test] +fn manifests_without_leaderboards_generate_no_module() { + let project = compile_client(input(LEADERBOARD_PAGE)).unwrap(); + assert!(project + .files + .iter() + .all(|file| file.path != "leaderboards.ts")); + assert!(!file(&project, "index.ts").contains("leaderboards")); +} + +type ManifestMutation = Box; + +#[test] +fn malformed_leaderboard_sections_are_refused() { + let cases: Vec<(&str, ManifestMutation)> = vec![ + ( + "a member argument on my_rank", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][0]["roots"][1]["arguments"] = + json!([{"name": "member", "type_name": "String", "nullable": false}]); + }), + ), + ( + "a live rank root", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][1]["roots"][0]["live"] = json!(true); + }), + ), + ( + "an unknown operation", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][0]["roots"][0]["operation"] = json!("everyone"); + }), + ), + ( + "a root colliding with a read-model root", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][0]["roots"][0]["name"] = json!("todos"); + }), + ), + ( + "a family root without board", + Box::new(|value: &mut JsonValue| { + value["leaderboards"][1]["roots"][0]["arguments"] = + json!([{"name": "member", "type_name": "String", "nullable": false}]); + }), + ), + ]; + for (case, mutate) in cases { + let mut value = leaderboard_manifest(); + mutate(&mut value); + super::manifest::refresh_schema_fingerprint(&mut value); + let error = compile_client(input_with_manifest(value, LEADERBOARD_PAGE)).expect_err(case); + assert_eq!(error.code, "client.manifest.leaderboard", "{case}: {error}"); + } + + // A stale fingerprint is still caught. + let mut tampered = leaderboard_manifest(); + tampered["leaderboards"][0]["max_first"] = json!(50); + let error = compile_client(input_with_manifest(tampered, LEADERBOARD_PAGE)).unwrap_err(); + assert_eq!(error.code, "client.manifest.schema_fingerprint"); +} diff --git a/distributed_cli/src/contracts/tests.rs b/distributed_cli/src/contracts/tests.rs index a23582ee8..8b56b33d5 100644 --- a/distributed_cli/src/contracts/tests.rs +++ b/distributed_cli/src/contracts/tests.rs @@ -1259,7 +1259,7 @@ fn migration_inventory_is_deterministic_and_preserves_runtime_order() { .iter() .map(|migration| migration.version) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]); assert_eq!( inventory.canonical_bytes().expect("canonical inventory"), inventory diff --git a/distributed_cli/src/game_client.rs b/distributed_cli/src/game_client.rs index 75eecb8b8..af0bb32c1 100644 --- a/distributed_cli/src/game_client.rs +++ b/distributed_cli/src/game_client.rs @@ -9,7 +9,7 @@ //! |---|---| //! | `schema.ts` | `KIND` ids, per-kind field interfaces + accessors over rendered entities, field wire types, `CORRECTIONS` defaults | //! | `input.ts` | the input interface, `BUTTON` bits, `encodeInput` / `decodeInput` / `sanitizeInput` | -//! | `events.ts` | record + event payload interfaces, `EVENT` ids, `EVENTS` codecs, `decodeEvent`, `onEvent`; `encode` / `decode` for the surface's standalone codecs | +//! | `events.ts` | record + event payload interfaces, `EVENT` ids, `EVENTS` codecs, `decodeEvent`, `onEvent`; `encode` / `decode` for the surface's standalone codecs; with zone inbox messages, `Message` interfaces, `MESSAGE` ids, `MESSAGES` codecs and `encodeMessage` (for services posting to `/admin/zone-message`) | //! | `client.ts` | `createClient`: a `GameClient` with the input encoder and correction defaults wired in, plus typed `on`; with a predicted step, prediction defaults to the compiled WebAssembly step | //! | `predict.ts` | only with a `#[predict]` step: its state/context layouts, the module URL (`wasm/.wasm`, built by `distributed game-wasm`) and `createPrediction` | //! | `pures.ts` | only with `#[game_pure]` functions: `createPures`, one typed wrapper per function over the same module | @@ -77,6 +77,8 @@ struct Surface { constants: Vec, #[serde(default)] codecs: Vec, + #[serde(default)] + messages: Vec, } /// A pure function compiled into the surface's module (`#[game_pure]`). @@ -1168,11 +1170,14 @@ impl<'a> Gen<'a> { "type GameEventCodecs", "type GameEventSource", ]; + let messages = &self.surface.messages; let uses_fields = events .iter() + .chain(messages) .any(|e| matches!(e.payload, Payload::Fields { .. })); let uses_value = events .iter() + .chain(messages) .any(|e| matches!(e.payload, Payload::Value { .. })); if uses_fields { sdk.extend(["decodePayload", "encodePayload", "type PayloadLayout"]); @@ -1220,51 +1225,9 @@ impl<'a> Gen<'a> { ); } - for event in events { - let type_name = format!("{}Event", pascal(&event.name)); - out.push('\n'); - match &event.payload { - Payload::Fields { fields } if fields.is_empty() => { - let _ = writeln!( - out, - "/** Event `{}` (id {}): no payload. */\nexport type {type_name} = Readonly>;", - event.name, event.id - ); - let _ = writeln!( - out, - "\nconst {}_LAYOUT: PayloadLayout = [];", - constant(&event.name) - ); - } - Payload::Fields { fields } => { - out.push_str(&self.interface( - &type_name, - &format!("Event `{}` (id {}).", event.name, event.id), - fields, - )); - let _ = writeln!( - out, - "\nconst {}_LAYOUT: PayloadLayout = {};", - constant(&event.name), - self.layout(fields) - ); - } - Payload::Value { ty } => { - let _ = writeln!( - out, - "/** Event `{}` (id {}). */\nexport type {type_name} = {};", - event.name, - event.id, - self.ts_type(ty) - ); - let _ = writeln!( - out, - "\nconst {}_PAYLOAD: WireType = {};", - constant(&event.name), - self.wire_literal(ty) - ); - } - } + self.payload_types(&mut out, events, "Event", ""); + if !messages.is_empty() { + self.payload_types(&mut out, messages, "Message", "MESSAGE_"); } out.push_str("\n/** Event type ids by name. */\nexport const EVENT = {\n"); @@ -1310,9 +1273,106 @@ impl<'a> Gen<'a> { out.push_str("/** A decoded event, discriminated by `name`. */\nexport type AnyEvent = DecodedGameEvent;\n\n"); out.push_str("/** Decode an Event message (`undefined` for ids this surface does not declare). */\nexport function decodeEvent(event: { readonly typeId: number; readonly payload: Uint8Array }): AnyEvent | undefined {\n\treturn decodeGameEvent(EVENTS, event);\n}\n\n"); out.push_str("/** Subscribe to one typed event on every zone of `client`; returns the unsubscribe. */\nexport function onEvent(\n\tclient: GameEventSource,\n\tname: K,\n\thandler: (value: Events[K], zoneId: string) => void\n): () => void {\n\treturn onGameEvent(client, EVENTS, name, handler);\n}\n"); + if !messages.is_empty() { + self.messages_ts(&mut out, messages); + } Ok(out) } + /// Zone inbox message ids, codecs and `encodeMessage` (the body of a + /// `/admin/zone-message` `message`). + fn messages_ts(&self, out: &mut String, messages: &[Event]) { + out.push_str("\n/** Zone inbox message type ids by name. */\nexport const MESSAGE = {\n"); + for m in messages { + let _ = writeln!(out, "\t{}: {},", key(&camel(&m.name)), m.id); + } + out.push_str("} as const;\n\n/** Zone inbox message payload types by name. */\nexport interface Messages {\n"); + for m in messages { + let _ = writeln!( + out, + "\t{}: {}Message;", + key(&camel(&m.name)), + pascal(&m.name) + ); + } + out.push_str("}\n\n/** Codecs of every zone inbox message. */\nexport const MESSAGES: GameEventCodecs = {\n"); + for m in messages { + let type_name = format!("{}Message", pascal(&m.name)); + let c = format!("MESSAGE_{}", constant(&m.name)); + let (encode, decode) = match &m.payload { + Payload::Fields { .. } => ( + format!("(value) => encodePayload({c}_LAYOUT, value)"), + format!( + "(payload) => decodePayload({c}_LAYOUT, payload) as unknown as {type_name}" + ), + ), + Payload::Value { .. } => ( + format!("(value) => encodeValue({c}_PAYLOAD, value)"), + format!("(payload) => decodeValue({c}_PAYLOAD, payload) as {type_name}"), + ), + }; + let _ = writeln!( + out, + "\t{}: {{\n\t\tid: {},\n\t\tname: {},\n\t\tencode: {encode},\n\t\tdecode: {decode}\n\t}},", + key(&camel(&m.name)), + m.id, + string(&m.name) + ); + } + out.push_str("};\n\n/** Encode a zone inbox message: `typeId` and `payload` of a `/admin/zone-message` body (base64url the payload). */\nexport function encodeMessage(\n\tname: K,\n\tvalue: Messages[K]\n): { readonly typeId: number; readonly payload: Uint8Array } {\n\tconst codec = MESSAGES[name];\n\treturn { typeId: codec.id, payload: codec.encode(value) };\n}\n"); + } + + /// Payload interfaces and layouts of events (`suffix` `Event`) or zone + /// inbox messages (`Message`, constants prefixed `MESSAGE_`). + fn payload_types(&self, out: &mut String, items: &[Event], suffix: &str, prefix: &str) { + for event in items { + let type_name = format!("{}{suffix}", pascal(&event.name)); + out.push('\n'); + match &event.payload { + Payload::Fields { fields } if fields.is_empty() => { + let _ = writeln!( + out, + "/** {suffix} `{}` (id {}): no payload. */\nexport type {type_name} = Readonly>;", + event.name, event.id + ); + let _ = writeln!( + out, + "\nconst {prefix}{}_LAYOUT: PayloadLayout = [];", + constant(&event.name) + ); + } + Payload::Fields { fields } => { + out.push_str(&self.interface( + &type_name, + &format!("{suffix} `{}` (id {}).", event.name, event.id), + fields, + )); + let _ = writeln!( + out, + "\nconst {prefix}{}_LAYOUT: PayloadLayout = {};", + constant(&event.name), + self.layout(fields) + ); + } + Payload::Value { ty } => { + let _ = writeln!( + out, + "/** {suffix} `{}` (id {}). */\nexport type {type_name} = {};", + event.name, + event.id, + self.ts_type(ty) + ); + let _ = writeln!( + out, + "\nconst {prefix}{}_PAYLOAD: WireType = {};", + constant(&event.name), + self.wire_literal(ty) + ); + } + } + } + } + /// Records an event payload (or the input) encodes, directly or nested. fn encoded_records(&self) -> BTreeSet<&'a str> { fn walk<'r>(ty: &'r Wire, out: &mut Vec<&'r str>) { @@ -1323,7 +1383,7 @@ impl<'a> Gen<'a> { } } let mut pending = Vec::new(); - for event in &self.surface.events { + for event in self.surface.events.iter().chain(&self.surface.messages) { match &event.payload { Payload::Fields { fields } => fields.iter().for_each(|f| walk(&f.ty, &mut pending)), Payload::Value { ty } => walk(ty, &mut pending), diff --git a/distributed_cli/src/lifecycle/build.rs b/distributed_cli/src/lifecycle/build.rs index 2f3f65a61..032c0dcda 100644 --- a/distributed_cli/src/lifecycle/build.rs +++ b/distributed_cli/src/lifecycle/build.rs @@ -1384,8 +1384,6 @@ mod tests { #[cfg(unix)] #[test] fn lifecycle_executor_receives_exact_cli_path_including_spaces() { - use std::os::unix::fs::PermissionsExt; - let fixture = tempfile::tempdir().expect("create executor fixture"); let stage = fixture.path().join("stage"); fs::create_dir_all(&stage).expect("create executor stage"); @@ -1395,9 +1393,6 @@ mod tests { "#!/bin/sh\nprintf '%s' \"$DISTRIBUTED_LIFECYCLE_CLI_EXECUTABLE\"\n", ) .expect("write executor script"); - let mut permissions = fs::metadata(&script).unwrap().permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&script, permissions).expect("make executor executable"); let cli = fixture.path().join("cli with spaces").join("distributed"); let mut outputs = BTreeSet::new(); outputs.insert("cli-path.txt".to_string()); @@ -1409,8 +1404,11 @@ mod tests { &outputs, &LifecycleExecutor { identity: "sha256:executor".to_string(), - program: script.to_string_lossy().into_owned(), - args: Vec::new(), + // Run the script through `sh` instead of exec-ing it: a + // freshly written executable can fail with ETXTBSY while a + // concurrently forked test child still holds its write fd. + program: "/bin/sh".to_string(), + args: vec![script.to_string_lossy().into_owned()], stdout: Some("cli-path.txt".to_string()), }, false, diff --git a/distributed_cli/tests/cli_lifecycle.rs b/distributed_cli/tests/cli_lifecycle.rs index 72d2363c4..cd9d7f193 100644 --- a/distributed_cli/tests/cli_lifecycle.rs +++ b/distributed_cli/tests/cli_lifecycle.rs @@ -277,11 +277,16 @@ exit 1 let path = root.join("distributed.lifecycle.json"); let mut config: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); config["dev"]["prepare_ms"] = serde_json::json!(100); + // The probe rejects a replacement by never passing, so this budget is + // also how long a rejection takes. It must still leave headroom for the + // initial generation's probe (a `/bin/sh` spawn) under a loaded + // `cargo test --workspace`: at 100 ms that probe timed out and the + // supervisor failed before the test's first assertion. config["dev"]["processes"]["api"]["ready"] = serde_json::json!({ "program": "/bin/sh", "args": ["{root}/generation-probe.sh", "{root}"], "interval_ms": 10, - "timeout_ms": 100 + "timeout_ms": 1500 }); fs::write(path, serde_json::to_vec_pretty(&config).unwrap()).unwrap(); } diff --git a/distributed_cli/tests/fixtures/game-client/sample/events.ts b/distributed_cli/tests/fixtures/game-client/sample/events.ts index d94e2bddc..3febc862a 100644 --- a/distributed_cli/tests/fixtures/game-client/sample/events.ts +++ b/distributed_cli/tests/fixtures/game-client/sample/events.ts @@ -130,6 +130,22 @@ const PING_LAYOUT: PayloadLayout = [ ['seq', { type: 'u32' }], ]; +/** Message `announce` (id 1). */ +export interface AnnounceMessage { + readonly text: string; + readonly urgent: boolean; +} + +const MESSAGE_ANNOUNCE_LAYOUT: PayloadLayout = [ + ['text', { type: 'str' }], + ['urgent', { type: 'bool' }], +]; + +/** Message `set_wind` (id 2). */ +export type SetWindMessage = SampleWind; + +const MESSAGE_SET_WIND_PAYLOAD: WireType = SAMPLE_WIND_TYPE; + /** Event type ids by name. */ export const EVENT = { hello: 1, @@ -198,3 +214,40 @@ export function onEvent( ): () => void { return onGameEvent(client, EVENTS, name, handler); } + +/** Zone inbox message type ids by name. */ +export const MESSAGE = { + announce: 1, + setWind: 2, +} as const; + +/** Zone inbox message payload types by name. */ +export interface Messages { + announce: AnnounceMessage; + setWind: SetWindMessage; +} + +/** Codecs of every zone inbox message. */ +export const MESSAGES: GameEventCodecs = { + announce: { + id: 1, + name: 'announce', + encode: (value) => encodePayload(MESSAGE_ANNOUNCE_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_ANNOUNCE_LAYOUT, payload) as unknown as AnnounceMessage + }, + setWind: { + id: 2, + name: 'set_wind', + encode: (value) => encodeValue(MESSAGE_SET_WIND_PAYLOAD, value), + decode: (payload) => decodeValue(MESSAGE_SET_WIND_PAYLOAD, payload) as SetWindMessage + }, +}; + +/** Encode a zone inbox message: `typeId` and `payload` of a `/admin/zone-message` body (base64url the payload). */ +export function encodeMessage( + name: K, + value: Messages[K] +): { readonly typeId: number; readonly payload: Uint8Array } { + const codec = MESSAGES[name]; + return { typeId: codec.id, payload: codec.encode(value) }; +} diff --git a/distributed_cli/tests/fixtures/game-client/sample/surface.json b/distributed_cli/tests/fixtures/game-client/sample/surface.json index 6629d2703..892008aaa 100644 --- a/distributed_cli/tests/fixtures/game-client/sample/surface.json +++ b/distributed_cli/tests/fixtures/game-client/sample/surface.json @@ -426,6 +426,40 @@ "priority": 1.0 } ], + "messages": [ + { + "id": 1, + "name": "announce", + "payload": { + "fields": [ + { + "name": "text", + "type": { + "type": "str" + } + }, + { + "name": "urgent", + "type": { + "type": "bool" + } + } + ], + "kind": "fields" + } + }, + { + "id": 2, + "name": "set_wind", + "payload": { + "kind": "value", + "type": { + "name": "SampleWind", + "type": "record" + } + } + } + ], "name": "sample", "predict": { "abi": 1, diff --git a/distributed_cli/tests/fixtures/game-surface/src/lib.rs b/distributed_cli/tests/fixtures/game-surface/src/lib.rs index 2712e56c4..75cdfdb42 100644 --- a/distributed_cli/tests/fixtures/game-surface/src/lib.rs +++ b/distributed_cli/tests/fixtures/game-surface/src/lib.rs @@ -62,6 +62,7 @@ impl Zone for Orbs { type Config = (); type Input = Steer; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { schema![OrbView] diff --git a/distributed_cli/tests/fixtures/generated-leaderboards.ts b/distributed_cli/tests/fixtures/generated-leaderboards.ts new file mode 100644 index 000000000..15c6b5e23 --- /dev/null +++ b/distributed_cli/tests/fixtures/generated-leaderboards.ts @@ -0,0 +1,50 @@ +/** GENERATED by distributed client. Server-ranked leaderboard queries; not replica-cached. */ + +export interface LeaderboardEntry { + /** Zero-based rank in the board's order. */ + readonly rank: number; + readonly member: string; + readonly score: number; +} + +export type XpLeaderboardVariables = { readonly first?: number | null; }; +export type XpLeaderboardData = { readonly xpLeaderboard: readonly LeaderboardEntry[] }; +export const XpLeaderboardLeaderboard = { + leaderboard: "xp", + operation: "top", + order: "high_first", + limits: { defaultFirst: 3, maxFirst: 5, maxRadius: 2 }, + document: "query XpLeaderboard($first: Int) { xpLeaderboard(first: $first) { rank member score } }", + liveDocument: "subscription XpLeaderboard_Live($first: Int) { xpLeaderboard(first: $first) { rank member score } }", + schemaHash: "sha256:ef236f93542aea73780b227b1318849fa1688f015459a9b4f4c5c3a9c3752d80", +} as const; + +export type XpMyRankVariables = {}; +export type XpMyRankData = { readonly xpMyRank: LeaderboardEntry | null }; +export const XpMyRankLeaderboard = { + leaderboard: "xp", + operation: "my_rank", + order: "high_first", + limits: { defaultFirst: 3, maxFirst: 5, maxRadius: 2 }, + document: "query XpMyRank { xpMyRank { rank member score } }", + liveDocument: "subscription XpMyRank_Live { xpMyRank { rank member score } }", + schemaHash: "sha256:ef236f93542aea73780b227b1318849fa1688f015459a9b4f4c5c3a9c3752d80", +} as const; + +export type SeasonRankVariables = { readonly board: string; readonly member: string; }; +export type SeasonRankData = { readonly seasonRank: LeaderboardEntry | null }; +export const SeasonRankLeaderboard = { + leaderboard: "season", + operation: "rank", + order: "low_first", + limits: { defaultFirst: 10, maxFirst: 100, maxRadius: 25 }, + document: "query SeasonRank($board: String!, $member: String!) { seasonRank(board: $board, member: $member) { rank member score } }", + liveDocument: null, + schemaHash: "sha256:ef236f93542aea73780b227b1318849fa1688f015459a9b4f4c5c3a9c3752d80", +} as const; + +export const LEADERBOARD_OPERATIONS = { + "xpLeaderboard": XpLeaderboardLeaderboard, + "xpMyRank": XpMyRankLeaderboard, + "seasonRank": SeasonRankLeaderboard, +} as const; diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index e9665bb55..9116199d7 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -15,7 +15,17 @@ default = [] # Tokio-driven fact worker (`spawn_fact_worker`). Core stays runtime-agnostic. runtime = ["dep:tokio"] # Adapter that commits facts into a `distributed` AggregateCell. -cell = ["dep:distributed", "dep:sha2"] +cell = ["dep:distributed", "dep:sha2", "dep:base64"] +# `command_sink::CommandFactSink`: facts dispatched as +# commands through any command host (a local service over any repository, +# or celld cells). +command-host = ["cell", "distributed/graphql"] +# celld clients: `CelldFactSink` (the command sink over a remote-only +# `CelldCommandHost`), `CellZoneOwnership`, `CellZoneStateStore`. +celld = ["cell", "command-host", "dep:reqwest", "dep:uuid"] +# celld worker glue (wasm32 Durable Objects): `world::zone_state::celld_worker`, +# the zone-state HTTP surface over a cell and an R2 bucket binding. +workers-rs = ["cell", "dep:worker", "distributed/workers-rs"] # Per-zone Prometheus metrics (dependency-free text exposition). metrics = [] # Load-test bots (library + `game-bots` binary). @@ -24,8 +34,6 @@ bots = ["net", "dep:tokio-tungstenite", "dep:futures-util", "tokio/rt-multi-thre net = ["runtime", "metrics", "dep:axum", "dep:hmac", "dep:sha2", "dep:base64", "tokio/net", "tokio/macros", "tokio/io-util"] # `world::FencedLeaseOwnership`: zone ownership over `distributed::lease`. lease = ["dep:distributed"] -# `world::SqliteZoneStateStore`: native SQLite hibernation snapshots. -sqlite = ["dep:rusqlite"] [dependencies] distributed_game_macros = { version = "0.1.0", path = "../distributed_game_macros" } @@ -39,9 +47,16 @@ sha2 = { version = "0.10", optional = true } base64 = { version = "0.23", optional = true } tokio-tungstenite = { version = "0.30", optional = true } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"], optional = true } -rusqlite = { version = "0.39", features = ["bundled"], optional = true } +reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true } +uuid = { version = "1", features = ["v7"], optional = true } +worker = { version = "0.8", optional = true } [dev-dependencies] +base64 = "0.23" +# `tests/zone_status`: the read model projected by a consumer service over +# a SQLite bus and read-model store. +distributed = { path = "..", default-features = false, features = ["sqlite", "graphql"] } +sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "sqlite"] } trybuild = "=1.0.120" tokio = { version = "1", features = ["full", "test-util"] } tokio-tungstenite = "0.30" @@ -68,6 +83,51 @@ name = "net_e2e" path = "tests/net_e2e/main.rs" required-features = ["net"] +[[test]] +name = "zone_lease" +path = "tests/zone_lease/main.rs" +required-features = ["cell"] + +[[test]] +name = "zone_state" +path = "tests/zone_state/main.rs" +required-features = ["cell"] + +[[test]] +name = "command_facts" +path = "tests/command_facts/main.rs" +required-features = ["command-host"] + +[[test]] +name = "zone_status" +path = "tests/zone_status/main.rs" +required-features = ["cell"] + +[[test]] +name = "celld_zone_lease" +path = "tests/celld_zone_lease/main.rs" +required-features = ["celld", "runtime", "net"] + +[[test]] +name = "celld_zone_state" +path = "tests/celld_zone_state/main.rs" +required-features = ["celld", "runtime"] + +[[test]] +name = "celld_facts" +path = "tests/celld_facts/main.rs" +required-features = ["celld", "runtime"] + +[[test]] +name = "inbox" +path = "tests/inbox/main.rs" +required-features = ["net"] + +[[test]] +name = "lifecycle" +path = "tests/lifecycle/main.rs" +required-features = ["runtime"] + [[test]] name = "cell_sink" path = "tests/cell_sink/main.rs" @@ -76,7 +136,7 @@ required-features = ["cell", "runtime"] [[test]] name = "world" path = "tests/world/main.rs" -required-features = ["net", "lease", "sqlite"] +required-features = ["net", "lease", "celld"] [[example]] name = "echo_arena" diff --git a/distributed_game/PROTOCOL.md b/distributed_game/PROTOCOL.md index 52a12eb94..568c8a6c7 100644 --- a/distributed_game/PROTOCOL.md +++ b/distributed_game/PROTOCOL.md @@ -308,7 +308,8 @@ Redirect := 0x89, string zone_id, string url Hello. The server closes the socket. - **Error**: codes `1` BAD_MESSAGE, `2` UNAUTHORIZED, `3` REJECTED, `4` ZONE_FULL, `5` VERSION_MISMATCH, `6` RATE_LIMITED, `7` NOT_FOUND, - `8` RECONNECT_FAILED. Errors during the handshake are followed by close; + `8` RECONNECT_FAILED, `9` ZONE_UNAVAILABLE (the zone could not be created + right now; retry later). Errors during the handshake are followed by close; `BAD_MESSAGE` and `RATE_LIMITED` after joining are not. Worked example — Welcome (session "s1", zone "z1", player, 30 Hz ticks, @@ -560,6 +561,7 @@ error codes and treat them as "session over" / "request failed"): - `0x89 Redirect` (server → client). - Kick code `7` RELOAD. +- Error code `9` ZONE_UNAVAILABLE. - Transfer tickets (§8.1) and content-version policies (§8.2); `Handoff` and `Freeze`/`Resume` encodings are unchanged. diff --git a/distributed_game/README.md b/distributed_game/README.md index 8f20ca5db..616f1da4b 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -64,7 +64,8 @@ The words used throughout: | **GameServer** | (`net` feature) A WebSocket server hosting many zones, creating them on demand, routing players, and serving `/metrics` and admin routes. | | **Tick** | One fixed simulation step (30 Hz by default). Separate from the **patch** rate (20 Hz by default), how often clients get updates. | | **Player / Observer** | A *player* controls something and sends inputs. An *observer* only watches: a spectator, a leaderboard page, or a client looking into the zone next door. | -| **Entity** | Something clients can see. The zone describes its entities each patch; the engine works out what changed for each client. | +| **Entity** | A thing inside a zone with its own `EntityId`: a player, a mob, a pillar. It holds server state, some of which no client may see. | +| **View** | What clients see of an entity: a `#[derive(Replicated)]` struct *projected* from it each patch (converted, derived, or left out). The engine works out what changed for each client. | | **Schema** | The kinds of entity and their fields (with quantization), usually from `#[derive(Replicated)]` views. Sent to clients on connect, so a generic client needs no generated code; a generated one adds types. | | **Fact** | An outcome worth keeping ("Alice killed the elite"). The zone emits it (typically the cell command's own input) and moves on; a background worker commits it to storage, where the command decides the reward, and reports back later. | | **Cell** | Durable, event-sourced state in `distributed`, for example one player's profile. Facts usually end up as commands on a cell. | @@ -93,7 +94,7 @@ The words used throughout: ## Try it -Seven runnable demos live in [`examples/game`](../examples/game/README.md), +Eight runnable demos live in [`examples/game`](../examples/game/README.md), each built to show one part of the engine: | Demo | Run | Shows | @@ -169,6 +170,7 @@ impl Zone for Room { type Config = (); type Input = Push; type Timer = (); + type Message = NoMessages; // or a #[derive(GameEvent)] type (zone inbox) fn schema() -> EntitySchema { schema![DotView] } @@ -193,7 +195,9 @@ impl Zone for Room { } } - // Describe the world; the engine diffs it per client. + // Describe the world; the engine diffs it per client. (Zones with + // entity collections register them in `entities()` instead and + // project views from them: see "What each client sees".) fn replicate(&self, out: &mut WorldView) { out.add(1, &DotView { x: self.x }); } @@ -218,13 +222,14 @@ for out in host.drain_outbound() { /* Outbound::Frame bytes go to the client */ assert_eq!(game_surface().kinds[0].name, "dot"); ``` -Serve it over WebSockets (`net` feature). The closure decides which zone -ids this server hosts and with what config: +Serve it over WebSockets (`net` feature). The zone factory decides which +zone ids this server hosts and with what config: ```rust,ignore -let server = GameServer::::new(ServerConfig::new(TicketKey::new(secret)), |zone_id| { - zone_id.starts_with("room-").then_some(()) -}); +let server = GameServer::::new( + ServerConfig::new(TicketKey::new(secret)), + ZoneFactory::sync(|activation| activation.zone_id.starts_with("room-").then_some(())), +); server.serve(tokio::net::TcpListener::bind("0.0.0.0:2567").await?, shutdown_signal).await?; ``` @@ -260,7 +265,8 @@ requestAnimationFrame(function frame(now) { ``` The rest of `Zone` is optional hooks: `on_auth`, `on_join`, `on_leave`, -`on_rejoin`, `on_observe`, `on_timer`, `on_fact_settled`, +`on_rejoin`, `on_connection_lost` (grace started), `on_reconnected`, +`on_observe`, `on_timer`, `on_fact_settled`, `on_tier_changed`, `on_dispose`, `hibernate` / `on_restore`, `on_handoff_join`, `on_child_event`, `is_gate`, `freeze_ready`, `on_content_version_changed`, `record_rewind`, `state_hash`. Each is @@ -322,9 +328,74 @@ that to drop inputs the server has already applied when it reconciles. ### What each client sees -Each patch, `replicate` describes every entity once: `out.add(id, &view)` -with a `#[derive(Replicated)]` view (or a hand-built `EntityState`). The -view's struct is the kind: `#[replicated(kind = "mob", id = 2, priority = +**Zones, entities, views and facts.** The same split as on the business +side of `distributed`: + +| game runtime | business side | here | +|---|---|---| +| zone | aggregate root | one consistency boundary, changed only on its own tick | +| zone entity (`Entity`, `EntityId`) | entity inside the aggregate | `Mob { id, home, target, hp, .. }`, server-only state included | +| view (`Replicated` + `Project`) | read model / projection | `MobView { x, y, hp% , tele }`, recomputed each patch | +| fact | domain event | `crawler.record_kill`, committed by a cell | + +Views are projections, not copies: a view converts (`hp` as a percent), +derives (a telegraph ring from a windup timer), flattens (a nested move +state), clamps, or returns `None` to leave the entity out (a dead mob), +and server-only fields (`target`, `home`, `principal`) simply have no +view field. Declare the entity type with `of` and mark straight copies +`#[from]` (same name) or `#[from(path.to.field)]`; the derive generates +`MobView::project(&mob, MobViewRest { .. })` with only the hand-filled +fields left in `MobViewRest` (a missing or mistyped `#[from]` field is a +compile error at the attribute), and when every field is `#[from]` it +also implements `Project` for you: + +```rust,ignore +#[derive(Replicated)] +#[replicated(kind = "mob", id = 2, of = Mob)] +pub struct MobView { + #[quant(min = -40.0, max = 40.0, bits = 14)] #[from(body.x)] pub x: f32, + #[quant(min = -40.0, max = 40.0, bits = 14)] #[from(body.y)] pub y: f32, + pub hp: u8, // percent: filled by hand +} + +impl Entity for Mob { + fn entity_id(&self) -> EntityId { self.id } +} + +impl Project for Mob { + fn view(&self) -> Option { + (self.hp > 0).then(|| MobView::project(self, MobViewRest { + hp: (self.hp * 100 / self.max_hp) as u8, + })) + } +} +``` + +The zone registers its collections once (`Zone::entities`, like +`schema()`); each patch the host projects every entity in registration +order (`Vec`, slices, `VecDeque`, `BTreeMap` values, `Option`, or one +entity such as the zone itself for a HUD view) and then calls +`Zone::replicate` for anything else: + +```rust,ignore +fn entities() -> Entities { + Entities::new() + .each(|z: &Self| &z.players) + .each(|z: &Self| &z.mobs) + .one(|z: &Self| z) // impl Project for the zone +} +``` + +`replicate` now defaults to nothing and stays for unusual zones: there, +`out.project::(&mob)` pushes one entity's view, and +`out.add(id, &view)` any `#[derive(Replicated)]` view (with or without +`of`; or push a hand-built `EntityState`). `ZoneHost::world_view()` +returns the patch's projected world (tests, tooling). Mob Pit +(`examples/game/server/src/mobpit/zone.rs`) has no `replicate` at all, +and a golden test there checks the projected world and the Delta bytes +against its former hand-written loop. + +**Kinds.** The view's struct is the kind: `#[replicated(kind = "mob", id = 2, priority = 2.0, global)]`, one field per replicated field in wire order (`bool`, `u8`, `u16`, `u32`, `i32`, `f32`, `String` as a short string, `Vec` as bytes), `#[quant(min, max, bits)]` / `#[quant(centered = R, bits)]` (zero @@ -451,7 +522,9 @@ runnable: [`examples/game/sveltekit`](../examples/game/sveltekit) role }`; any other body field is refused, so identity, ownership and rewards never come from the request. Same-origin, a per-principal rate limit and `cache-control: no-store` are built in; the zone server still - enforces signature, zone, expiry and single use. `js/tests` reproduce + enforces signature, zone, expiry and single use. Extension claims (match + id, team…) come only from the server-side `claims(context)` callback and + are signed under `ext`. `js/tests` reproduce `tests/fixtures/sveltekit_tickets.json` byte for byte and `net_e2e` admits those tickets on a real `GameServer` (and refuses tampered, expired, wrong-zone and replayed ones). @@ -663,23 +736,55 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), - The host attaches the killer's **verified identity** from their ticket. The zone cannot claim to be someone else, and the payload names no reward amount; the command handler decides the XP from what it trusts. - The derive refuses identity- or reward-looking fields (`principal`, - `user_id`, `owner`, `xp`, `reward`, … including serde renames) at - compile time. + The derive refuses identity-, reward- and economy-looking fields + (`principal`, `user_id`, `owner`, `xp`, `reward`, `amount`, `coins`, + `gold`, `currency`, `price`, `cost`, `balance`, `grant`, `payout`, + `credit`, `debit`; `_`/`-`/case ignored, serde renames and aliases + included) at compile time, one error per field. - Keys are restart-safe automatically: `ctx.emit(player, input)` uses the input's `#[fact(key(field, ..))]` parts, or the tick and emission order; `ctx.emit_keyed` takes explicit parts; `ctx.emit_zone` emits a zone fact with no principal; `ctx.fact(&input)` builds the `Fact` for a retry backlog. `Fact::new(key, command, json)` still works. - A background worker commits facts with retries. - `cell::CellFactSink` turns each one into a command on that player's - cell, with a command id derived from the fact key. A retry or a - duplicate is therefore a harmless replay, never a double reward. + `command_sink::CommandFactSink` dispatches each one as a + command through a command host — `celld::CelldFactSink` to its celld cell + (that player's cell, a match cell, ...) over the internal path, or a + `LocalCommandHost` to a service over any repository — with a command id + derived from the fact key. A retry or a duplicate is + therefore a harmless replay, never a double reward. (`cell::CellFactSink` + does the same into an in-process reference cell, for unit tests.) - `on_fact_settled` reports the outcome on a later tick. - Keys must be unique across restarts; `ctx.fact_key` includes the activation's boot epoch for you (see [Durable facts](#durable-facts)). - If the process crashes, each in-flight fact is either fully saved or not saved at all. Mob Pit shows this with `kill -9`. +- **Player lifecycle facts.** With `HostConfig::lifecycle_facts = true` + the host itself emits `lifecycle::PlayerJoined` (ticket, rejoin or + handoff), `ConnectionLost` (grace started; dropped or told to reload), + `PlayerReconnected`, `PlayerDisconnected` (grace expired) and + `PlayerLeft { reason }` as facts bound to the verified principal, so + presence, match admission and abandon need no hand-written facts. Keys + are `{command}:{zone}:{boot_epoch}:p{slot}:{n}` (stable across retries, + new after a restart); they share the principal's commit lane (order kept + with `max_concurrent_commits > 1`) and wait in a host backlog rather + than being dropped when the queue is full. `lifecycle::Lifecycle::of(&fact)` + decodes them in a sink. Mob Pit routes them to per-player celld presence + cells (`CommandFactSink::lifecycle_route`), ordered by stamp. +- Every fact carries a `FactStamp`: zone, activation (boot epoch), tick, + host wall time `emitted_at_ms` (`Clock::unix_ms`; `ManualClock` in + tests), content version, config version (`HostConfig::config_version`, + `ctx.set_config_version`) and `seq`, the host's per-activation emission + sequence (every stamped fact gets the next number). The cell sink passes it to handlers as + trusted `x-game-*` session variables (`cell::session_stamp`), and + `stamp_metadata` / `FactStamp::from_metadata` copy it onto events. +- **Commit order is not emission order.** Order facts with + `FactStamp::emission_order` (same zone: activation, then `seq`, so two + facts of one tick and millisecond still order strictly; across + zones: wall time). After a handoff the target's stamps start after the + source's departure time, which the transfer ticket carries, so "joined + B" sorts after "left A" even when it commits first or the two hosts' + clocks disagree. ### Scaling out: many zones, many servers @@ -692,6 +797,8 @@ ctx.emit_keyed(killer, RecordKillInput { mob, tier }, &[&format_args!("m{id}"), - **On demand and asleep.** Zones start on first use. With `hibernate_after`, an empty zone is saved (`Zone::hibernate`) and unloaded, and restored (`on_restore`) the next time someone arrives. + On celld the snapshot is the zone's own `ZoneState` cell: a sleeping + zone is a sleeping Durable Object. - **Handoff.** `ctx.handoff(player, "next-zone", carried_bytes)` moves a player to another zone, on any server, with a signed transfer ticket. The target receives `on_handoff_join` with the carried state. When the @@ -812,7 +919,7 @@ errors for unsupported types and attributes. | Derive | On | Container attribute | Field attributes | Implements | |---|---|---|---|---| -| `Replicated` | struct | `#[replicated(kind = "name", id = N, priority = P, global, x = f, y = f)]` | `quant`, `angle`, `field_type`, `priority`, `correct` | `Replicated` (`kind_def`, `field_values`, `position`) | +| `Replicated` | struct | `#[replicated(kind = "name", id = N, priority = P, global, x = f, y = f, of = Entity)]` | `quant`, `angle`, `field_type`, `priority`, `correct`, `from` / `from(a.b)` (with `of`) | `Replicated` (`kind_def`, `field_values`, `position`); with `of`: `View::project` + `ViewRest` (or, all `#[from]`, `impl Project for Entity`) | | `GameInput` | struct | – | `range`, `buttons`, `quant`, `angle`, `field_type` | `GameInput` (`from_payload`, `to_payload`, `sanitize`) + button consts | | `GameEvent` | enum / struct | `#[game_event(id = N, name = "..")]` per variant / on the struct | `quant`, `angle`, `field_type`, `long` | `GameEvent` (`decode_event`, `to_payload`) + id consts | | `GameRecord` | struct | – | `quant`, `angle`, `field_type`, `long` | `Wire` (nested payload record) | @@ -834,12 +941,14 @@ trait and its ABI (`run_abi`, `call_bytes`). |---|---|---| | *(default)* | Zone/host runtime, facts queue, protocol, schema, replication, authoring derives, game surface, predicted steps (compiles to `wasm32-unknown-unknown`, where `#[predict]` exports its steps) | `serde`, `serde_json`, `distributed_game_macros` | | `runtime` | `spawn_fact_worker` on Tokio | `tokio` (rt, time, sync) | -| `cell` | `cell::CellFactSink` → `AggregateCell::dispatch_idempotent`, `stamp_event_record` | `distributed`, `sha2` | +| `cell` | `cell::CellFactSink` → in-process `AggregateCell::dispatch_idempotent` (native, unit tests); `cell::fact_session_from_headers` / `session_stamp` for cells (also wasm32); `stamp_event_record`; the `world::zone_lease` and `world::zone_state` cell aggregates | `distributed`, `sha2`, `base64` | +| `command-host` | `command_sink::CommandFactSink`: facts → commands through any command host, routed by command (`FactRoute`) | `cell` + `distributed/graphql` | +| `celld` | `celld::CelldFactSink` (the command sink over a remote-only `CelldCommandHost`): facts → celld cells over the internal HTTP path; `celld::CellZoneOwnership`: zone leases in celld `ZoneLease` cells; `celld::CellZoneStateStore`: hibernation snapshots in celld `ZoneState` cells | `command-host` + `reqwest`, `uuid` | +| `workers-rs` | `world::zone_state::celld_worker`: the `zone-state/{zone}` route of a celld worker (cell + R2 bucket binding for large snapshots) | `cell` + `worker` | | `metrics` | `metrics::GameMetrics` per-zone Prometheus text | – | | `net` | `net::GameServer` (axum WebSocket), HMAC tickets, `GET /metrics` (implies `metrics`), handoff, admin peer routes, `WorldCoordinator` | `axum`, `hmac`, `sha2`, `base64`, `tokio` | | `bots` | `bots` load-test client library + `game-bots` binary | `net` + `tokio-tungstenite` | | `lease` | `world::FencedLeaseOwnership` over `distributed::lease` | `distributed` | -| `sqlite` | `world::SqliteZoneStateStore` (hibernation snapshots) | `rusqlite` (bundled) | ## Durable facts @@ -869,7 +978,7 @@ Settlements: `settlement.is::()`, and > do it for you. The host binds the principal from the player's **verified ticket** and -stamps zone/tick/content version. A `FactSink` commits facts on a worker +stamps zone, activation, tick, wall time, content and config version and emission sequence. A `FactSink` commits facts on a worker with bounded retries/backoff; `Zone::on_fact_settled` receives the outcome on a later tick. `cell::CellFactSink` dispatches into an `AggregateCell` with a deterministic command id per fact key, so retries replay instead of @@ -878,6 +987,46 @@ double-crediting. A custom sink needs only crate-root imports: (`BoxFuture` is also re-exported from `world` and `net` for `ZoneOwnership` / `BarrierParticipant` impls). +**Facts as commands.** A zone server sends its facts to the durable +aggregates as commands through a `CommandHost`, with one +`command_sink::CommandFactSink` (feature `command-host`), routing each fact +by its command to an aggregate kind and shard (`FactRoute`: by principal, +by input field, by zone, or a function of the input and trusted session). +The same routes serve every host: `celld::CelldFactSink` (feature `celld`) +addresses celld cells by kind and shard; over a `LocalCommandHost` a +service routes by command and its handler picks the stream +(`tests/command_facts` runs the celld test's routes against an in-memory +repository): + +```rust,ignore +let sink = CelldFactSink::celld(celld_url, InternalHttpSecret::new(secret)?, "zone-server", "zone:pit-1")? + .route(FactRoute::by_principal::("crawler")) // Crawler:{principal} + .route(FactRoute::by_input_field::("match", "match_id")) + .lifecycle_route("presence"); // zone.* → Presence:{account} +// or: CommandFactSink::new(LocalCommandHost::new(service), "zone-server", "zone:pit-1")?.route(..) +let (queue, feed) = fact_channel(capacity, RetryPolicy::until_settled(base, max)); +spawn_fact_worker(feed.with_max_concurrent_commits(n), sink); +``` + +Over celld each fact is `POST {CELLD_URL}/{kind}/{shard}/{command}` with +`{commandId, input}` (the remote-only `CelldCommandHost`); the command id is +`cell::command_id_for(service, key)`, so a retry after a timeout replays the +cell's ledger receipt (`replayed`) instead of committing twice. Cells answer +the standard wait-path body (`CellDispatchResult::wait_path_wire`). The +principal travels as `x-user-id` and the command-ledger principal partition, +the stamp as `x-game-*` headers (`CelldCommandHost::forward_session_prefix`), +and every request carries +`DISTRIBUTED_INTERNAL_SECRET`; the cell rebuilds the trusted session with +`cell::fact_session_from_headers` after checking the secret. Player +lifecycle facts route to presence, never to gameplay cells. celld slow, +unreachable or answering 5xx/`COMMAND_IN_PROGRESS` is `Retryable`: with +`RetryPolicy::until_settled` the fact stays in the queue (no local durable +fallback), the zone keeps ticking, a full queue answers `Backpressure`, +and after celld returns the backlog drains in per-principal order. An +unrouted command, a rejected command or a bad shard is `Permanent`. +`tests/celld_facts` proves this on `celld dev` against +`tests/celld/fact-worker`, including a crash and restart of celld. + **Throughput.** One worker per zone commits facts in emission order, one at a time (a slow commit delays every later fact). Set `ServerConfig::max_concurrent_commits` (or @@ -899,6 +1048,20 @@ principals commit concurrently; settlements arrive in completion order. enforces route zone, expiry and single use; `TicketClaims::name` becomes `PlayerInfo::name` / `ObserverInfo::name` and travels with handoffs. Transfer tickets always use `ticket_key`. +- **Extension claims**: game-declared claims the issuer vouches for (match + id, team, party) travel in the ticket's `ext` object. Declare them once — + `#[derive(Serialize, Deserialize, TicketExtension)] struct MatchClaims { + match_id: String, team: u8 }` — and admit them with + `ServerConfig::with_ticket_extensions::()`. The derive + refuses identity-, reward- and economy-looking names (the `FactCommand` + deny list, shared with the JS handler through + `tests/fixtures/trusted_field_denylist.json`); the server refuses tickets + whose `ext` has undeclared names or wrong types, and any `ext` when no + schema is declared (join tickets from any verifier and transfer tickets + alike). Zones read them typed in `on_auth` + (`req.claims.extensions.typed::()`) and afterwards from + `PlayerInfo::extensions`; a rejoin replaces them with the fresh ticket's + and handoffs carry them to the next zone. - **Input pacing**: `InputCredits::new(max)` gives each player one input credit per tick (banking up to `max` for jitter bursts); `credits.take(ctx, player)` consumes queued inputs while credits last, so @@ -918,7 +1081,7 @@ principals commit concurrently; settlements arrive in completion order. stored history. The interpolation delay travels in the game's input payload (no wire change). - **Record/replay** (`replay`): drive a host through `Recorder` (same - methods as `ZoneHost`), save the compact `Recording` (`DGR2`), and + methods as `ZoneHost`), save the compact `Recording` (`DGR3`), and `replay::(&recording, config)` re-runs it on a `ManualClock`, checking a per-tick state hash (`Zone::state_hash`, or the replicated `WorldView`). A mismatch returns a `Divergence` naming the tick and the @@ -955,8 +1118,9 @@ principals commit concurrently; settlements arrive in completion order. | Piece | Purpose | |---|---| | `ZoneDirectory` (`StaticDirectory`, `HashRing`) | zone → owning `HostAddr`. `HashRing` uses virtual nodes; adding/removing one of N hosts moves ~1/N of the zones. A Hello for a zone owned elsewhere gets `Redirect(zone, url)`; `url` is the owner's `HostAddr::ws_url` (advertise a proxied public URL with `HostAddr::with_ws_url`) or `ServerConfig::with_redirect_url(\|host, zone\| …)`. | -| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. | -| `ZoneStateStore` (`MemoryZoneStateStore`, `SqliteZoneStateStore`) | hibernation snapshots, written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | +| `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the lease's expiry schedule (`ctx.schedule`; the routes require `Capability::Scheduling` and `Clock`) publishes `zone.lease_expired` when a holder stops renewing, so services see a crashed zone server; each event carries the lease's `ZoneLeaseStatus { zone_id, holder, token, status }`. `world::ownership_conformance` is the behaviour every implementation passes. | +| `ZoneStateStore` (`celld::CellZoneStateStore`; `MemoryZoneStateStore` for unit tests) | hibernation snapshots (async `load` / `save` / `delete`), written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). On celld each zone's snapshot is its own `ZoneState:{zone}` cell (`world::zone_state`): inline up to 64 KiB, larger ones in the worker's R2 bucket with the digest in the cell; saves and deletes publish `zone_state.saved` / `zone_state.deleted` with a `ZoneSnapshotStatus { zone_id, token, stored, content_version, bytes }` (no bytes). `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | +| `world::zone_status` (feature `cell`) | the read model of those events: `zone_leases` and `zone_snapshots` rows (`projection!` upserts of the state bodies), `projectors()` to mount on any consumer service and read-model store, and `ZoneStatus::of` → `Active { holder }` / `Hibernated { bytes }` / `Cold`. For views (world maps, minimaps): it answers "which zones are active or asleep?" without opening a cell per zone (on celld a read of a never-used cell creates it). Eventually consistent; ownership and restore always go to the cells. | | `ZoneTopology` (`DottedTopology` default: `eldoria.port.docks` → `eldoria.port` → `eldoria`) | parent links for tier-up events. | **Per-zone host settings.** `GameServer::with_host_config(|zone_id, host| @@ -964,6 +1128,41 @@ principals commit concurrently; settlements arrive in completion order. 2 Hz while its neighborhoods run at 30 Hz); the world content version is applied afterwards. +**Zone inbox.** Services talk to a running zone through typed messages: +`type Message = MyMessage` on the zone (a `#[derive(GameEvent)]` struct or +enum, so it has stable ids and the event payload codec; `NoMessages` for +none) and `Zone::on_message(ctx, message)`, which runs on the zone's task +between ticks, never during one. Deliver with `GameServer::send(zone, +ZoneInbound::Message(m))` locally, `POST /admin/zone-message` (admin +secret; `{zone, message: {type_id, payload}}` or `{zone, kick: +{principal, reason}}`) from peers, or `net::ZoneInboxClient` from a +service: it finds the owner through the `ZoneDirectory` and follows `421 +wrong_owner` answers. The built-in `ZoneInbound::Kick { principal, reason +}` removes every connection of the principal (`LeaveReason::Kicked`, and a +`PlayerLeft` lifecycle fact). A message for a zone that is not running is +refused (`404 inactive` / `InboxError::Inactive`) and never activates it. +Messages appear in the game surface under `messages`; the generated +TypeScript has `MESSAGE`, `MESSAGES` and `encodeMessage` for TS services. +Recorded and replayed like the other entry points (`Recorder::deliver`). + +**Zone factory and allocation.** `ZoneFactory::new(|activation| async move +{ … })` may await storage: it runs on the activation task after the +ownership claim and snapshot load, before the zone exists. It receives a +`ZoneActivation { zone_id, activation, allocation }` — `activation` is the +fencing token (= `boot_epoch` = the facts' stamp activation), `allocation` +the `ZoneAllocation` record (roster, mode, config version) that the service +which allocated the zone wrote into the store passed to +`GameServer::with_allocation_store` (`MemoryZoneAllocationStore`, or your +own `ZoneAllocationStore`). Return `Ok(Some(config))` to create the zone +(`on_create` gets it), `Ok(None)` for "no such zone" (`Error(NOT_FOUND)`), +`Err(ZoneCreateError)` when something it needs failed +(`Error(ZONE_UNAVAILABLE)`). Store failures and a factory that outlives +`ServerConfig::activation_timeout` also answer `ZONE_UNAVAILABLE`, never a +hang. Joins that arrive during creation wait and are admitted afterwards; +the tick never awaits. `ZoneFactory::sync(|activation| …)` covers factories +without I/O. `GameServer::activate(zone).await` waits for a zone to run; +`begin_activation(zone)` only starts it. + **Activate on demand / hibernate.** `GameServer` creates a zone on first join (or tier-up event). With `ServerConfig::hibernate_after = Some(d)` a zone that stays Dormant for `d` (no clients, no in-flight facts, not diff --git a/distributed_game/examples/echo_arena.rs b/distributed_game/examples/echo_arena.rs index 3da2662d9..7b417f64b 100644 --- a/distributed_game/examples/echo_arena.rs +++ b/distributed_game/examples/echo_arena.rs @@ -16,7 +16,7 @@ use std::collections::BTreeMap; use std::io::{BufRead, Write}; -use distributed_game::net::{GameServer, ServerConfig, TicketKey}; +use distributed_game::net::{GameServer, ServerConfig, TicketKey, ZoneFactory}; use distributed_game::{ ConsumePolicy, Ctx, EntitySchema, EntityState, FieldType, FieldValue, KindDef, LeaveReason, PlayerId, View, WorldView, Zone, @@ -33,6 +33,7 @@ impl Zone for EchoArena { type Config = (); type Input = (i8, i8); type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { let pos = FieldType::QuantF32 { @@ -105,8 +106,10 @@ async fn main() -> std::io::Result<()> { let mut config = ServerConfig::new(TicketKey::new(secret)); config.host.tick_rate = 30; config.host.patch_rate = 20; - let server = - GameServer::::new(config, |zone| zone.starts_with("arena-").then_some(())); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with("arena-").then_some(())), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; println!("LISTENING ws://{addr}"); diff --git a/distributed_game/src/authoring.rs b/distributed_game/src/authoring.rs index 95e89b0ac..ebbb6ad69 100644 --- a/distributed_game/src/authoring.rs +++ b/distributed_game/src/authoring.rs @@ -84,8 +84,8 @@ impl Pushed<'_> { impl WorldView { /// Push entity `id` described by a `Replicated` view. - pub fn add(&mut self, id: u32, view: &R) -> Pushed<'_> { - self.push(view.entity(id)); + pub fn add(&mut self, id: impl Into, view: &R) -> Pushed<'_> { + self.push(view.entity(id.into().get())); Pushed(self.last_mut().expect("just pushed")) } } diff --git a/distributed_game/src/bots.rs b/distributed_game/src/bots.rs index 3cf03bce3..8f76e9a8e 100644 --- a/distributed_game/src/bots.rs +++ b/distributed_game/src/bots.rs @@ -362,6 +362,7 @@ async fn drive( nonce: format!("{principal}-{}-{}", config.seed, unix), role: Role::Player, name: None, + extensions: Default::default(), }); let url = format!("{}/zones/{}", config.url.trim_end_matches('/'), config.zone); let (mut ws, _) = tokio_tungstenite::connect_async(url.as_str()) diff --git a/distributed_game/src/cell.rs b/distributed_game/src/cell.rs index 62e363df0..d12ba304e 100644 --- a/distributed_game/src/cell.rs +++ b/distributed_game/src/cell.rs @@ -17,16 +17,25 @@ //! `InProgress` and `Internal` errors are retryable; everything else is //! permanent. +#[cfg(not(target_arch = "wasm32"))] use std::sync::Arc; +#[cfg(not(target_arch = "wasm32"))] use distributed::cell_host::{AggregateCell, CellCommandIdentity, CellDispatchError}; use distributed::microsvc::Session; -use distributed::{Aggregate, USER_ID_KEY}; +#[cfg(not(target_arch = "wasm32"))] +use distributed::Aggregate; +use distributed::USER_ID_KEY; use sha2::{Digest, Sha256}; -use crate::facts::{BoxFuture, Fact, FactError, FactReceipt, FactSink}; +use crate::facts::Fact; +#[cfg(not(target_arch = "wasm32"))] +use crate::facts::{BoxFuture, FactError, FactReceipt, FactSink}; -/// Fact sink backed by an aggregate cell. +/// Fact sink backed by an aggregate cell hosted in the zone server's own +/// process (native only: on wasm32 a cell is a Durable Object, reached over +/// the network with [`crate::celld::CelldFactSink`]). +#[cfg(not(target_arch = "wasm32"))] pub struct CellFactSink where A: Aggregate + Send + Sync + 'static, @@ -36,6 +45,7 @@ where system_principal: String, } +#[cfg(not(target_arch = "wasm32"))] impl CellFactSink where A: Aggregate + Send + Sync + 'static, @@ -87,6 +97,35 @@ pub fn trusted_session(principal: &str, fact: &Fact) -> Session { session } +/// The trusted session of a fact received by a celld cell over the internal +/// path ([`crate::celld::CelldFactSink`]): `x-user-id` plus the `x-game-*` +/// stamp headers. `header` reads one request header. Call it only after the +/// request authenticated with `DISTRIBUTED_INTERNAL_SECRET`; returns `None` +/// when the principal or a stamp field is missing (not a fact request). +pub fn fact_session_from_headers(header: impl Fn(&str) -> Option) -> Option { + let principal = header(USER_ID_KEY).filter(|value| !value.is_empty())?; + let mut vars = std::collections::HashMap::new(); + for (name, _) in crate::FactStamp::default().session_vars() { + vars.insert(name.clone(), header(&name)?); + } + let stamp = crate::FactStamp::from_session_vars(&vars)?; + let mut session = Session::new(); + session.set(USER_ID_KEY, principal); + for (key, value) in stamp.session_vars() { + session.set(key, value); + } + Some(session) +} + +/// The [`FactStamp`](crate::FactStamp) a [`trusted_session`] carries +/// (None for a session that did not come from a fact). Handlers use it to +/// order facts ([`FactStamp::emission_order`](crate::FactStamp::emission_order)) +/// and to [`stamp_metadata`](crate::stamp_metadata) their events. +pub fn session_stamp(session: &Session) -> Option { + crate::FactStamp::from_session_vars(session.variables()) +} + +#[cfg(not(target_arch = "wasm32"))] fn classify(error: CellDispatchError) -> FactError { match error { CellDispatchError::InProgress | CellDispatchError::Internal(_) => { @@ -96,6 +135,7 @@ fn classify(error: CellDispatchError) -> FactError { } } +#[cfg(not(target_arch = "wasm32"))] impl FactSink for CellFactSink where A: Aggregate + Send + Sync + 'static, diff --git a/distributed_game/src/celld.rs b/distributed_game/src/celld.rs new file mode 100644 index 000000000..7d51654b6 --- /dev/null +++ b/distributed_game/src/celld.rs @@ -0,0 +1,453 @@ +//! A zone server's celld clients: [`CelldFactSink`] (facts as commands to +//! the durable aggregates in celld cells, game-services decision D3: the +//! [`CommandFactSink`](crate::command_sink::CommandFactSink) over a +//! remote-only `CelldCommandHost`), [`CellZoneOwnership`] (zone leases) and +//! [`CellZoneStateStore`] (hibernation snapshots). + +pub use crate::command_sink::{CelldFactSink, FactRoute, DEFAULT_CELLD_FACT_TIMEOUT}; + +use std::time::Duration; + +use distributed::cell_host::{ + InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, + CELL_SERVICE_ID_HEADER, +}; +use distributed::USER_ID_KEY; +use serde_json::{json, Value}; + +use crate::facts::BoxFuture; + +/// Path segments are non-empty, short, and printable. +fn valid_segment(segment: &str) -> bool { + !segment.is_empty() + && segment.len() <= 256 + && segment.bytes().all(|byte| (0x21..=0x7e).contains(&byte)) +} + +fn encode_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b':') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +/// [`ZoneOwnership`](crate::world::ZoneOwnership) over celld +/// `ZoneLease:{zone_id}` cells ([`crate::world::zone_lease`]): zone servers +/// acquire, renew and release over the internal path, and the cell's alarm +/// declares expiry (`zone.lease_expired`) when a holder stops renewing, so +/// services see a crashed zone server. +/// +/// Every call uses a fresh command id. A retried `acquire` by the same +/// holder returns its existing claim, so an ambiguous response is safe. +#[derive(Clone)] +pub struct CellZoneOwnership { + client: reqwest::Client, + base: String, + secret: InternalHttpSecret, + service_id: String, + host: String, + kind: String, +} + +/// Cell kind (URL segment) of zone-lease cells. +pub const ZONE_LEASE_KIND: &str = "zone-lease"; + +impl CellZoneOwnership { + /// Ownership for zone server `host` (its internal identity) against + /// `celld_url`, under command-ledger `service_id`. + pub fn new( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + host: impl Into, + ) -> Result { + let base = celld_url.into().trim_end_matches('/').to_string(); + if !(base.starts_with("http://") || base.starts_with("https://")) { + return Err(format!("celld URL must be http(s): {base}")); + } + let service_id = service_id.into(); + let host = host.into(); + if service_id.trim().is_empty() || host.trim().is_empty() { + return Err("cell zone ownership needs a service id and a host identity".into()); + } + let client = reqwest::Client::builder() + .timeout(DEFAULT_CELLD_FACT_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("celld ownership client: {error}"))?; + Ok(Self { + client, + base, + secret, + service_id, + host, + kind: ZONE_LEASE_KIND.to_string(), + }) + } + + async fn call( + &self, + zone_id: &str, + command: &str, + input: Value, + ) -> Result { + use crate::world::zone_lease::ZONE_HOST_ROLE; + use crate::world::OwnershipError; + if !valid_segment(zone_id) { + return Err(OwnershipError::Backend(format!( + "invalid zone id `{zone_id}`" + ))); + } + let url = format!( + "{}/{}/{}/{}", + self.base, + encode_segment(&self.kind), + encode_segment(zone_id), + encode_segment(command) + ); + let response = self + .client + .post(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .header(CELL_SERVICE_ID_HEADER, &self.service_id) + .header(CELL_PRINCIPAL_PARTITION_HEADER, &self.host) + .header(USER_ID_KEY, &self.host) + .header(distributed::microsvc::ROLE_KEY, ZONE_HOST_ROLE) + .json(&json!({ + "commandId": uuid::Uuid::now_v7().hyphenated().to_string(), + "input": input, + })) + .send() + .await + .map_err(|error| OwnershipError::Backend(format!("celld unreachable: {error}")))?; + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if (200..300).contains(&status) { + return serde_json::from_value(body.get("payload").cloned().unwrap_or(Value::Null)) + .map_err(|error| OwnershipError::Backend(format!("bad claim payload: {error}"))); + } + let message = body.get("error").and_then(Value::as_str).unwrap_or(""); + Err(match crate::world::zone_lease::ownership_error(message) { + OwnershipError::Backend(message) => { + OwnershipError::Backend(format!("HTTP {status}: {message}")) + } + error => error, + }) + } +} + +fn claim_of(payload: crate::world::zone_lease::ClaimPayload) -> crate::world::ZoneClaim { + crate::world::ZoneClaim { + zone_id: payload.zone_id, + holder: payload.holder, + token: payload.token, + expires_at: std::time::SystemTime::UNIX_EPOCH + + Duration::from_millis(payload.expires_at_ms), + } +} + +fn ttl_ms(ttl: Duration) -> u64 { + u64::try_from(ttl.as_millis()).unwrap_or(u64::MAX).max(1) +} + +impl crate::world::ZoneOwnership for CellZoneOwnership { + fn acquire<'a>( + &'a self, + zone_id: &'a str, + holder: &'a str, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + zone_id, + crate::world::zone_lease::ACQUIRE, + json!({ "zone_id": zone_id, "holder": holder, "ttl_ms": ttl_ms(ttl) }), + ) + .await + .map(claim_of) + }) + } + + fn renew<'a>( + &'a self, + claim: &'a crate::world::ZoneClaim, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + &claim.zone_id, + crate::world::zone_lease::RENEW, + json!({ + "zone_id": claim.zone_id, + "holder": claim.holder, + "token": claim.token, + "ttl_ms": ttl_ms(ttl), + }), + ) + .await + .map(claim_of) + }) + } + + fn release<'a>(&'a self, claim: &'a crate::world::ZoneClaim) -> BoxFuture<'a, ()> { + Box::pin(async move { + let _ = self + .call( + &claim.zone_id, + crate::world::zone_lease::RELEASE, + json!({ "zone_id": claim.zone_id, "holder": claim.holder, "token": claim.token }), + ) + .await; + }) + } +} + +/// [`ZoneStateStore`](crate::world::ZoneStateStore) over celld +/// `ZoneState:{zone_id}` cells ([`crate::world::zone_state`]): hibernation +/// snapshots of a zone server live in one celld cell per zone (inline below +/// [`ZONE_STATE_INLINE_MAX`](crate::world::zone_state::ZONE_STATE_INLINE_MAX), +/// in celld's object storage above it), fenced by the ownership token. +/// +/// Every write uses a fresh command id: a retried save under the same token +/// is accepted again (equal tokens are allowed), so an ambiguous response is +/// safe. +#[derive(Clone)] +pub struct CellZoneStateStore { + client: reqwest::Client, + base: String, + secret: InternalHttpSecret, + service_id: String, + host: String, +} + +/// Cell kind (URL segment) of zone-state cells. +pub const ZONE_STATE_KIND: &str = "zone-state"; + +impl CellZoneStateStore { + /// Store for zone server `host` against `celld_url`, under + /// command-ledger `service_id`. + pub fn new( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + host: impl Into, + ) -> Result { + let base = celld_url.into().trim_end_matches('/').to_string(); + if !(base.starts_with("http://") || base.starts_with("https://")) { + return Err(format!("celld URL must be http(s): {base}")); + } + let service_id = service_id.into(); + let host = host.into(); + if service_id.trim().is_empty() || host.trim().is_empty() { + return Err("cell zone state needs a service id and a host identity".into()); + } + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("celld zone-state client: {error}"))?; + Ok(Self { + client, + base, + secret, + service_id, + host, + }) + } + + fn url( + &self, + zone_id: &str, + command: Option<&str>, + ) -> Result { + if !valid_segment(zone_id) { + return Err(crate::world::StateError::Backend(format!( + "invalid zone id `{zone_id}`" + ))); + } + let mut url = format!( + "{}/{}/{}", + self.base, + encode_segment(ZONE_STATE_KIND), + encode_segment(zone_id) + ); + if let Some(command) = command { + url.push('/'); + url.push_str(&encode_segment(command)); + } + Ok(url) + } + + async fn command( + &self, + zone_id: &str, + command: &str, + input: Value, + ) -> Result<(), crate::world::StateError> { + use crate::world::zone_lease::ZONE_HOST_ROLE; + use crate::world::StateError; + let url = self.url(zone_id, Some(command))?; + let token = input.get("token").and_then(Value::as_u64).unwrap_or(0); + let response = self + .client + .post(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .header(CELL_SERVICE_ID_HEADER, &self.service_id) + .header(CELL_PRINCIPAL_PARTITION_HEADER, &self.host) + .header(USER_ID_KEY, &self.host) + .header(distributed::microsvc::ROLE_KEY, ZONE_HOST_ROLE) + .json(&json!({ + "commandId": uuid::Uuid::now_v7().hyphenated().to_string(), + "input": input, + })) + .send() + .await + .map_err(|error| StateError::Backend(format!("celld unreachable: {error}")))?; + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if (200..300).contains(&status) { + return Ok(()); + } + let message = body.get("error").and_then(Value::as_str).unwrap_or(""); + match crate::world::zone_state::stale_current(message) { + Some(current) => Err(StateError::Stale { + presented: token, + current, + }), + None => Err(StateError::Backend(format!("HTTP {status}: {message}"))), + } + } +} + +impl crate::world::ZoneStateStore for CellZoneStateStore { + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, crate::world::StateError>> + { + use crate::world::StateError; + Box::pin(async move { + use base64::Engine as _; + let url = self.url(zone_id, None)?; + let response = self + .client + .get(&url) + .header(CELL_INTERNAL_SECRET_HEADER, self.secret.header_value()) + .send() + .await + .map_err(|error| StateError::Backend(format!("celld unreachable: {error}")))?; + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap_or(Value::Null); + if !(200..300).contains(&status) { + return Err(StateError::Backend(format!("HTTP {status}: {body}"))); + } + let Some(state) = body.get("state").filter(|state| !state.is_null()) else { + return Ok(None); + }; + let field = |name: &str| { + state + .get(name) + .ok_or_else(|| StateError::Backend(format!("zone state without `{name}`"))) + }; + let bytes = base64::engine::general_purpose::STANDARD + .decode(field("state_b64")?.as_str().unwrap_or_default()) + .map_err(|_| StateError::Backend("zone state is not base64".into()))?; + Ok(Some(crate::world::StoredZoneState { + token: field("token")?.as_u64().unwrap_or(0), + content_version: field("content_version")? + .as_str() + .unwrap_or_default() + .to_string(), + state: bytes, + })) + }) + } + + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a crate::world::StoredZoneState, + ) -> BoxFuture<'a, Result<(), crate::world::StateError>> { + Box::pin(async move { + use base64::Engine as _; + self.command( + zone_id, + crate::world::zone_state::SAVE, + json!({ + "zone_id": zone_id, + "token": state.token, + "content_version": state.content_version, + "state_b64": base64::engine::general_purpose::STANDARD.encode(&state.state), + }), + ) + .await + }) + } + + fn delete<'a>( + &'a self, + zone_id: &'a str, + token: u64, + ) -> BoxFuture<'a, Result<(), crate::world::StateError>> { + Box::pin(async move { + self.command( + zone_id, + crate::world::zone_state::DELETE, + json!({ "zone_id": zone_id, "token": token }), + ) + .await + }) + } +} + +#[cfg(test)] +mod tests { + use crate::facts::Fact; + use crate::stamp::FactStamp; + use distributed::USER_ID_KEY; + use serde_json::json; + + #[test] + fn the_cell_rebuilds_the_trusted_fact_session_from_the_headers() { + let stamp = FactStamp { + zone_id: "pit-1".into(), + tick: 42, + content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_000, + activation: 3, + config_version: "cfg-2".into(), + seq: 5, + }; + let fact = Fact::trusted( + "k", + "crawler.record_kill", + json!({}), + Some("alice".into()), + stamp.clone(), + ); + // Exactly the headers `commit` sends (besides the internal ones). + let mut headers = std::collections::HashMap::new(); + headers.insert(USER_ID_KEY.to_string(), "alice".to_string()); + for (name, value) in fact.stamp().session_vars() { + headers.insert(name, value); + } + let session = + crate::cell::fact_session_from_headers(|name| headers.get(name).cloned()).unwrap(); + assert_eq!(session.user_id(), Some("alice")); + assert_eq!(crate::cell::session_stamp(&session), Some(stamp)); + headers.remove("x-game-tick"); + assert!( + crate::cell::fact_session_from_headers(|name| headers.get(name).cloned()).is_none() + ); + headers.insert("x-game-tick".into(), "42".into()); + headers.remove(USER_ID_KEY); + assert!( + crate::cell::fact_session_from_headers(|name| headers.get(name).cloned()).is_none() + ); + } +} diff --git a/distributed_game/src/claims.rs b/distributed_game/src/claims.rs new file mode 100644 index 000000000..197b66d16 --- /dev/null +++ b/distributed_game/src/claims.rs @@ -0,0 +1,247 @@ +//! Ticket extension claims: game-declared facts the ticket issuer vouches +//! for (match id, team, party, mode), carried next to the fixed +//! [`TicketClaims`](crate::TicketClaims) fields. +//! +//! The issuer (the web backend, a matchmaker) signs them under the ticket's +//! `ext` object; the zone reads them from +//! [`TicketClaims::extensions`](crate::TicketClaims::extensions) in +//! [`Zone::on_auth`](crate::Zone::on_auth) and from +//! [`PlayerInfo::extensions`](crate::PlayerInfo::extensions) afterwards, typed +//! with [`TicketExtensions::typed`]. Handoff transfer tickets carry them to +//! the next zone unchanged. +//! +//! A game declares the schema once with `#[derive(TicketExtension)]`: +//! +//! ```rust,ignore +//! #[derive(Serialize, Deserialize, TicketExtension)] +//! pub struct MatchClaims { pub match_id: String, pub team: u8 } +//! // net: ServerConfig { ticket_extensions: Some(ExtensionSchema::of::()), .. } +//! ``` +//! +//! The derive refuses identity-, reward- and economy-looking names (the +//! same deny list as `FactCommand`): identity is only ever the verified +//! `sub`, and amounts are decided by command handlers. The `net` server +//! refuses a ticket whose extensions do not match the declared schema +//! (unknown names, wrong types), and any ticket with extensions when no +//! schema is declared. + +use std::collections::BTreeMap; +use std::fmt; + +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::{Map, Value}; + +/// Largest accepted extension object, as compact JSON (bytes). +pub const MAX_EXTENSION_BYTES: usize = 2048; + +/// A game-declared extension claims type (`#[derive(TicketExtension)]`). +pub trait TicketExtension: Serialize + DeserializeOwned { + /// Claim names as serialized (serde renames applied). + const CLAIMS: &'static [&'static str]; +} + +/// Why extension claims were refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExtensionError { + /// `ext` is not a JSON object. + NotAnObject, + /// Larger than [`MAX_EXTENSION_BYTES`]. + TooLarge, + /// A name the declared schema does not know. + Unknown(String), + /// Extensions present but the server declares no schema. + Undeclared, + /// The values do not decode as the declared type. + Invalid(String), +} + +impl fmt::Display for ExtensionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotAnObject => f.write_str("extension claims must be an object"), + Self::TooLarge => write!(f, "extension claims exceed {MAX_EXTENSION_BYTES} bytes"), + Self::Unknown(name) => write!(f, "undeclared extension claim `{name}`"), + Self::Undeclared => f.write_str("this server declares no ticket extension claims"), + Self::Invalid(why) => write!(f, "invalid extension claims: {why}"), + } + } +} + +impl std::error::Error for ExtensionError {} + +/// Verified extension claims (name → JSON value), empty for a ticket +/// without any. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct TicketExtensions(BTreeMap); + +impl TicketExtensions { + /// No extensions. + pub fn new() -> Self { + Self::default() + } + + /// From a typed declaration. + pub fn from_typed(claims: &E) -> Result { + let value = + serde_json::to_value(claims).map_err(|e| ExtensionError::Invalid(e.to_string()))?; + Self::from_json(value) + } + + /// From a JSON object (the ticket's `ext`), size-checked. + pub fn from_json(value: Value) -> Result { + let Value::Object(map) = value else { + return Err(ExtensionError::NotAnObject); + }; + let extensions = Self(map.into_iter().collect()); + if extensions.encoded_len() > MAX_EXTENSION_BYTES { + return Err(ExtensionError::TooLarge); + } + Ok(extensions) + } + + /// As a JSON object (what the ticket signs under `ext`). + pub fn to_json(&self) -> Value { + Value::Object(self.0.clone().into_iter().collect::>()) + } + + fn encoded_len(&self) -> usize { + self.to_json().to_string().len() + } + + /// True when the ticket carried none. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// One raw claim. + pub fn get(&self, name: &str) -> Option<&Value> { + self.0.get(name) + } + + /// All claims, by name. + pub fn iter(&self) -> impl Iterator { + self.0.iter().map(|(k, v)| (k.as_str(), v)) + } + + /// Decode as the declared type. Names `E` does not declare are refused. + pub fn typed(&self) -> Result { + if let Some(unknown) = self + .0 + .keys() + .find(|name| !E::CLAIMS.contains(&name.as_str())) + { + return Err(ExtensionError::Unknown(unknown.clone())); + } + serde_json::from_value(self.to_json()).map_err(|e| ExtensionError::Invalid(e.to_string())) + } +} + +/// The extension schema a server admits ([`ExtensionSchema::of`]). +#[derive(Clone, Copy)] +pub struct ExtensionSchema { + claims: &'static [&'static str], + check: fn(&TicketExtensions) -> Result<(), ExtensionError>, +} + +impl fmt::Debug for ExtensionSchema { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExtensionSchema") + .field("claims", &self.claims) + .finish() + } +} + +impl ExtensionSchema { + /// Admit extensions that decode as `E` (missing optional fields are + /// fine; unknown names are not). + pub fn of() -> Self { + Self { + claims: E::CLAIMS, + check: |ext| ext.typed::().map(drop), + } + } + + /// Declared claim names. + pub fn claims(&self) -> &'static [&'static str] { + self.claims + } + + /// Check verified extensions against the schema. + pub fn validate(&self, extensions: &TicketExtensions) -> Result<(), ExtensionError> { + if extensions.is_empty() { + return Ok(()); + } + (self.check)(extensions) + } +} + +/// Check extensions against an optional server schema: none declared means +/// none admitted. +pub fn validate_extensions( + schema: Option<&ExtensionSchema>, + extensions: &TicketExtensions, +) -> Result<(), ExtensionError> { + match schema { + Some(schema) => schema.validate(extensions), + None if extensions.is_empty() => Ok(()), + None => Err(ExtensionError::Undeclared), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + use serde_json::json; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct MatchClaims { + match_id: String, + #[serde(default)] + team: Option, + } + + impl TicketExtension for MatchClaims { + const CLAIMS: &'static [&'static str] = &["match_id", "team"]; + } + + #[test] + fn typed_round_trip_and_schema_checks() { + let claims = MatchClaims { + match_id: "m-1".into(), + team: Some(2), + }; + let ext = TicketExtensions::from_typed(&claims).unwrap(); + assert_eq!(ext.get("team"), Some(&json!(2))); + assert_eq!(ext.typed::().unwrap(), claims); + let schema = ExtensionSchema::of::(); + schema.validate(&ext).unwrap(); + schema.validate(&TicketExtensions::new()).unwrap(); + + let unknown = TicketExtensions::from_json(json!({"match_id": "m", "gold": 9})).unwrap(); + assert_eq!( + schema.validate(&unknown), + Err(ExtensionError::Unknown("gold".into())) + ); + let mistyped = TicketExtensions::from_json(json!({"match_id": 7})).unwrap(); + assert!(matches!( + schema.validate(&mistyped), + Err(ExtensionError::Invalid(_)) + )); + assert_eq!( + TicketExtensions::from_json(json!(["x"])), + Err(ExtensionError::NotAnObject) + ); + let big = "x".repeat(MAX_EXTENSION_BYTES); + assert_eq!( + TicketExtensions::from_json(json!({ "match_id": big })), + Err(ExtensionError::TooLarge) + ); + assert_eq!( + validate_extensions(None, &ext), + Err(ExtensionError::Undeclared) + ); + validate_extensions(None, &TicketExtensions::new()).unwrap(); + } +} diff --git a/distributed_game/src/clock.rs b/distributed_game/src/clock.rs index 1554b7158..a1b60758c 100644 --- a/distributed_game/src/clock.rs +++ b/distributed_game/src/clock.rs @@ -8,21 +8,32 @@ //! A clock reports elapsed time since its own origin as a [`Duration`]. Only //! differences matter to the runtime; values are also sent to clients as //! "server time" in milliseconds for clock synchronization. +//! +//! [`Clock::unix_ms`] is the clock's wall time (Unix milliseconds), used to +//! stamp facts ([`FactStamp::emitted_at_ms`](crate::FactStamp::emitted_at_ms)) +//! so facts from different zones and hosts can be ordered. It advances with +//! [`now`](Clock::now) and never goes backwards either. use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; /// Monotonic time source. pub trait Clock: Send + Sync + 'static { /// Time elapsed since this clock's origin. Must never go backwards. fn now(&self) -> Duration; + + /// Wall time in Unix milliseconds. Must never go backwards. + fn unix_ms(&self) -> u64; } -/// Real monotonic clock backed by [`Instant`]. +/// Real monotonic clock backed by [`Instant`]. Its wall time is the system +/// time read once at creation plus the monotonic elapsed time, so a system +/// clock step (NTP) never moves fact stamps backwards. #[derive(Debug, Clone)] pub struct SystemClock { origin: Instant, + origin_unix: Duration, } impl SystemClock { @@ -30,6 +41,9 @@ impl SystemClock { pub fn new() -> Self { Self { origin: Instant::now(), + origin_unix: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(), } } } @@ -44,12 +58,19 @@ impl Clock for SystemClock { fn now(&self) -> Duration { self.origin.elapsed() } + + fn unix_ms(&self) -> u64 { + (self.origin_unix + self.origin.elapsed()).as_millis() as u64 + } } -/// Virtual clock advanced explicitly; clones share the same time. +/// Virtual clock advanced explicitly; clones share the same time. Its wall +/// time is a fixed Unix origin (0 unless set with +/// [`with_unix_origin_ms`](Self::with_unix_origin_ms)) plus the virtual time. #[derive(Debug, Clone, Default)] pub struct ManualClock { nanos: Arc, + unix_origin_ms: u64, } impl ManualClock { @@ -58,6 +79,14 @@ impl ManualClock { Self::default() } + /// The same clock reporting wall time `origin_ms + now()` (e.g. two + /// hosts whose wall clocks disagree). Clones made before this call keep + /// their own origin but share the virtual time. + pub fn with_unix_origin_ms(mut self, origin_ms: u64) -> Self { + self.unix_origin_ms = origin_ms; + self + } + /// Move time forward by `by`. pub fn advance(&self, by: Duration) { self.nanos.fetch_add(by.as_nanos() as u64, Ordering::SeqCst); @@ -73,4 +102,35 @@ impl Clock for ManualClock { fn now(&self) -> Duration { Duration::from_nanos(self.nanos.load(Ordering::SeqCst)) } + + fn unix_ms(&self) -> u64 { + self.unix_origin_ms + self.now().as_millis() as u64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manual_wall_time_follows_virtual_time() { + let clock = ManualClock::new().with_unix_origin_ms(1_000); + let shared = clock.clone(); + assert_eq!(clock.unix_ms(), 1_000); + shared.advance(Duration::from_millis(250)); + assert_eq!(clock.unix_ms(), 1_250); + let skewed = shared.clone().with_unix_origin_ms(500); + assert_eq!(skewed.unix_ms(), 750); + } + + #[test] + fn system_wall_time_is_close_to_system_time() { + let clock = SystemClock::new(); + let system = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + assert!(clock.unix_ms().abs_diff(system) < 1_000); + assert!(clock.unix_ms() <= clock.unix_ms()); + } } diff --git a/distributed_game/src/command_sink.rs b/distributed_game/src/command_sink.rs new file mode 100644 index 000000000..da08e751e --- /dev/null +++ b/distributed_game/src/command_sink.rs @@ -0,0 +1,482 @@ +//! [`CommandFactSink`]: a zone server's facts dispatched as commands through +//! any [`CommandHost`] — the same durable aggregates whether they run in +//! celld cells ([`CelldCommandHost`], `POST {celld}/{kind}/{shard}/{command}` +//! over the internal path) or in an ordinary service over any repository +//! ([`LocalCommandHost`]: in-memory, SQLite, Postgres). +//! +//! One sink per zone server routes every fact by its command +//! ([`FactCommand::COMMAND`](crate::FactCommand::COMMAND)) to a registered +//! [`FactRoute`]: a cell kind and a shard function of the fact's input and +//! trusted session (principal + [`FactStamp`](crate::FactStamp)). A host +//! that addresses cells uses both ([`FactRouting`]); a service routes by +//! command and its handler picks the stream from the same session or input. +//! Player lifecycle facts (`zone.*`, [`crate::lifecycle`]) route to the +//! presence aggregate of the player's account ([`CommandFactSink::lifecycle_route`]). +//! +//! - **Exactly one commit per key.** The command id is +//! [`command_id_for`]`(service_id, fact key)`, so a retry after a timeout +//! replays through the aggregate's command ledger (`replayed` receipt). +//! - **Trusted identity.** The principal (bound by the zone host from the +//! verified ticket) is the session's `x-user-id` and the verified +//! principal ([`VerifiedPrincipal::from_trusted_transport`]); the stamp +//! travels as `x-game-*` session variables (forwarded to cells as trusted +//! metadata, [`CelldCommandHost::forward_session_prefix`]). +//! - **No local durable fallback.** A host that is slow or unreachable, a +//! 5xx, or a command still in progress is [`FactError::Retryable`]: the fact +//! stays in the zone's queue (use +//! [`RetryPolicy::until_settled`](crate::RetryPolicy::until_settled)). An +//! unrouted or rejected command is [`FactError::Permanent`]. + +use std::sync::Arc; +use std::time::Duration; + +use distributed::cell_host::{CelldCommandHost, InternalHttpSecret}; +use distributed::command_dispatch::{CommandHost, LocalCommandHost}; +use distributed::graphql::identity::VerifiedPrincipal; +use distributed::microsvc::{CausalDispatchError, Session}; +use serde_json::Value; + +use crate::cell::{command_id_for, session_stamp, trusted_session}; +use crate::facts::{BoxFuture, Fact, FactCommand, FactError, FactReceipt, FactSink}; +use crate::lifecycle::LIFECYCLE_COMMANDS; +use crate::stamp::SESSION_PREFIX; + +type ShardFn = Arc Option + Send + Sync>; + +/// Where facts of one command go: an aggregate kind and a shard function of +/// the fact's input and trusted session. +#[derive(Clone)] +pub struct FactRoute { + command: String, + kind: String, + shard: ShardFn, +} + +impl FactRoute { + /// Route `command` to `{kind}/{shard(input, session)}`. The session holds + /// the fact's principal (`user_id()`) and stamp + /// ([`session_stamp`]). Return `None` when the fact cannot be routed. + pub fn new( + command: impl Into, + kind: impl Into, + shard: impl Fn(&Value, &Session) -> Option + Send + Sync + 'static, + ) -> Self { + Self { + command: command.into(), + kind: kind.into(), + shard: Arc::new(shard), + } + } + + /// Route the typed fact `C` to the aggregate of its principal. + pub fn by_principal(kind: impl Into) -> Self { + Self::new(C::COMMAND, kind, |_, session| { + session.user_id().map(str::to_string) + }) + } + + /// Route the typed fact `C` by a field of its input. + pub fn by_input_field(kind: impl Into, field: &'static str) -> Self { + Self::new(C::COMMAND, kind, move |input, _| { + match input.get(field)? { + Value::String(value) => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + _ => None, + } + }) + } + + /// Route the typed fact `C` to the aggregate of the zone that emitted it. + pub fn by_zone(kind: impl Into) -> Self { + Self::new(C::COMMAND, kind, |_, session| { + session_stamp(session).map(|stamp| stamp.zone_id) + }) + } + + /// The routed command. + pub fn command(&self) -> &str { + &self.command + } + + /// The aggregate kind (cell URL segment). + pub fn kind(&self) -> &str { + &self.kind + } + + /// The shard of a fact with `input` and `session`. + pub fn shard(&self, input: &Value, session: &Session) -> Option { + (self.shard)(input, session) + } +} + +/// A [`CommandHost`] that learns the fact routes it needs. +pub trait FactRouting { + /// Register `route` (a host that addresses aggregates by kind and shard + /// records it; one that routes by command ignores it). + fn register_fact_route(&mut self, route: &FactRoute); +} + +/// A service routes by command: its handler selects the stream. +impl FactRouting for LocalCommandHost { + fn register_fact_route(&mut self, _route: &FactRoute) {} +} + +/// celld addresses cells: `{kind}/{shard}/{command}`. +impl FactRouting for CelldCommandHost { + fn register_fact_route(&mut self, route: &FactRoute) { + let shard = route.shard.clone(); + self.add_cell_route(route.command(), route.kind(), move |input, session| { + shard(input, session) + }); + } +} + +/// See the module docs. +pub struct CommandFactSink { + host: Arc, + service_id: String, + system_principal: String, + routes: Vec, +} + +impl Clone for CommandFactSink { + fn clone(&self) -> Self { + Self { + host: self.host.clone(), + service_id: self.service_id.clone(), + system_principal: self.system_principal.clone(), + routes: self.routes.clone(), + } + } +} + +impl CommandFactSink { + /// A sink dispatching through `host` under command-ledger `service_id`; + /// zone facts without a player use `system_principal`. + pub fn new( + host: H, + service_id: impl Into, + system_principal: impl Into, + ) -> Result { + let service_id = service_id.into(); + let system_principal = system_principal.into(); + if service_id.trim().is_empty() || system_principal.trim().is_empty() { + return Err("a fact sink needs a service id and a system principal".into()); + } + Ok(Self { + host: Arc::new(host), + service_id, + system_principal, + routes: Vec::new(), + }) + } + + /// Add a route (before the sink is cloned). A later route for the same + /// command replaces the earlier. + pub fn route(mut self, route: FactRoute) -> Self { + Arc::get_mut(&mut self.host) + .expect("add fact routes before cloning the sink") + .register_fact_route(&route); + self.routes + .retain(|existing| existing.command != route.command); + self.routes.push(route); + self + } + + /// Route every player lifecycle fact (`zone.*`) to the presence + /// aggregate `{kind}/{principal}` (the player's account). + pub fn lifecycle_route(mut self, kind: impl Into) -> Self { + let kind = kind.into(); + for command in LIFECYCLE_COMMANDS { + self = self.route(FactRoute::new(command, kind.clone(), |_, session| { + session.user_id().map(str::to_string) + })); + } + self + } +} + +impl CommandFactSink { + /// The host. + pub fn host(&self) -> &H { + &self.host + } + + /// The route a fact would take, if any. + pub fn route_for(&self, command: &str) -> Option<&FactRoute> { + self.routes.iter().find(|route| route.command == command) + } + + /// `(principal, trusted session, command id)` of a routed fact. + pub fn prepare(&self, fact: &Fact) -> Result<(String, Session, String), FactError> { + let route = self.route_for(fact.command()).ok_or_else(|| { + FactError::Permanent(format!("no route for fact command `{}`", fact.command())) + })?; + let principal = fact + .principal() + .unwrap_or(&self.system_principal) + .to_string(); + let session = trusted_session(&principal, fact); + route + .shard(fact.input(), &session) + .filter(|shard| { + !shard.is_empty() + && shard.len() <= 256 + && shard.bytes().all(|byte| (0x21..=0x7e).contains(&byte)) + }) + .ok_or_else(|| { + FactError::Permanent(format!( + "fact `{}` has no valid `{}` shard", + fact.key(), + route.kind + )) + })?; + Ok(( + principal, + session, + command_id_for(&self.service_id, fact.key()), + )) + } +} + +/// celld slow, down or still running the command: keep the fact queued. +fn classify(error: CausalDispatchError) -> FactError { + let detail = format!("{}: {error:?}", error.code()); + match &error { + CausalDispatchError::InProgress | CausalDispatchError::Internal(_) => { + FactError::Retryable(detail) + } + CausalDispatchError::Rejected { status, .. } + if *status >= 500 || matches!(*status, 408 | 429) => + { + FactError::Retryable(detail) + } + CausalDispatchError::Handler(handler) if handler.status_code() >= 500 => { + FactError::Retryable(detail) + } + _ => FactError::Permanent(detail), + } +} + +impl FactSink for CommandFactSink { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (principal, session, command_id) = self.prepare(fact)?; + let result = self + .host + .invoke( + fact.command(), + &command_id, + fact.input().clone(), + session, + VerifiedPrincipal::from_trusted_transport(&principal), + None, + ) + .await + .map_err(classify)?; + Ok(FactReceipt { + payload: result.payload().clone(), + replayed: result.replayed(), + }) + }) + } +} + +/// Default per-request timeout of [`CelldFactSink`]. +pub const DEFAULT_CELLD_FACT_TIMEOUT: Duration = Duration::from_secs(10); + +/// Facts into celld cells over the internal path. +pub type CelldFactSink = CommandFactSink; + +impl CommandFactSink { + /// A sink for the cells at `celld_url` under command-ledger + /// `service_id`: a remote-only [`CelldCommandHost`] forwarding the fact + /// stamp (`x-game-*`). + pub fn celld( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + system_principal: impl Into, + ) -> Result { + Self::celld_with_timeout( + celld_url, + secret, + service_id, + system_principal, + DEFAULT_CELLD_FACT_TIMEOUT, + ) + } + + /// As [`celld`](Self::celld) with an explicit per-request timeout. + pub fn celld_with_timeout( + celld_url: impl Into, + secret: InternalHttpSecret, + service_id: impl Into, + system_principal: impl Into, + timeout: Duration, + ) -> Result { + let service_id = service_id.into(); + let host = CelldCommandHost::remote(celld_url, &service_id, secret, timeout) + .map_err(|error| format!("celld command host: {error:?}"))? + .forward_session_prefix(SESSION_PREFIX) + .map_err(|error| format!("celld command host: {error:?}"))?; + Self::new(host, service_id, system_principal) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::stamp::FactStamp; + use serde_json::json; + + const SECRET: &str = "test-only-internal-secret-change-me-2026"; + + #[derive(serde::Serialize)] + struct Kill { + mob: String, + } + impl FactCommand for Kill { + const COMMAND: &'static str = "crawler.record_kill"; + } + #[derive(serde::Serialize)] + struct Result_ { + match_id: String, + } + impl FactCommand for Result_ { + const COMMAND: &'static str = "match.record_result"; + } + + fn sink() -> CelldFactSink { + CelldFactSink::celld( + "http://127.0.0.1:9/", + InternalHttpSecret::new(SECRET).unwrap(), + "zone-server", + "zone:pit-1", + ) + .unwrap() + .route(FactRoute::by_principal::("crawler")) + .route(FactRoute::by_input_field::("match", "match_id")) + .lifecycle_route("presence") + } + + fn fact(command: &str, input: Value, principal: Option<&str>) -> Fact { + Fact::trusted( + format!("{command}:pit-1:1:0"), + command, + input, + principal.map(str::to_string), + FactStamp { + zone_id: "pit-1".into(), + ..FactStamp::default() + }, + ) + } + + fn shard(sink: &CelldFactSink, fact: &Fact) -> Option { + let (_, session, _) = sink.prepare(fact).ok()?; + sink.route_for(fact.command())? + .shard(fact.input(), &session) + } + + #[test] + fn routes_by_command_to_kind_and_shard_with_a_stable_command_id() { + let sink = sink(); + let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); + let (principal, session, id) = sink.prepare(&kill).unwrap(); + assert_eq!(principal, "alice"); + assert_eq!(session.user_id(), Some("alice")); + assert_eq!(session_stamp(&session).unwrap().zone_id, "pit-1"); + assert_eq!(id, command_id_for("zone-server", kill.key())); + assert_eq!(sink.prepare(&kill).unwrap().2, id, "stable across retries"); + assert_eq!(sink.route_for(kill.command()).unwrap().kind(), "crawler"); + assert_eq!(shard(&sink, &kill).as_deref(), Some("alice")); + + let result = fact("match.record_result", json!({ "match_id": "m/7" }), None); + assert_eq!(shard(&sink, &result).as_deref(), Some("m/7")); + assert_eq!( + sink.prepare(&result).unwrap().0, + "zone:pit-1", + "zone facts use the system principal" + ); + let spaced = fact("match.record_result", json!({ "match_id": "m 7" }), None); + assert!(matches!( + sink.prepare(&spaced), + Err(FactError::Permanent(_)) + )); + + for command in LIFECYCLE_COMMANDS { + let lifecycle = fact(command, json!({ "zone": "pit-1" }), Some("alice")); + assert_eq!(sink.route_for(command).unwrap().kind(), "presence"); + assert_eq!(shard(&sink, &lifecycle).as_deref(), Some("alice")); + } + let by_zone = FactRoute::by_zone::("zone-log"); + let (_, session, _) = sink.prepare(&kill).unwrap(); + assert_eq!( + by_zone.shard(&json!({}), &session).as_deref(), + Some("pit-1") + ); + } + + #[test] + fn unknown_commands_and_unroutable_facts_are_permanent() { + let sink = sink(); + let unknown = fact("guild.whatever", json!({}), Some("alice")); + assert!(matches!( + sink.prepare(&unknown), + Err(FactError::Permanent(_)) + )); + let missing = fact("match.record_result", json!({}), None); + assert!(matches!( + sink.prepare(&missing), + Err(FactError::Permanent(_)) + )); + } + + #[test] + fn classifies_dispatch_errors() { + let retryable = |error| matches!(classify(error), FactError::Retryable(_)); + assert!(retryable(CausalDispatchError::InProgress)); + assert!(retryable(CausalDispatchError::Internal("down".into()))); + assert!(retryable(CausalDispatchError::Rejected { + code: "REJECTED", + status: 503, + message: String::new() + })); + assert!(retryable(CausalDispatchError::Rejected { + code: "REJECTED", + status: 429, + message: String::new() + })); + assert!(!retryable(CausalDispatchError::BadRequest("bad".into()))); + assert!(!retryable(CausalDispatchError::Forbidden)); + assert!(!retryable(CausalDispatchError::Rejected { + code: "UNAUTHORIZED", + status: 401, + message: String::new() + })); + assert!(!retryable(CausalDispatchError::Rejected { + code: "REJECTED", + status: 422, + message: String::new() + })); + } + + #[tokio::test] + async fn an_unreachable_host_is_retryable() { + let sink = sink(); + let kill = fact("crawler.record_kill", json!({ "mob": "m1" }), Some("alice")); + assert!(matches!( + sink.commit(&kill).await, + Err(FactError::Retryable(_)) + )); + } + + #[test] + fn rejects_bad_configuration() { + let secret = || InternalHttpSecret::new(SECRET).unwrap(); + assert!(CelldFactSink::celld("ftp://x", secret(), "s", "p").is_err()); + assert!(CelldFactSink::celld("http://x", secret(), " ", "p").is_err()); + assert!(CelldFactSink::celld("http://x", secret(), "s", " ").is_err()); + } +} diff --git a/distributed_game/src/entity.rs b/distributed_game/src/entity.rs new file mode 100644 index 000000000..fbcd7c3a8 --- /dev/null +++ b/distributed_game/src/entity.rs @@ -0,0 +1,245 @@ +//! Zone entities and the views projected from them. +//! +//! A zone is the **aggregate root**: one authoritative boundary that owns +//! its state and changes it only on its own tick. The things inside it +//! (players, mobs, projectiles) are **entities**: server state with an +//! [`EntityId`], including data no client may see (targets, spawn points, +//! credits, principals). What clients receive are **views**, the zone's +//! **read models**: `#[derive(Replicated)]` structs projected from an +//! entity each patch. A view is a projection, not a copy: it may convert +//! (HP to a percent), derive (a telegraph from a timer), flatten nested +//! state, or omit the entity entirely (a dead mob). Durable outcomes leave +//! the zone as **facts**, its **domain events**, committed by cells. +//! +//! | game runtime | business side | +//! |---|---| +//! | zone | aggregate root | +//! | zone entity ([`Entity`]) | entity inside the aggregate | +//! | view ([`Replicated`], [`Project`]) | read model / projection | +//! | fact ([`Fact`](crate::Fact)) | domain event | +//! +//! **Declaring a projection.** A view names its entity type with +//! `#[replicated(.., of = Mob)]` and marks the fields that are straight +//! copies `#[from]` (same name) or `#[from(state.x)]` (a field path on the +//! entity). The derive then generates `MobView::project(&mob, MobViewRest +//! { .. })` where `MobViewRest` holds only the hand-filled fields, and the +//! zone writes the projection once: +//! +//! ```ignore +//! impl Project for Mob { +//! fn view(&self) -> Option { +//! (self.state != MobState::Dead).then(|| { +//! MobView::project(self, MobViewRest { hp: self.hp_percent(), tele: self.tele() }) +//! }) +//! } +//! } +//! ``` +//! +//! A view whose fields are all `#[from]` gets the `Project` impl generated. +//! A view without `of` is unchanged: build it by hand or implement +//! `Project` for any type. +//! +//! **Registering collections.** [`Zone::entities`](crate::Zone::entities) +//! returns the zone's entity collections once, at creation; every patch +//! the host calls [`Project::view`] on each entity (`None` omits it) and +//! then [`Zone::replicate`](crate::Zone::replicate) for anything else: +//! +//! ```ignore +//! fn entities() -> Entities { +//! Entities::new() +//! .each(|z: &Self| &z.players) +//! .each(|z: &Self| &z.mobs) +//! .one(|z: &Self| z) // the zone itself projects its HUD view +//! } +//! ``` +//! +//! Collections are visited in registration order and each in its own order +//! (`Vec`, slices, `VecDeque`, `BTreeMap` values, `Option`): the projected +//! world is deterministic, which the replicated state hash relies on. + +use std::collections::{BTreeMap, VecDeque}; +use std::fmt; + +use crate::authoring::{Pushed, Replicated}; +use crate::replication::WorldView; + +/// Id of a replicated zone entity (unique within the zone). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct EntityId(pub u32); + +impl EntityId { + /// The raw id. + pub const fn get(self) -> u32 { + self.0 + } +} + +impl From for EntityId { + fn from(id: u32) -> Self { + Self(id) + } +} + +impl From for u32 { + fn from(id: EntityId) -> Self { + id.0 + } +} + +impl fmt::Display for EntityId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// A zone entity: anything the zone replicates under its own id. +pub trait Entity { + /// Its id. + fn entity_id(&self) -> EntityId; +} + +/// Projection of a zone entity (or the zone itself) into view `V`. +pub trait Project { + /// The view this patch; `None` leaves the entity out (clients that + /// know it get a remove). + fn view(&self) -> Option; +} + +/// Something holding entities, visited in a deterministic order. +pub trait EntityCollection { + /// Entity type. + type Item; + /// Visit every entity. + fn visit(&self, f: &mut dyn FnMut(&Self::Item)); +} + +impl EntityCollection for [E] { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for Vec { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for [E; N] { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for VecDeque { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +impl EntityCollection for BTreeMap { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.values().for_each(f); + } +} + +impl EntityCollection for Option { + type Item = E; + fn visit(&self, f: &mut dyn FnMut(&E)) { + self.iter().for_each(f); + } +} + +type Projection = Box; + +/// A zone's registered entity collections ([`Zone::entities`](crate::Zone::entities)). +pub struct Entities { + projections: Vec>, +} + +impl Default for Entities { + fn default() -> Self { + Self { + projections: Vec::new(), + } + } +} + +impl fmt::Debug for Entities { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Entities") + .field("collections", &self.projections.len()) + .finish() + } +} + +impl Entities { + /// No collections. + pub fn new() -> Self { + Self::default() + } + + /// Project every entity of a collection into view `V` (inferred when + /// the entity type projects into one view; otherwise + /// `.each::(..)`). + pub fn each(mut self, collection: fn(&Z) -> &C) -> Self + where + V: Replicated, + C: EntityCollection + ?Sized + 'static, + C::Item: Entity + Project, + { + self.projections.push(Box::new(move |zone, out| { + collection(zone).visit(&mut |entity| { + out.project::(entity); + }); + })); + self + } + + /// Project a single entity (the zone itself, `|z| z`, for a HUD view). + pub fn one(mut self, entity: fn(&Z) -> &E) -> Self + where + V: Replicated, + E: Entity + Project + ?Sized + 'static, + { + self.projections.push(Box::new(move |zone, out| { + out.project::(entity(zone)); + })); + self + } + + /// Collections registered. + pub fn len(&self) -> usize { + self.projections.len() + } + + /// True when nothing is registered. + pub fn is_empty(&self) -> bool { + self.projections.is_empty() + } + + /// Push every registered entity's view into `out`. + pub fn project(&self, zone: &Z, out: &mut WorldView) { + for projection in &self.projections { + projection(zone, out); + } + } +} + +impl WorldView { + /// Push `entity`'s view `V` under its id; `None` when the projection + /// omits it. For hand-written [`Zone::replicate`](crate::Zone::replicate). + pub fn project(&mut self, entity: &E) -> Option> + where + V: Replicated, + E: Entity + Project + ?Sized, + { + let view = entity.view()?; + Some(self.add(entity.entity_id(), &view)) + } +} diff --git a/distributed_game/src/facts.rs b/distributed_game/src/facts.rs index 9855304dc..baf00eb59 100644 --- a/distributed_game/src/facts.rs +++ b/distributed_game/src/facts.rs @@ -80,7 +80,8 @@ pub fn fact_key(prefix: &str, zone: &str, epoch: u64, parts: &[&dyn std::fmt::Di /// Trust rules (enforced by the host, not by this trait): the principal is /// bound from the player's verified ticket, never from the input; the input /// reports *what happened* (which mob died) and the command handler decides -/// any reward. The derive rejects identity- and reward-looking field names. +/// any reward or economy amount. The derive rejects identity-, reward- and +/// economy-looking field names (`user_id`, `xp`, `amount`, `gold`, …). pub trait FactCommand: serde::Serialize { /// Durable command name. const COMMAND: &'static str; @@ -259,6 +260,19 @@ impl Default for RetryPolicy { } impl RetryPolicy { + /// Retry retryable failures until the fact settles (backing off up to + /// `max_backoff`): a network sink whose target is down keeps the fact + /// pending in the queue, so the zone sees backpressure at capacity + /// instead of losing durable outcomes. Permanent failures still settle + /// immediately. + pub fn until_settled(base_backoff: Duration, max_backoff: Duration) -> Self { + Self { + max_attempts: u32::MAX, + base_backoff, + max_backoff, + } + } + /// Backoff before attempt `attempt + 1` (attempt >= 1). pub fn backoff(&self, attempt: u32) -> Duration { let factor = 1u32 << attempt.saturating_sub(1).min(20); diff --git a/distributed_game/src/handoff.rs b/distributed_game/src/handoff.rs index e8f67d081..72ae4dfc9 100644 --- a/distributed_game/src/handoff.rs +++ b/distributed_game/src/handoff.rs @@ -9,6 +9,7 @@ //! calls [`Zone::on_handoff_join`](crate::Zone::on_handoff_join) with the //! carried bytes. +use crate::claims::TicketExtensions; use crate::zone::TicketClaims; /// Largest carried state accepted in a handoff (it travels inside the @@ -22,12 +23,18 @@ pub struct HandoffRequest { pub principal: String, /// The player's display name (from its admission ticket), carried over. pub name: Option, + /// The player's ticket extension claims, carried over. + pub extensions: TicketExtensions, /// Zone the player leaves. pub source_zone: String, /// Zone the player goes to. pub target_zone: String, /// Game state carried along (inventory, buffs, position hint...). pub carried: Vec, + /// Source stamp time of the departure (Unix ms): the facts the source + /// emits while the player leaves carry it, and the target stamps + /// everything after it ([`FactStamp`](crate::FactStamp) ordering). + pub departed_at_ms: u64, } /// Verified transfer ticket contents. @@ -43,6 +50,9 @@ pub struct Transfer { pub generation: u64, /// Carried state. pub carried: Vec, + /// Source stamp time of the departure (Unix ms, see + /// [`HandoffRequest::departed_at_ms`]). + pub departed_at_ms: u64, } /// Why [`Ctx::handoff`](crate::Ctx::handoff) refused. diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index a68a42cdb..a2cb7b046 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -26,7 +26,7 @@ //! reported in [`AdvanceReport::transitions`] and via //! [`Zone::on_tier_changed`]. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use std::sync::Arc; use std::time::Duration; @@ -34,8 +34,13 @@ use crate::authoring::GameEvent; use crate::clock::Clock; use crate::facts::{EmitError, Fact, FactCommand, FactQueue, FactSettlement}; use crate::handoff::{HandoffError, HandoffRequest, Transfer, TransferIssuer, MAX_CARRIED_BYTES}; +use crate::inbox::{Delivered, ZoneInbound}; use crate::input::{ConsumePolicy, InputBuffer, PushOutcome, SequencedInput}; use crate::interest::{InterestConfig, View}; +use crate::lifecycle::{ + ConnectionLost, JoinVia, LeftReason, LostCause, PlayerDisconnected, PlayerJoined, PlayerLeft, + PlayerReconnected, +}; use crate::protocol::{error, kick, Notice, Role, ServerMessage, Welcome}; use crate::replication::{ClientReplica, DeltaStats, EncodedWorld, WorldView}; use crate::rewind::{Rewind, RewindConfig, RewindFrame}; @@ -92,6 +97,15 @@ pub struct HostConfig { /// Unix time in microseconds when the host is created. The `net` server /// sets it to the activation's fencing token. pub boot_epoch: u64, + /// Remote config version the zone runs (opaque; e.g. `"combat@12"`): + /// stamped on every fact ([`FactStamp::config_version`]). Change it at + /// runtime with [`ZoneHost::set_config_version`] / + /// [`Ctx::set_config_version`]. + pub config_version: String, + /// Emit the player lifecycle facts ([`crate::lifecycle`]: joined, + /// connection lost, reconnected, disconnected, left) on the zone's fact + /// queue, bound to each player's verified principal. Off by default. + pub lifecycle_facts: bool, } impl Default for HostConfig { @@ -114,6 +128,8 @@ impl Default for HostConfig { rewind: RewindConfig::default(), stale_policy: StalePolicy::Strict, boot_epoch: 0, + config_version: String::new(), + lifecycle_facts: false, } } } @@ -212,6 +228,8 @@ struct PlayerSlot { reconnect_token: String, session_id: String, content_version: String, + /// Lifecycle facts emitted for this slot (part of their keys). + transitions: u32, } struct ObserverSlot { @@ -247,12 +265,24 @@ pub(crate) struct Core { pending_kicks: Vec<(PlayerId, String)>, rewind: Rewind, transfer_issuer: Option>, - pending_handoffs: Vec<(PlayerId, ServerMessage)>, + /// Handoffs to finish after the hook: player, `Handoff` message and the + /// departure stamp time signed into the transfer ticket. + pending_handoffs: Vec<(PlayerId, ServerMessage, u64)>, upward: Vec, child_inbox: Vec, dropped_frozen: u64, /// Typed facts keyed without dedupe parts this tick: `(tick, count)`. fact_seq: (u64, u32), + /// Facts stamped this activation ([`FactStamp::seq`] of the latest). + stamped: u64, + /// Stamps are never earlier than this (Unix ms): raised past a handoff + /// source's departure time when a transfer is admitted. + stamp_floor_ms: u64, + /// Stamp time forced while a handoff departure runs `on_leave`, so the + /// source's departure facts carry exactly the time in the ticket. + pinned_stamp_ms: Option, + /// Lifecycle facts waiting for queue capacity, in emission order. + lifecycle_backlog: VecDeque, } impl Core { @@ -282,7 +312,84 @@ impl Core { } fn in_flight(&self) -> usize { - self.facts.as_ref().map_or(0, FactQueue::in_flight) + self.facts.as_ref().map_or(0, FactQueue::in_flight) + self.lifecycle_backlog.len() + } + + /// Queue a lifecycle fact for `principal`'s slot `player` (transition + /// `n`), keyed restart-safely and stamped now. + fn lifecycle(&mut self, player: PlayerId, principal: &str, n: u32, input: C) { + if !self.config.lifecycle_facts || self.facts.is_none() { + return; + } + let key = crate::facts::fact_key( + C::COMMAND, + &self.zone_id, + self.config.boot_epoch, + &[&format_args!("p{}", player.0), &n], + ); + let Ok(fact) = Fact::typed(key, &input) else { + return; + }; + let fact = fact.bind(Some(principal.to_string()), self.stamp_fact()); + self.lifecycle_backlog.push_back(fact); + self.flush_lifecycle(); + } + + /// Send backlogged lifecycle facts while the queue has room. + fn flush_lifecycle(&mut self) { + let Some(queue) = self.facts.as_mut() else { + self.lifecycle_backlog.clear(); + return; + }; + while let Some(fact) = self.lifecycle_backlog.front() { + match queue.try_send(fact.clone()) { + Err(EmitError::Backpressure) => return, + // Sent, or a duplicate already in flight: done either way. + _ => { + self.lifecycle_backlog.pop_front(); + } + } + } + } + + /// Next transition number of a slot. + fn transition(&mut self, player: PlayerId) -> Option<(String, u32)> { + let slot = self.players.get_mut(&player)?; + slot.transitions += 1; + Some((slot.info.principal.clone(), slot.transitions)) + } + + fn emit_lifecycle(&mut self, player: PlayerId, input: C) { + if !self.config.lifecycle_facts { + return; + } + if let Some((principal, n)) = self.transition(player) { + self.lifecycle(player, &principal, n, input); + } + } + + /// Wall time stamped on anything produced now (Unix ms). + fn stamp_ms(&self) -> u64 { + self.pinned_stamp_ms + .unwrap_or_else(|| self.clock.unix_ms().max(self.stamp_floor_ms)) + } + + fn stamp(&self) -> FactStamp { + FactStamp { + zone_id: self.zone_id.clone(), + tick: self.tick, + content_version: self.config.content_version.clone(), + emitted_at_ms: self.stamp_ms(), + activation: self.config.boot_epoch, + config_version: self.config.config_version.clone(), + seq: self.stamped, + } + } + + /// Stamp for a fact: the next emission sequence number. + fn stamp_fact(&mut self) -> FactStamp { + self.stamped += 1; + self.stamp() } } @@ -332,6 +439,17 @@ impl Ctx<'_, Z> { &self.core.config.content_version } + /// Remote config version ([`HostConfig::config_version`]). + pub fn config_version(&self) -> &str { + &self.core.config.config_version + } + + /// Switch the remote config version (e.g. on a config-changed + /// message); facts and stamps produced afterwards carry it. + pub fn set_config_version(&mut self, version: impl Into) { + self.core.config.config_version = version.into(); + } + /// Epoch of this zone activation ([`HostConfig::boot_epoch`]): unique /// per activation and increasing across restarts and hibernation. /// Tick counts and entity generations restart with the process, so a @@ -509,11 +627,7 @@ impl Ctx<'_, Z> { } fn emit_bound(&mut self, fact: Fact, principal: Option) -> Result<(), EmitError> { - let stamp = FactStamp { - zone_id: self.core.zone_id.clone(), - tick: self.core.tick, - content_version: self.core.config.content_version.clone(), - }; + let stamp = self.core.stamp_fact(); self.core .facts .as_mut() @@ -582,15 +696,11 @@ impl Ctx<'_, Z> { self.core.pending_kicks.push((player, message.into())); } - /// Provenance of anything produced now (zone, tick, content version); - /// embed it in events so clients and logs can tell which content - /// version produced them. + /// Provenance of anything produced now (zone, activation, tick, wall + /// time, content and config version); embed it in events so clients + /// and logs can tell which content version produced them. pub fn stamp(&self) -> FactStamp { - FactStamp { - zone_id: self.core.zone_id.clone(), - tick: self.core.tick, - content_version: self.core.config.content_version.clone(), - } + self.core.stamp() } /// Move `player` to `target_zone` with `carried` state. A signed @@ -611,26 +721,35 @@ impl Ctx<'_, Z> { if carried.len() > MAX_CARRIED_BYTES { return Err(HandoffError::TooLarge); } - let (principal, name) = self + let (principal, name, extensions) = self .core .players .get(&player) - .map(|slot| (slot.info.principal.clone(), slot.info.name.clone())) + .map(|slot| { + ( + slot.info.principal.clone(), + slot.info.name.clone(), + slot.info.extensions.clone(), + ) + }) .ok_or(HandoffError::UnknownPlayer)?; if self .core .pending_handoffs .iter() - .any(|(queued, _)| *queued == player) + .any(|(queued, _, _)| *queued == player) { return Err(HandoffError::Refused); } + let departed_at_ms = self.core.stamp_ms(); let request = HandoffRequest { principal, name, + extensions, source_zone: self.core.zone_id.clone(), target_zone: target_zone.to_string(), carried, + departed_at_ms, }; let issuer = self .core @@ -644,6 +763,7 @@ impl Ctx<'_, Z> { target_zone: request.target_zone, transfer_ticket: ticket, }, + departed_at_ms, )); Ok(()) } @@ -671,6 +791,7 @@ pub struct ZoneHost { zone: Z, core: Core, schema: EntitySchema, + entities: crate::Entities, hot: FixedTimestep, idle: Option, last_advance: Duration, @@ -729,6 +850,10 @@ impl ZoneHost { child_inbox: Vec::new(), dropped_frozen: 0, fact_seq: (0, 0), + stamped: 0, + stamp_floor_ms: 0, + pinned_stamp_ms: None, + lifecycle_backlog: VecDeque::new(), config, }; let zone = Z::on_create(&mut Ctx { core: &mut core }, zone_config); @@ -741,6 +866,7 @@ impl ZoneHost { zone, core, schema: Z::schema(), + entities: Z::entities(), hot, idle, last_advance: now, @@ -866,11 +992,10 @@ impl ZoneHost { /// [`Zone::state_hash`] (or, when the zone has none, its replicated /// [`WorldView`]). pub fn state_hash(&self) -> u64 { - let zone = self.zone.state_hash().unwrap_or_else(|| { - let mut view = WorldView::default(); - self.zone.replicate(&mut view); - crate::replay::hash_world(&view) - }); + let zone = self + .zone + .state_hash() + .unwrap_or_else(|| crate::replay::hash_world(&self.world_view())); let mut h = crate::rng::Fnv64::new(); h.write_u64(self.core.tick) .write_u64(self.core.sim_time.as_nanos() as u64) @@ -922,6 +1047,10 @@ impl ZoneHost { /// Admit a player arriving with a verified transfer ticket; the zone /// sees [`Zone::on_handoff_join`] with the carried state (also when the /// principal is still present, e.g. inside its reconnect grace). + /// + /// From now on this zone stamps facts later than the source stamped the + /// departure ([`Transfer::departed_at_ms`]), whatever the two hosts' + /// clocks say. pub fn join_transfer( &mut self, transfer: &Transfer, @@ -934,6 +1063,10 @@ impl ZoneHost { "transfers are for players", )); } + self.core.stamp_floor_ms = self + .core + .stamp_floor_ms + .max(transfer.departed_at_ms.saturating_add(1)); self.admit( &transfer.claims, capabilities, @@ -1009,11 +1142,29 @@ impl ZoneHost { if claims.name.is_some() { slot.info.name = claims.name.clone(); } + slot.info.extensions = claims.extensions.clone(); } self.bind_player(player, client, capabilities); + let zone = self.core.zone_id.clone(); if let Some((from, carried)) = handoff.take() { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Handoff, + from_zone: Some(from.to_string()), + }, + ); hook!(self.on_handoff_join(player, from, carried)); } else { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Rejoin, + from_zone: None, + }, + ); hook!(self.on_rejoin(player)); } Joined { @@ -1032,6 +1183,7 @@ impl ZoneHost { id: player, principal: claims.principal.clone(), name: claims.name.clone(), + extensions: claims.extensions.clone(), capabilities, connected: true, }, @@ -1042,12 +1194,34 @@ impl ZoneHost { reconnect_token: String::new(), session_id: (self.token_source)(), content_version: content_version.to_string(), + transitions: 0, }, ); self.bind_player(player, client, capabilities); + let zone = self.core.zone_id.clone(); match handoff.take() { - Some((from, carried)) => hook!(self.on_handoff_join(player, from, carried)), - None => hook!(self.on_join(player)), + Some((from, carried)) => { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Handoff, + from_zone: Some(from.to_string()), + }, + ); + hook!(self.on_handoff_join(player, from, carried)); + } + None => { + self.core.emit_lifecycle( + player, + PlayerJoined { + zone, + via: JoinVia::Ticket, + from_zone: None, + }, + ); + hook!(self.on_join(player)); + } } Joined { client, @@ -1064,6 +1238,7 @@ impl ZoneHost { id: observer, principal: claims.principal.clone(), name: claims.name.clone(), + extensions: claims.extensions.clone(), capabilities, }, client, @@ -1147,9 +1322,13 @@ impl ZoneHost { slot.content_version = version.to_string(); } } + let zone = self.core.zone_id.clone(); + self.core.emit_lifecycle(player, PlayerReconnected { zone }); + hook!(self.on_reconnected(player)); if reload { self.reload_player(player); } + self.process_kicks(); Ok(Joined { client, viewer: Viewer::Player(player), @@ -1177,9 +1356,21 @@ impl ZoneHost { self.core.clients.remove(&client); self.core.server_message(client, &message); self.core.outbound.push(Outbound::Close { client }); + self.connection_lost(player, LostCause::Reload); } } + /// Grace started: lifecycle fact, then [`Zone::on_connection_lost`]. + fn connection_lost(&mut self, player: PlayerId, cause: LostCause) { + let lost = ConnectionLost { + zone: self.core.zone_id.clone(), + grace_ms: self.core.config.reconnect_grace.as_millis() as u64, + cause, + }; + self.core.emit_lifecycle(player, lost); + hook!(self.on_connection_lost(player)); + } + /// [`reload_player`](Self::reload_player) every connected player whose /// content version differs from the zone's (hard rollout). Returns the /// number reloaded. @@ -1209,6 +1400,12 @@ impl ZoneHost { .map(|slot| slot.content_version.as_str()) } + /// Switch the remote config version ([`HostConfig::config_version`]); + /// facts and stamps produced afterwards carry it. + pub fn set_config_version(&mut self, version: impl Into) { + self.core.config.config_version = version.into(); + } + /// Switch the zone to a new content version /// ([`Zone::on_content_version_changed`]); facts and stamps produced /// afterwards carry it. @@ -1308,6 +1505,7 @@ impl ZoneHost { p.info.connected = false; p.inputs.clear(); p.grace_until = Some(self.core.clock.now() + self.core.config.reconnect_grace); + self.connection_lost(player, LostCause::Dropped); } } Viewer::Observer(observer) => self.remove_observer(observer, None), @@ -1365,6 +1563,48 @@ impl ZoneHost { self.send_event(to, event.event_id(), event.to_payload()); } + /// Deliver a service message ([`crate::inbox`]): a kick removes every + /// connection of the principal (players leave with + /// [`LeaveReason::Kicked`]), a game message runs + /// [`Zone::on_message`]. Call between ticks (the host is not ticking + /// while the caller holds `&mut self`). + pub fn deliver(&mut self, inbound: ZoneInbound) -> Delivered { + let mut delivered = Delivered::default(); + match inbound { + ZoneInbound::Kick { principal, reason } => { + let players: Vec<_> = self + .core + .players + .values() + .filter(|slot| slot.info.principal == principal) + .map(|slot| slot.info.id) + .collect(); + delivered.players_kicked = players.len(); + for player in players { + self.core.pending_kicks.push((player, reason.clone())); + } + let observers: Vec<_> = self + .core + .observers + .values() + .filter(|slot| slot.info.principal == principal) + .map(|slot| slot.info.id) + .collect(); + delivered.observers_kicked = observers.len(); + let bye = ServerMessage::Kick { + code: kick::BY_ZONE, + message: reason, + }; + for observer in observers { + self.remove_observer(observer, Some(&bye)); + } + } + ZoneInbound::Message(message) => hook!(self.on_message(message)), + } + self.process_kicks(); + delivered + } + /// Kick a player now. pub fn kick(&mut self, player: PlayerId, message: impl Into) { self.core.pending_kicks.push((player, message.into())); @@ -1486,6 +1726,19 @@ impl ZoneHost { self.core.outbound.push(Outbound::Close { client }); } hook!(self.on_leave(player, reason)); + if self.core.config.lifecycle_facts { + let zone = self.core.zone_id.clone(); + let (principal, n) = (slot.info.principal, slot.transitions + 1); + match LeftReason::from_leave(reason) { + Some(reason) => { + self.core + .lifecycle(player, &principal, n, PlayerLeft { zone, reason }) + } + None => self + .core + .lifecycle(player, &principal, n, PlayerDisconnected { zone }), + } + } } fn remove_observer(&mut self, observer: ObserverId, last: Option<&ServerMessage>) { @@ -1505,8 +1758,10 @@ impl ZoneHost { } fn process_kicks(&mut self) { - while let Some((player, message)) = self.core.pending_handoffs.pop() { + while let Some((player, message, departed_at_ms)) = self.core.pending_handoffs.pop() { + self.core.pinned_stamp_ms = Some(departed_at_ms); self.remove_player(player, LeaveReason::Handoff, Some(message)); + self.core.pinned_stamp_ms = None; } while let Some((player, message)) = self.core.pending_kicks.pop() { let kick = ServerMessage::Kick { @@ -1619,9 +1874,17 @@ impl ZoneHost { } } - fn patch(&mut self) { + /// This patch's replicated world: every registered entity collection's + /// projected views ([`Zone::entities`]), then [`Zone::replicate`]. + pub fn world_view(&self) -> WorldView { let mut view = WorldView::default(); + self.entities.project(&self.zone, &mut view); self.zone.replicate(&mut view); + view + } + + fn patch(&mut self) { + let view = self.world_view(); let core = &mut self.core; self.pending.entities = Some(view.entities().len()); let world = EncodedWorld::build(&self.schema, view, &core.config.interest); @@ -1670,6 +1933,7 @@ impl ZoneHost { let now = self.core.clock.now(); let elapsed = now.saturating_sub(self.last_advance); self.last_advance = now; + self.core.flush_lifecycle(); let expired: Vec<_> = self .core diff --git a/distributed_game/src/inbox.rs b/distributed_game/src/inbox.rs new file mode 100644 index 000000000..89e795f0b --- /dev/null +++ b/distributed_game/src/inbox.rs @@ -0,0 +1,69 @@ +//! The zone inbox: typed messages from services into a running zone. +//! +//! A zone declares [`Zone::Message`](crate::Zone::Message) — a +//! `#[derive(GameEvent)]` struct or enum, so messages have stable type ids +//! and the same typed binary payload codec as events (and appear in the +//! zone's [`GameSurface`](crate::GameSurface) under `messages`) — and +//! handles them in [`Zone::on_message`](crate::Zone::on_message). Zones that +//! take none use [`NoMessages`]. +//! +//! Delivery ([`ZoneHost::deliver`](crate::ZoneHost::deliver)) runs on the +//! zone's own task between ticks, never concurrently with one. The host +//! handles the built-in [`ZoneInbound::Kick`] itself (the player leaves with +//! `LeaveReason::Kicked` and, with lifecycle facts on, a `PlayerLeft` +//! fact). Over the network: `GameServer::send` locally, `POST +//! /admin/zone-message` for peers and services (admin secret), and +//! `net::ZoneInboxClient`, which finds the owning host through the +//! `ZoneDirectory` and follows `wrong_owner` answers. A message for a zone +//! that is not active is refused; it never activates the zone. + +use crate::authoring::GameEvent; +use crate::protocol::wire::{DecodeError, Writer}; +use crate::surface::{EventSurface, RecordSurface}; + +/// What a service can deliver into a zone. +#[derive(Debug, Clone, PartialEq)] +pub enum ZoneInbound { + /// Remove every connection of `principal` (players leave with + /// `LeaveReason::Kicked`, observers are unsubscribed). Clients get + /// `Kick(BY_ZONE, reason)`. + Kick { + /// Verified principal to remove (a service decision, e.g. a ban). + principal: String, + /// Shown to the client. + reason: String, + }, + /// A game message for [`Zone::on_message`](crate::Zone::on_message). + Message(M), +} + +/// Outcome of a delivery. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Delivered { + /// Players removed by a kick. + pub players_kicked: usize, + /// Observers removed by a kick. + pub observers_kicked: usize, +} + +/// The message type of a zone that takes none. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NoMessages {} + +impl GameEvent for NoMessages { + fn event_id(&self) -> u32 { + match *self {} + } + + fn encode_payload(&self, _w: &mut Writer) { + match *self {} + } + + fn decode_event(_type_id: u32, _payload: &[u8]) -> Result, DecodeError> { + Ok(None) + } + + fn event_surfaces(_records: &mut Vec) -> Vec { + Vec::new() + } +} diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index a5670bdbe..721d48d87 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -54,13 +54,21 @@ pub mod authoring; pub mod bots; #[cfg(feature = "cell")] pub mod cell; +#[cfg(feature = "celld")] +pub mod celld; +pub mod claims; pub mod clock; pub mod codec; +#[cfg(feature = "command-host")] +pub mod command_sink; +pub mod entity; pub mod facts; pub mod handoff; pub mod host; +pub mod inbox; pub mod input; pub mod interest; +pub mod lifecycle; #[cfg(feature = "metrics")] pub mod metrics; #[cfg(feature = "net")] @@ -81,12 +89,14 @@ pub mod world; pub mod zone; pub use authoring::{GameEvent, GameInput, Pushed, Replicated}; +pub use claims::{ExtensionError, ExtensionSchema, TicketExtension, TicketExtensions}; pub use clock::{Clock, ManualClock, SystemClock}; pub use codec::{Wire, WireType}; /// Authoring derives (see the crate docs and README). pub use distributed_game_macros::{ - game_pure, predict, FactCommand, GameEvent, GameInput, GameRecord, Replicated, + game_pure, predict, FactCommand, GameEvent, GameInput, GameRecord, Replicated, TicketExtension, }; +pub use entity::{Entities, Entity, EntityCollection, EntityId, Project}; #[cfg(feature = "runtime")] pub use facts::spawn_fact_worker; pub use facts::{ @@ -96,6 +106,7 @@ pub use facts::{ }; pub use handoff::{HandoffError, HandoffRequest, Transfer}; pub use host::{AdvanceReport, Ctx, FrameStats, HostConfig, JoinError, Joined, Outbound, ZoneHost}; +pub use inbox::{Delivered, NoMessages, ZoneInbound}; pub use input::{ConsumePolicy, InputBuffer, InputCredits, PushOutcome, SequencedInput}; pub use interest::{InterestConfig, View}; pub use prediction::{PredictContext, PredictStep}; diff --git a/distributed_game/src/lifecycle.rs b/distributed_game/src/lifecycle.rs new file mode 100644 index 000000000..f51c2f055 --- /dev/null +++ b/distributed_game/src/lifecycle.rs @@ -0,0 +1,209 @@ +//! Player lifecycle facts emitted by the host itself. +//! +//! With [`HostConfig::lifecycle_facts`](crate::HostConfig::lifecycle_facts) +//! on, every player transition becomes a durable fact on the zone's +//! [`FactQueue`](crate::FactQueue), bound to the player's **verified +//! principal** (never to anything a zone or client wrote): +//! +//! | Fact | Command | When | +//! |---|---|---| +//! | [`PlayerJoined`] | [`PLAYER_JOINED`] | admitted with a ticket, a fresh ticket for a held slot (rejoin) or a transfer ticket (handoff) | +//! | [`ConnectionLost`] | [`CONNECTION_LOST`] | the connection dropped (or was told to reload); the reconnect grace starts | +//! | [`PlayerReconnected`] | [`PLAYER_RECONNECTED`] | a reconnect token resumed the slot within the grace | +//! | [`PlayerDisconnected`] | [`PLAYER_DISCONNECTED`] | the grace expired; the slot is gone | +//! | [`PlayerLeft`] | [`PLAYER_LEFT`] | the slot was removed for any other [`LeftReason`] | +//! +//! Every slot ends with exactly one of [`PlayerLeft`] / [`PlayerDisconnected`]. +//! +//! **Keys** are `"{command}:{zone}:{boot_epoch}:p{slot}:{n}"` where `n` +//! counts the slot's transitions: computed once at emission (a retry keeps +//! it), never repeated within an activation, and distinct after a restart +//! (new boot epoch). **Order** per principal follows emission: the facts +//! travel in the principal's lane of the fact worker (also with +//! `max_concurrent_commits > 1`), and when the queue is full the host keeps +//! them in a backlog and sends them, in order, as capacity frees up — a +//! lifecycle fact is never dropped for backpressure. A `PlayerJoined` is +//! queued before anything the zone emits in `on_join`, a `PlayerLeft` after +//! `on_leave`. A handoff departure is stamped with the time in the transfer +//! ticket, so the target's `PlayerJoined` sorts after the source's +//! `PlayerLeft` ([`FactStamp::emission_order`](crate::FactStamp::emission_order)). + +use serde::{Deserialize, Serialize}; + +use crate::zone::LeaveReason; + +/// Command of [`PlayerJoined`]. +pub const PLAYER_JOINED: &str = "zone.player_joined"; +/// Command of [`PlayerLeft`]. +pub const PLAYER_LEFT: &str = "zone.player_left"; +/// Command of [`ConnectionLost`]. +pub const CONNECTION_LOST: &str = "zone.connection_lost"; +/// Command of [`PlayerReconnected`]. +pub const PLAYER_RECONNECTED: &str = "zone.player_reconnected"; +/// Command of [`PlayerDisconnected`]. +pub const PLAYER_DISCONNECTED: &str = "zone.player_disconnected"; + +/// Every lifecycle command, in the table's order. +pub const LIFECYCLE_COMMANDS: [&str; 5] = [ + PLAYER_JOINED, + CONNECTION_LOST, + PLAYER_RECONNECTED, + PLAYER_DISCONNECTED, + PLAYER_LEFT, +]; + +/// How a player entered the zone. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JoinVia { + /// An admission ticket for a new slot. + Ticket, + /// A fresh ticket for a slot the principal already held (page reload, + /// second tab). + Rejoin, + /// A handoff transfer ticket. + Handoff, +} + +/// Player admitted. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_JOINED)] +pub struct PlayerJoined { + /// Zone id. + pub zone: String, + /// Ticket, rejoin or handoff. + pub via: JoinVia, + /// Source zone of a handoff. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_zone: Option, +} + +/// Why a slot was removed (a grace expiry is [`PlayerDisconnected`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LeftReason { + /// The client left gracefully. + Left, + /// Kicked by the zone or the server (incl. the zone inbox). + Kicked, + /// Moved to another zone. + Handoff, + /// The zone was disposed (shutdown, hibernation with players is + /// impossible, ownership loss). + Disposed, +} + +impl LeftReason { + /// The fact reason for a host leave reason (`None` for `Disconnected`, + /// which is its own fact, and `Rejoined`, which keeps the slot). + pub fn from_leave(reason: LeaveReason) -> Option { + match reason { + LeaveReason::Left => Some(Self::Left), + LeaveReason::Kicked => Some(Self::Kicked), + LeaveReason::Handoff => Some(Self::Handoff), + LeaveReason::Disposed => Some(Self::Disposed), + LeaveReason::Disconnected | LeaveReason::Rejoined => None, + } + } +} + +/// Slot removed (terminal). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_LEFT)] +pub struct PlayerLeft { + /// Zone id. + pub zone: String, + /// Why. + pub reason: LeftReason, +} + +/// What ended the connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LostCause { + /// The transport dropped. + Dropped, + /// The server told the client to reload its content (`Kick(RELOAD)`). + Reload, +} + +/// Connection lost; the slot is kept for `grace_ms`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = CONNECTION_LOST)] +pub struct ConnectionLost { + /// Zone id. + pub zone: String, + /// Reconnect grace (ms). + pub grace_ms: u64, + /// Dropped or told to reload. + pub cause: LostCause, +} + +/// Slot resumed with a reconnect token. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_RECONNECTED)] +pub struct PlayerReconnected { + /// Zone id. + pub zone: String, +} + +/// The reconnect grace expired; the slot is gone (terminal). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, crate::FactCommand)] +#[fact(command = PLAYER_DISCONNECTED)] +pub struct PlayerDisconnected { + /// Zone id. + pub zone: String, +} + +/// Any lifecycle fact input, decoded from a [`Fact`](crate::Fact) or +/// [`FactSettlement`](crate::FactSettlement) by command name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Lifecycle { + /// [`PlayerJoined`]. + Joined(PlayerJoined), + /// [`ConnectionLost`]. + ConnectionLost(ConnectionLost), + /// [`PlayerReconnected`]. + Reconnected(PlayerReconnected), + /// [`PlayerDisconnected`]. + Disconnected(PlayerDisconnected), + /// [`PlayerLeft`]. + Left(PlayerLeft), +} + +impl Lifecycle { + /// Decode `input` of `command`; `None` for other commands or a + /// malformed input. + pub fn decode(command: &str, input: &serde_json::Value) -> Option { + let input = input.clone(); + Some(match command { + PLAYER_JOINED => Self::Joined(serde_json::from_value(input).ok()?), + CONNECTION_LOST => Self::ConnectionLost(serde_json::from_value(input).ok()?), + PLAYER_RECONNECTED => Self::Reconnected(serde_json::from_value(input).ok()?), + PLAYER_DISCONNECTED => Self::Disconnected(serde_json::from_value(input).ok()?), + PLAYER_LEFT => Self::Left(serde_json::from_value(input).ok()?), + _ => return None, + }) + } + + /// Decode a fact. + pub fn of(fact: &crate::Fact) -> Option { + Self::decode(fact.command(), fact.input()) + } + + /// Zone id. + pub fn zone(&self) -> &str { + match self { + Self::Joined(f) => &f.zone, + Self::ConnectionLost(f) => &f.zone, + Self::Reconnected(f) => &f.zone, + Self::Disconnected(f) => &f.zone, + Self::Left(f) => &f.zone, + } + } + + /// True for the terminal facts (the slot is gone). + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Disconnected(_) | Self::Left(_)) + } +} diff --git a/distributed_game/src/metrics.rs b/distributed_game/src/metrics.rs index 00595189d..b4093ff71 100644 --- a/distributed_game/src/metrics.rs +++ b/distributed_game/src/metrics.rs @@ -37,6 +37,7 @@ //! | `distributed_game_zone_facts_pending` | gauge | zone | //! | `distributed_game_zone_facts_settled_total` | counter | zone, outcome | //! | `distributed_game_hibernation_saves_total` | counter | outcome (`saved`, `stale`, `error`) | +//! | `distributed_game_activations_refused_total` | counter | reason (`not_found`, `unavailable`) | use crate::facts::FactOutcome; use crate::host::{AdvanceReport, ZoneHost}; @@ -163,11 +164,16 @@ struct State { other: Counters, /// Server-wide (zones are forgotten when they hibernate). hibernation_saves: BTreeMap<&'static str, u64>, + /// Server-wide activations refused before the zone ran. + activations_refused: BTreeMap<&'static str, u64>, } /// Hibernation snapshot save outcome label values (fixed set). pub const HIBERNATION_OUTCOMES: [&str; 3] = ["saved", "stale", "error"]; +/// Refused activation reason label values (fixed set). +pub const ACTIVATION_REFUSALS: [&str; 2] = ["not_found", "unavailable"]; + impl State { fn zone(&mut self, zone: &str, max: usize) -> &mut ZoneSeries { if !self.zones.contains_key(zone) { @@ -294,6 +300,26 @@ impl GameMetrics { } } + /// A zone activation refused before the zone ran, by protocol error + /// code (`NOT_FOUND` → `not_found`, anything else → `unavailable`). + pub fn record_activation_refused(&self, code: u32) { + let label = if code == crate::protocol::error::NOT_FOUND { + "not_found" + } else { + "unavailable" + }; + *self.lock().activations_refused.entry(label).or_insert(0) += 1; + } + + /// Refused activations recorded with `reason`. + pub fn activations_refused(&self, reason: &str) -> u64 { + self.lock() + .activations_refused + .get(reason) + .copied() + .unwrap_or(0) + } + /// Hibernation saves recorded with `outcome`. pub fn hibernation_saves(&self, outcome: &str) -> u64 { self.lock() @@ -502,6 +528,19 @@ impl GameMetrics { "distributed_game_hibernation_saves_total{{outcome=\"{label}\"}} {v}" ); } + header( + &mut out, + "activations_refused_total", + "counter", + "Zone activations refused before the zone ran (unknown zone, factory or allocation failure).", + ); + for label in ACTIVATION_REFUSALS { + let v = state.activations_refused.get(label).copied().unwrap_or(0); + let _ = writeln!( + out, + "distributed_game_activations_refused_total{{reason=\"{label}\"}} {v}" + ); + } out } } @@ -603,6 +642,7 @@ mod tests { type Config = (); type Input = u8; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { EntitySchema::new().kind(KindDef::new(1, "dot").field("v", FieldType::U8)) @@ -641,6 +681,7 @@ mod tests { nonce: "n".into(), role: Role::Player, name: None, + extensions: Default::default(), }; let joined = h.join(&claims, 0, "").unwrap(); metrics.record_input("z1", h.push_input(joined.client, 1, 0, &[1])); diff --git a/distributed_game/src/net/inbox.rs b/distributed_game/src/net/inbox.rs new file mode 100644 index 000000000..85290e356 --- /dev/null +++ b/distributed_game/src/net/inbox.rs @@ -0,0 +1,300 @@ +//! Zone inbox over the network: [`GameServer::send`](super::GameServer::send) +//! locally, `POST /admin/zone-message` for peers and services, and +//! [`ZoneInboxClient`] for services (see [`crate::inbox`]). +//! +//! `POST /admin/zone-message` (header `x-game-admin-secret`), JSON body: +//! +//! ```text +//! {"zone": "pit-1", "kick": {"principal": "u-42", "reason": "banned"}} +//! {"zone": "pit-1", "message": {"type_id": 3, "payload": ""}} +//! ``` +//! +//! | Status | Body | Meaning | +//! |---|---|---| +//! | 200 | `{ok, players_kicked, observers_kicked}` | delivered on the zone task | +//! | 400 | `{error}` | malformed body, unknown message id or bad payload | +//! | 401 | `{error: "unauthorized"}` | missing / wrong admin secret (or none configured) | +//! | 404 | `{error: "inactive"}` | the zone is not running here (never activated by a message) | +//! | 421 | `{error: "wrong_owner", owner, admin_url}` | another host owns the zone (directory) | + +use std::sync::Arc; +use std::time::Duration; + +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use serde_json::{json, Value}; + +use super::peer::post_json; +use crate::authoring::GameEvent; +use crate::inbox::{Delivered, ZoneInbound}; +use crate::world::{HostAddr, ZoneDirectory}; + +/// Route path. +pub const ZONE_MESSAGE_PATH: &str = "/admin/zone-message"; + +/// Why a message was not delivered. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InboxError { + /// The zone is not active on the host asked (it is not activated). + Inactive, + /// Another host owns the zone. + WrongOwner(HostAddr), + /// Malformed message (unknown id, payload that does not decode). + Invalid(String), + /// The admin secret was refused. + Unauthorized, + /// Transport failure or an unexpected answer. + Unavailable(String), +} + +impl std::fmt::Display for InboxError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Inactive => f.write_str("zone not active"), + Self::WrongOwner(host) => write!(f, "zone owned by {}", host.id), + Self::Invalid(why) => write!(f, "invalid zone message: {why}"), + Self::Unauthorized => f.write_str("admin secret refused"), + Self::Unavailable(why) => write!(f, "zone inbox unavailable: {why}"), + } + } +} + +impl std::error::Error for InboxError {} + +/// A zone message on the wire between tasks and hosts: decoded into +/// `Z::Message` on the zone's own task. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum WireInbound { + Kick { principal: String, reason: String }, + Message { type_id: u32, payload: Vec }, +} + +impl WireInbound { + pub(crate) fn of(inbound: &ZoneInbound) -> Self { + match inbound { + ZoneInbound::Kick { principal, reason } => Self::Kick { + principal: principal.clone(), + reason: reason.clone(), + }, + ZoneInbound::Message(message) => Self::Message { + type_id: message.event_id(), + payload: message.to_payload(), + }, + } + } + + pub(crate) fn decode(self) -> Result, InboxError> { + match self { + Self::Kick { principal, reason } => { + if principal.is_empty() { + return Err(InboxError::Invalid("kick needs a principal".into())); + } + Ok(ZoneInbound::Kick { principal, reason }) + } + Self::Message { type_id, payload } => match M::decode_event(type_id, &payload) { + Ok(Some(message)) => Ok(ZoneInbound::Message(message)), + Ok(None) => Err(InboxError::Invalid(format!( + "this zone declares no message {type_id}" + ))), + Err(e) => Err(InboxError::Invalid(format!("message {type_id}: {e:?}"))), + }, + } + } + + pub(crate) fn to_json(&self, zone: &str) -> Value { + match self { + Self::Kick { principal, reason } => { + json!({ "zone": zone, "kick": { "principal": principal, "reason": reason } }) + } + Self::Message { type_id, payload } => json!({ + "zone": zone, + "message": { "type_id": type_id, "payload": URL_SAFE_NO_PAD.encode(payload) }, + }), + } + } + + /// `(zone, inbound)` from a route body. + pub(crate) fn from_json(body: &Value) -> Result<(String, Self), String> { + let zone = body + .get("zone") + .and_then(Value::as_str) + .filter(|z| !z.is_empty()) + .ok_or("zone")? + .to_string(); + let text = |v: &Value, name: &str| v.get(name).and_then(Value::as_str).map(str::to_string); + let inbound = match (body.get("kick"), body.get("message")) { + (Some(kick), None) => Self::Kick { + principal: text(kick, "principal").ok_or("kick.principal")?, + reason: text(kick, "reason").unwrap_or_default(), + }, + (None, Some(message)) => Self::Message { + type_id: message + .get("type_id") + .and_then(Value::as_u64) + .and_then(|id| u32::try_from(id).ok()) + .ok_or("message.type_id")?, + payload: URL_SAFE_NO_PAD + .decode(text(message, "payload").ok_or("message.payload")?) + .map_err(|_| "message.payload")?, + }, + _ => return Err("exactly one of `kick` or `message`".into()), + }; + Ok((zone, inbound)) + } +} + +pub(crate) fn delivered_json(delivered: Delivered) -> Value { + json!({ + "ok": true, + "players_kicked": delivered.players_kicked, + "observers_kicked": delivered.observers_kicked, + }) +} + +/// Sends zone messages from a service: finds the owning host through the +/// [`ZoneDirectory`], posts to its `/admin/zone-message` and follows +/// `wrong_owner` answers (ownership moved) a bounded number of times. +#[derive(Clone)] +pub struct ZoneInboxClient { + directory: Arc, + secret: String, + timeout: Duration, + max_redirects: u32, +} + +impl std::fmt::Debug for ZoneInboxClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ZoneInboxClient") + .field("timeout", &self.timeout) + .finish_non_exhaustive() + } +} + +impl ZoneInboxClient { + /// Client over `directory` with the hosts' shared admin secret. + pub fn new(directory: Arc, admin_secret: impl Into) -> Self { + Self { + directory, + secret: admin_secret.into(), + timeout: Duration::from_secs(3), + max_redirects: 3, + } + } + + /// Per-request timeout (default 3 s). + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + /// Deliver a typed game message to `zone`. + pub async fn send(&self, zone: &str, message: &M) -> Result<(), InboxError> { + self.post( + zone, + WireInbound::Message { + type_id: message.event_id(), + payload: message.to_payload(), + }, + ) + .await + .map(drop) + } + + /// Remove every connection of `principal` from `zone`. + pub async fn kick( + &self, + zone: &str, + principal: &str, + reason: &str, + ) -> Result { + self.post( + zone, + WireInbound::Kick { + principal: principal.into(), + reason: reason.into(), + }, + ) + .await + } + + async fn post(&self, zone: &str, inbound: WireInbound) -> Result { + let body = inbound.to_json(zone); + let mut admin_url = self + .directory + .locate(zone) + .ok_or(InboxError::Inactive)? + .admin_url; + for _ in 0..=self.max_redirects { + let (status, reply) = post_json( + &admin_url, + ZONE_MESSAGE_PATH, + &self.secret, + &body, + self.timeout, + ) + .await + .map_err(InboxError::Unavailable)?; + let error = || { + reply + .get("error") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + match status { + 200 => { + let count = + |name: &str| reply.get(name).and_then(Value::as_u64).unwrap_or(0) as usize; + return Ok(Delivered { + players_kicked: count("players_kicked"), + observers_kicked: count("observers_kicked"), + }); + } + 421 => { + admin_url = reply + .get("admin_url") + .and_then(Value::as_str) + .ok_or_else(|| { + InboxError::Unavailable("wrong_owner without admin_url".into()) + })? + .to_string(); + } + 400 => return Err(InboxError::Invalid(error())), + 401 => return Err(InboxError::Unauthorized), + 404 => return Err(InboxError::Inactive), + other => return Err(InboxError::Unavailable(format!("{other}: {}", error()))), + } + } + Err(InboxError::Unavailable( + "too many wrong_owner redirects".into(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn route_bodies_round_trip() { + for inbound in [ + WireInbound::Kick { + principal: "u-1".into(), + reason: "banned".into(), + }, + WireInbound::Message { + type_id: 7, + payload: vec![1, 2, 255], + }, + ] { + let (zone, back) = WireInbound::from_json(&inbound.to_json("pit-1")).unwrap(); + assert_eq!((zone.as_str(), back), ("pit-1", inbound)); + } + assert!(WireInbound::from_json(&json!({"zone": "z", "kick": {}})).is_err()); + assert!(WireInbound::from_json(&json!({"zone": "z"})).is_err()); + assert!(WireInbound::from_json( + &json!({"zone": "z", "kick": {"principal": "p"}, "message": {"type_id": 1, "payload": ""}}) + ) + .is_err()); + } +} diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index ff5827c08..dbc7e22fb 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -1,8 +1,13 @@ //! WebSocket transport (`net` feature): one axum server hosting many zones. //! //! - Route: `GET /zones/{zone_id}` upgrades to a WebSocket. Zones are created -//! on first use through the game's factory and each runs on its own Tokio -//! task driving a [`ZoneHost`]. +//! on first use through the game's async [`ZoneFactory`] and each runs on +//! its own Tokio task driving a [`ZoneHost`]. Creation (ownership claim, +//! snapshot, [`ZoneAllocation`](crate::world::ZoneAllocation) record, +//! factory) happens before the first tick; joins that arrive meanwhile +//! wait and are admitted afterwards, or answered `Error(NOT_FOUND)` (the +//! factory knows no such zone) / `Error(ZONE_UNAVAILABLE)` (it failed or +//! timed out). //! - Binary frames only (one protocol message per frame); text frames get //! `Error(BAD_MESSAGE)` and are closed. //! - Admission: the first frame must be Hello within `hello_timeout`. A fresh @@ -38,6 +43,7 @@ //! - **Barrier**: [`GameServer::prepare_barrier`] / `commit_barrier` / //! `abort_barrier`, driven by a [`world::WorldCoordinator`]. +pub mod inbox; pub mod peer; pub mod ticket; pub mod transfer; @@ -61,6 +67,7 @@ use tokio::task::JoinHandle; use tokio::time::Instant; pub use crate::facts::BoxFuture; +pub use inbox::{InboxError, ZoneInboxClient, ZONE_MESSAGE_PATH}; pub use ticket::{ decode_hs256, NonceCache, SharedTicketVerifier, TicketError, TicketKey, TicketVerifier, VerifiedTicket, @@ -80,8 +87,8 @@ use crate::protocol::{error, kick, ClientMessage, Notice, ServerMessage, PROTOCO use crate::rng::ZoneRng; use crate::world::{ BarrierMode, ChildEvent, DottedTopology, FreezeState, HostAddr, LocalOwnership, OwnershipError, - StalePolicy, StateError, StoredZoneState, ZoneClaim, ZoneDirectory, ZoneOwnership, - ZoneStateStore, ZoneTopology, + StalePolicy, StateError, StoredZoneState, ZoneActivation, ZoneAllocationStore, ZoneClaim, + ZoneCreateError, ZoneDirectory, ZoneOwnership, ZoneStateStore, ZoneTopology, }; use crate::zone::{ClientId, TicketClaims, Tier, Zone}; @@ -174,6 +181,12 @@ pub struct ServerConfig { /// `None` = HS256 tickets signed with `ticket_key`. Expiry, route zone /// and single use are enforced by the server either way. pub ticket_verifier: Option, + /// Extension claims tickets may carry + /// ([`ExtensionSchema::of::()`](crate::ExtensionSchema::of) for a + /// `#[derive(TicketExtension)]` type). `None` = a ticket with any `ext` + /// claims is refused. Applies to join (any verifier) and transfer + /// tickets. + pub ticket_extensions: Option, /// Deadline for the first (Hello) frame. pub hello_timeout: Duration, /// Sustained inputs per second per connection. @@ -251,12 +264,19 @@ impl ServerConfig { self } + /// Admit ticket extension claims of type `E`. + pub fn with_ticket_extensions(mut self) -> Self { + self.ticket_extensions = Some(crate::claims::ExtensionSchema::of::()); + self + } + /// Defaults with the given ticket key. pub fn new(ticket_key: TicketKey) -> Self { Self { host: HostConfig::default(), ticket_key, ticket_verifier: None, + ticket_extensions: None, hello_timeout: Duration::from_secs(5), input_rate: 120.0, input_burst: 240.0, @@ -321,6 +341,11 @@ enum ZoneCmd { client: ClientId, graceful: bool, }, + /// A zone inbox message (decoded into `Z::Message` on the zone task). + Deliver { + inbound: inbox::WireInbound, + reply: oneshot::Sender>, + }, } enum ConnOut { @@ -328,7 +353,41 @@ enum ConnOut { Close, } -type ConfigFactory = dyn Fn(&str) -> Option<::Config> + Send + Sync; +type ZoneFactoryFn = + dyn Fn(ZoneActivation) -> BoxFuture<'static, Result, ZoneCreateError>> + Send + Sync; + +/// Creates the configuration of a zone being activated, or says it does +/// not exist (`Ok(None)` → `Error(NOT_FOUND)`) or cannot be created now +/// (`Err` → `Error(ZONE_UNAVAILABLE)`). Runs on the activation task before +/// the zone exists: it may await storage, and joins wait for it (bounded by +/// `ServerConfig::activation_timeout`). The tick never waits for it. +pub struct ZoneFactory(Box>); + +impl std::fmt::Debug for ZoneFactory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ZoneFactory(..)") + } +} + +impl ZoneFactory { + /// Async factory: `ZoneFactory::new(|activation| async move { … })`. + pub fn new(factory: F) -> Self + where + F: Fn(ZoneActivation) -> Fut + Send + Sync + 'static, + Fut: Future, ZoneCreateError>> + Send + 'static, + { + Self(Box::new(move |activation| Box::pin(factory(activation)))) + } + + /// Factory that needs no I/O (the zone id and the allocation record + /// decide): `ZoneFactory::sync(|a| a.zone_id.starts_with("arena-").then(Config::default))`. + pub fn sync(factory: impl Fn(&ZoneActivation) -> Option + Send + Sync + 'static) -> Self { + Self(Box::new(move |activation| { + let config = factory(&activation); + Box::pin(async move { Ok(config) }) + })) + } +} type FactsFactory = dyn Fn(&str) -> Option<(FactQueue, JoinHandle<()>)> + Send + Sync; type HostConfigFn = dyn Fn(&str, &mut HostConfig) + Send + Sync; @@ -348,7 +407,8 @@ struct WorldState { struct Inner { config: ServerConfig, clock: Arc, - factory: Box>, + factory: ZoneFactory, + allocations: Option>, facts: Option>, /// Per-zone adjustments of `config.host` (tick rates, capacity …). host_config: Option>, @@ -389,11 +449,9 @@ where Z: Zone, Z::Config: Send, { - /// Server whose zones are created by `factory(zone_id)`; `None` = 404. - pub fn new( - config: ServerConfig, - factory: impl Fn(&str) -> Option + Send + Sync + 'static, - ) -> Self { + /// Server whose zones are configured by `factory` when they activate + /// (see [`ZoneFactory`]). + pub fn new(config: ServerConfig, factory: ZoneFactory) -> Self { Self::with_clock(config, Arc::new(SystemClock::new()), factory) } @@ -401,7 +459,7 @@ where pub fn with_clock( config: ServerConfig, clock: Arc, - factory: impl Fn(&str) -> Option + Send + Sync + 'static, + factory: ZoneFactory, ) -> Self { let (shutdown, _) = watch::channel(false); let (net_sim, _) = watch::channel(config.net_sim); @@ -409,7 +467,8 @@ where Self { inner: Arc::new(Inner { clock, - factory: Box::new(factory), + factory, + allocations: None, facts: None, host_config: None, nonces: Mutex::new(NonceCache::default()), @@ -457,6 +516,13 @@ where self.configure(|inner| inner.ownership = ownership) } + /// Where zone allocation records are read at activation + /// ([`ZoneActivation::allocation`]); a store error refuses the + /// activation with `ZONE_UNAVAILABLE`. + pub fn with_allocation_store(self, store: Arc) -> Self { + self.configure(|inner| inner.allocations = Some(store)) + } + /// Where hibernated zone snapshots go. pub fn with_state_store(self, store: Arc) -> Self { self.configure(|inner| inner.state_store = Some(store)) @@ -488,7 +554,7 @@ where self.inner.world.lock().expect("world").version.clone() } - /// True when `zone_id` is active on this server. + /// True when `zone_id` is active (or being created) on this server. pub fn is_active(&self, zone_id: &str) -> bool { self.inner .zones @@ -519,11 +585,67 @@ where rx.await.ok() } - /// Activate `zone_id` locally (as if a client asked for it). - pub fn activate(&self, zone_id: &str) -> bool { + /// Deliver a zone inbox message ([`crate::inbox`]) to `zone_id` on this + /// server. Runs on the zone's task between ticks. Refused with + /// [`InboxError::Inactive`] when the zone is not running here (a message + /// never activates a zone) and [`InboxError::WrongOwner`] when the + /// directory places it on another host. + pub async fn send( + &self, + zone_id: &str, + inbound: crate::inbox::ZoneInbound, + ) -> Result { + self.deliver_wire(zone_id, inbox::WireInbound::of(&inbound)) + .await + } + + pub(crate) async fn deliver_wire( + &self, + zone_id: &str, + inbound: inbox::WireInbound, + ) -> Result { + if let Some(owner) = self + .inner + .directory + .as_ref() + .and_then(|d| d.locate(zone_id)) + .filter(|h| h.id != self.inner.host_id) + { + return Err(InboxError::WrongOwner(owner)); + } + let tx = { + let zones = self.inner.zones.lock().expect("zones"); + zones + .get(zone_id) + .filter(|e| !e.tx.is_closed()) + .map(|e| e.tx.clone()) + .ok_or(InboxError::Inactive)? + }; + let (reply, rx) = oneshot::channel(); + tx.send(ZoneCmd::Deliver { inbound, reply }) + .await + .map_err(|_| InboxError::Inactive)?; + rx.await.map_err(|_| InboxError::Inactive)? + } + + /// Start activating `zone_id` locally without waiting (it is registered + /// at once, so barriers and joins already reach it). False only while + /// shutting down; an unknown zone unregisters itself once its factory + /// answers. + pub fn begin_activation(&self, zone_id: &str) -> bool { activate_local(&self.inner, zone_id).is_some() } + /// Activate `zone_id` locally (as if a client asked for it) and wait + /// until it runs. False when the factory knows no such zone, creation + /// failed, or the server is shutting down. + pub async fn activate(&self, zone_id: &str) -> bool { + if activate_local(&self.inner, zone_id).is_none() { + return false; + } + self.inspect(zone_id, |_| ()).await.is_some() + } + /// Barrier phase 1 on this server: freeze every active zone (and every /// zone activated until commit/abort) with `notice`, then wait up to /// `timeout` for all to acknowledge. `Err` lists zones that did not. @@ -655,6 +777,7 @@ where router = router .route("/admin/transfer", post(world::admin_transfer::)) .route("/admin/child-event", post(world::admin_child_event::)) + .route(ZONE_MESSAGE_PATH, post(world::admin_zone_message::)) .route("/admin/barrier/{phase}", post(world::admin_barrier::)); } router.with_state(self.inner.clone()) @@ -776,7 +899,6 @@ where if let Some(entry) = zones.get(zone_id).filter(|e| !e.tx.is_closed()) { return Some(entry.tx.clone()); } - let zone_config = (inner.factory)(zone_id)?; let (tx, rx) = mpsc::channel(1024); let activation = inner.activations.fetch_add(1, Ordering::Relaxed) + 1; let world = inner.world.lock().expect("world").clone(); @@ -791,7 +913,6 @@ where let task = tokio::spawn(run_zone::( inner.clone(), zone_id.to_string(), - zone_config, facts, rx, activation, @@ -945,12 +1066,13 @@ async fn acquire_claim( } } -/// Activation failed: answer everything queued and unregister. +/// Activation failed: answer everything queued with `code` and unregister. fn refuse_activation( inner: &Arc>, zone_id: &str, activation: u64, mut rx: mpsc::Receiver, + code: u32, message: &str, ) { { @@ -967,7 +1089,7 @@ fn refuse_activation( match cmd { ZoneCmd::Join { reply, .. } | ZoneCmd::Reconnect { reply, .. } => { let _ = reply.send(Err(JoinError { - code: error::NOT_FOUND, + code, message: message.to_string(), })); } @@ -975,6 +1097,9 @@ fn refuse_activation( ZoneCmd::Freeze { ack, .. } => { let _ = ack.send(()); } + ZoneCmd::Deliver { reply, .. } => { + let _ = reply.send(Err(InboxError::Inactive)); + } _ => {} } } @@ -1005,7 +1130,6 @@ async fn renew_claim( async fn run_zone( inner: Arc>, zone_id: String, - zone_config: Z::Config, facts: Option<(FactQueue, JoinHandle<()>)>, mut rx: mpsc::Receiver, activation: u64, @@ -1018,29 +1142,58 @@ async fn run_zone( let holder = format!("{}#{activation}", inner.host_id); let claim = match acquire_claim(&inner, &zone_id, &holder).await { Ok(claim) => claim, - Err(message) => return refuse_activation(&inner, &zone_id, activation, rx, &message), + Err(message) => { + return refuse_activation(&inner, &zone_id, activation, rx, error::NOT_FOUND, &message) + } }; // 2. Restore the hibernated snapshot, if any. let saved = match &inner.state_store { - Some(store) => { - let store = store.clone(); - let id = zone_id.clone(); - match tokio::task::spawn_blocking(move || store.load(&id)).await { - Ok(Ok(saved)) => saved, - _ => { - inner.ownership.release(&claim).await; - return refuse_activation( - &inner, - &zone_id, - activation, - rx, - "zone state unavailable", - ); - } + Some(store) => match store.load(&zone_id).await { + Ok(saved) => saved, + _ => { + inner.ownership.release(&claim).await; + return refuse_activation( + &inner, + &zone_id, + activation, + rx, + error::ZONE_UNAVAILABLE, + "zone state unavailable", + ); } - } + }, None => None, }; + // 3. The allocation record and the game's factory, bounded together. + let created = tokio::time::timeout(inner.config.activation_timeout, async { + let allocation = match &inner.allocations { + Some(store) => store.load(&zone_id).await?, + None => None, + }; + (inner.factory.0)(ZoneActivation { + zone_id: zone_id.clone(), + activation: claim.token, + allocation, + }) + .await + }) + .await; + let zone_config = match created { + Ok(Ok(Some(config))) => config, + refused => { + inner.ownership.release(&claim).await; + let (code, message) = match refused { + Ok(Ok(None)) => (error::NOT_FOUND, "unknown zone".to_string()), + Ok(Err(e)) => (error::ZONE_UNAVAILABLE, e.to_string()), + _ => ( + error::ZONE_UNAVAILABLE, + "zone creation timed out".to_string(), + ), + }; + inner.metrics.record_activation_refused(code); + return refuse_activation(&inner, &zone_id, activation, rx, code, &message); + } + }; let mut host_config = inner.config.host.clone(); if let Some(adjust) = &inner.host_config { adjust(&zone_id, &mut host_config); @@ -1076,11 +1229,13 @@ async fn run_zone( nonce, role: Role::Player, name: request.name.clone(), + extensions: request.extensions.clone(), }, source_zone: request.source_zone.clone(), issuer: inner.host_id.clone(), generation, carried: request.carried.clone(), + departed_at_ms: request.departed_at_ms, })) }); if let Some(saved) = &saved { @@ -1097,7 +1252,7 @@ async fn run_zone( host.request_freeze(notice); } - // 3. Keep the claim alive. + // 4. Keep the claim alive. let current = Arc::new(Mutex::new(claim)); let (lost_tx, mut lost_rx) = oneshot::channel(); let renewer = tokio::spawn(renew_claim( @@ -1207,11 +1362,11 @@ async fn run_zone( }; // A stale token means a newer owner saved already: ours is // dropped, but never silently (metrics + stderr). - let saved = tokio::task::spawn_blocking(move || store.save(&id, &stored)).await; + let saved = store.save(&id, &stored).await; let outcome = match &saved { - Ok(Ok(())) => "saved", - Ok(Err(StateError::Stale { .. })) => "stale", - _ => "error", + Ok(()) => "saved", + Err(StateError::Stale { .. }) => "stale", + Err(_) => "error", }; inner.metrics.record_hibernation_save(outcome); if outcome != "saved" { @@ -1227,6 +1382,11 @@ async fn run_zone( if matches!(cmd, ZoneCmd::Input { .. } | ZoneCmd::Disconnect { .. }) { continue; } + // Messages never activate a zone. + if let ZoneCmd::Deliver { reply, .. } = cmd { + let _ = reply.send(Err(InboxError::Inactive)); + continue; + } if let Some(tx) = activate_local(&inner, &zone_id) { let _ = tx.send(cmd).await; } @@ -1359,6 +1519,9 @@ fn handle_cmd( host.disconnect(client, graceful); } ZoneCmd::ChildEvent(event) => host.deliver_child_event(event), + ZoneCmd::Deliver { inbound, reply } => { + let _ = reply.send(inbound.decode::().map(|m| host.deliver(m))); + } ZoneCmd::Freeze { notice, ack } => { if host.request_freeze(notice) == FreezeState::Frozen { let _ = ack.send(()); @@ -1694,15 +1857,17 @@ where } let result = reply_rx.await; if let Some((issuer, nonce)) = reserved.take() { - let op = if matches!(result, Ok(Ok(_))) { - TransferOp::Confirm + if matches!(result, Ok(Ok(_))) { + let inner = inner.clone(); + tokio::spawn(async move { + let _ = transfer_op(&inner, &issuer, TransferOp::Confirm, &nonce).await; + }); } else { - TransferOp::Release - }; - let inner = inner.clone(); - tokio::spawn(async move { - let _ = transfer_op(&inner, &issuer, op, &nonce).await; - }); + // Release before the client hears the refusal: it may present + // the same ticket to its source right away, and the issuer only + // accepts that return once the reservation is released. + let _ = transfer_op(&inner, &issuer, TransferOp::Release, &nonce).await; + } } let client = match result { Ok(Ok(joined)) => joined.client, @@ -1729,10 +1894,14 @@ where while let Some(item) = outbound.pop_due() { match send_out(&mut socket, item).await { Ok(true) => {} - _ => { + // The zone closed this connection and already forgot the client. + Ok(false) => { notify_zone = false; break 'session; } + // The socket died under a write: the zone must still learn + // the connection is gone (grace + ConnectionLost). + Err(_) => break 'session, } } while let Some(item) = inbound.pop_due() { @@ -1748,9 +1917,13 @@ where out = out_rx.recv() => { let Some(out) = out else { notify_zone = false; break }; if let Some(out) = outbound.push(out) { - if !send_out(&mut socket, out).await.unwrap_or(false) { - notify_zone = false; - break; + match send_out(&mut socket, out).await { + Ok(true) => {} + Ok(false) => { + notify_zone = false; + break; + } + Err(_) => break, } } } @@ -1835,6 +2008,12 @@ fn verify_hello_ticket( return Err(TicketError::Expired); } } + let extensions = match &verified { + VerifiedTicket::Join(claims) => &claims.extensions, + VerifiedTicket::Transfer(transfer) => &transfer.claims.extensions, + }; + crate::claims::validate_extensions(cfg.ticket_extensions.as_ref(), extensions) + .map_err(|e| TicketError::Invalid(e.to_string()))?; Ok(verified) } diff --git a/distributed_game/src/net/ticket.rs b/distributed_game/src/net/ticket.rs index 2122bdf31..33978b72b 100644 --- a/distributed_game/src/net/ticket.rs +++ b/distributed_game/src/net/ticket.rs @@ -9,10 +9,16 @@ //! //! **Transfer tickets** (zone handoff) are the same format with //! `"kind":"transfer"` plus `src` (source zone), `iss` (issuing host id), -//! `gen` (source ownership token) and `carry` (base64url carried state). +//! `gen` (source ownership token), `left` (source departure stamp time, +//! Unix ms) and `carry` (base64url carried state). //! [`TicketKey::verify`] rejects them ([`TicketError::WrongKind`]); use //! [`TicketKey::verify_ticket`] to accept either kind. //! +//! **Extension claims.** An optional `ext` object carries game-declared +//! claims ([`TicketExtensions`]: match id, team…). Join and transfer +//! tickets sign them alike; the server admits them only when they match +//! `ServerConfig::ticket_extensions` (see [`crate::claims`]). +//! //! **Custom admission tickets.** Join tickets go through a //! [`TicketVerifier`] (`ServerConfig::ticket_verifier`); the default is the //! `TicketKey` itself. A game whose web backend already mints its own tickets @@ -32,6 +38,7 @@ use hmac::{Hmac, Mac}; use serde_json::{json, Value}; use sha2::Sha256; +use crate::claims::TicketExtensions; use crate::handoff::Transfer; use crate::protocol::Role; use crate::zone::TicketClaims; @@ -185,6 +192,7 @@ impl TicketKey { "src": transfer.source_zone, "iss": transfer.issuer, "gen": transfer.generation, + "left": transfer.departed_at_ms, "carry": URL_SAFE_NO_PAD.encode(&transfer.carried), }); self.sign_body(&with_name(body, claims)) @@ -238,6 +246,10 @@ impl TicketKey { .get("gen") .and_then(Value::as_u64) .ok_or(TicketError::MissingClaim("gen"))?, + departed_at_ms: body + .get("left") + .and_then(Value::as_u64) + .ok_or(TicketError::MissingClaim("left"))?, carried, claims, })) @@ -317,6 +329,11 @@ impl TicketKey { .and_then(Value::as_str) .filter(|v| !v.is_empty()) .map(str::to_string), + extensions: match claims.get("ext") { + None => TicketExtensions::new(), + Some(ext) => TicketExtensions::from_json(ext.clone()) + .map_err(|e| TicketError::Invalid(e.to_string()))?, + }, }, claims, )) @@ -324,8 +341,13 @@ impl TicketKey { } fn with_name(mut body: Value, claims: &TicketClaims) -> Value { - if let (Some(name), Some(object)) = (&claims.name, body.as_object_mut()) { - object.insert("name".into(), Value::String(name.clone())); + if let Some(object) = body.as_object_mut() { + if let Some(name) = &claims.name { + object.insert("name".into(), Value::String(name.clone())); + } + if !claims.extensions.is_empty() { + object.insert("ext".into(), claims.extensions.to_json()); + } } body } @@ -376,6 +398,7 @@ mod tests { nonce: "n1".into(), role: Role::Player, name: None, + extensions: Default::default(), } } @@ -411,6 +434,7 @@ mod tests { issuer: "h1".into(), generation: 9, carried: vec![1, 2, 3], + departed_at_ms: 1_700_000_000_123, }; let ticket = key.sign_transfer(&transfer); assert_eq!( diff --git a/distributed_game/src/net/world.rs b/distributed_game/src/net/world.rs index a0795d507..ed0a7a74e 100644 --- a/distributed_game/src/net/world.rs +++ b/distributed_game/src/net/world.rs @@ -8,6 +8,7 @@ //! |---|---|---| //! | `/admin/transfer` | `{op: reserve\|confirm\|release\|return, nonce}` | transfer record at the issuing host | //! | `/admin/child-event` | `{zone, event}` | deliver a tier-up event to a local zone | +//! | `/admin/zone-message` | `{zone, kick: {principal, reason}}` or `{zone, message: {type_id, payload}}` | zone inbox ([`super::inbox`]): 404 inactive, 421 wrong owner | //! | `/admin/barrier/prepare` | `{barrier, reason, version, notes, timeout_ms}` | freeze all zones, wait for acks (409 + `missing`) | //! | `/admin/barrier/commit` | `{barrier, version, notes, mode: rolling\|hard}` | apply version, resume | //! | `/admin/barrier/abort` | `{barrier, notes}` | resume on the old version | @@ -46,6 +47,7 @@ use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; use serde_json::{json, Value}; +use super::inbox::{delivered_json, InboxError, WireInbound}; use super::peer::{post_json, secret_eq, ADMIN_SECRET_HEADER}; use super::{activate_local, transfer_op, unix_now, GameServer, Inner, TransferOp, ZoneCmd}; use crate::facts::BoxFuture; @@ -133,6 +135,41 @@ where } } +pub(super) async fn admin_zone_message( + State(inner): State>>, + headers: HeaderMap, + body: Bytes, +) -> Reply +where + Z: Zone, + Z::Config: Send, +{ + let body = match authorize(&inner, &headers, &body) { + Ok(body) => body, + Err(r) => return r, + }; + let (zone, inbound) = match WireInbound::from_json(&body) { + Ok(parsed) => parsed, + Err(e) => return reply(StatusCode::BAD_REQUEST, json!({"ok": false, "error": e})), + }; + let server = GameServer { inner }; + match server.deliver_wire(&zone, inbound).await { + Ok(delivered) => reply(StatusCode::OK, delivered_json(delivered)), + Err(InboxError::WrongOwner(owner)) => reply( + StatusCode::MISDIRECTED_REQUEST, + json!({"ok": false, "error": "wrong_owner", "owner": owner.id, "admin_url": owner.admin_url}), + ), + Err(InboxError::Inactive) => reply( + StatusCode::NOT_FOUND, + json!({"ok": false, "error": "inactive"}), + ), + Err(e) => reply( + StatusCode::BAD_REQUEST, + json!({"ok": false, "error": e.to_string()}), + ), + } +} + pub(super) async fn admin_child_event( State(inner): State>>, headers: HeaderMap, diff --git a/distributed_game/src/protocol/messages.rs b/distributed_game/src/protocol/messages.rs index 61561f82d..c0a010abe 100644 --- a/distributed_game/src/protocol/messages.rs +++ b/distributed_game/src/protocol/messages.rs @@ -77,6 +77,9 @@ pub mod error { pub const NOT_FOUND: u32 = 7; /// Reconnect token unknown or grace window elapsed. pub const RECONNECT_FAILED: u32 = 8; + /// The zone could not be created right now (its factory or allocation + /// store failed or timed out); retry later. + pub const ZONE_UNAVAILABLE: u32 = 9; } /// Client handshake. diff --git a/distributed_game/src/replay.rs b/distributed_game/src/replay.rs index 3caf4b262..4cbd1ecc0 100644 --- a/distributed_game/src/replay.rs +++ b/distributed_game/src/replay.rs @@ -50,7 +50,7 @@ use crate::schema::FieldValue; use crate::world::{ChildEvent, StalePolicy}; use crate::zone::{ClientId, PlayerId, TicketClaims, Zone}; -const MAGIC: &[u8; 4] = b"DGR2"; +const MAGIC: &[u8; 4] = b"DGR3"; /// FNV digest of a replicated world (entity order, ids, kinds, positions, /// priorities and field values). @@ -216,6 +216,31 @@ pub enum Event { /// New version. version: String, }, + /// [`ZoneHost::set_config_version`]. + SetConfigVersion { + /// Clock. + at: Duration, + /// New version. + version: String, + }, + /// [`ZoneHost::deliver`]: a kick of `principal`. + DeliverKick { + /// Clock. + at: Duration, + /// Principal kicked. + principal: String, + /// Reason. + reason: String, + }, + /// [`ZoneHost::deliver`]: a game message (encoded `Z::Message`). + DeliverMessage { + /// Clock. + at: Duration, + /// Message type id. + type_id: u32, + /// Encoded payload. + payload: Vec, + }, /// [`ZoneHost::request_freeze`]. RequestFreeze { /// Clock. @@ -269,6 +294,9 @@ impl Event { | Self::JoinTransfer { at, .. } | Self::ReconnectVersion { at, .. } | Self::SetContentVersion { at, .. } + | Self::SetConfigVersion { at, .. } + | Self::DeliverKick { at, .. } + | Self::DeliverMessage { at, .. } | Self::RequestFreeze { at, .. } | Self::CancelFreeze { at, .. } | Self::ReloadStale { at } @@ -336,6 +364,15 @@ impl fmt::Display for Event { Self::SetContentVersion { version, .. } => { write!(f, "@{ms:.1}ms content version {version}") } + Self::SetConfigVersion { version, .. } => { + write!(f, "@{ms:.1}ms config version {version}") + } + Self::DeliverKick { principal, .. } => { + write!(f, "@{ms:.1}ms inbox kick {principal}") + } + Self::DeliverMessage { type_id, .. } => { + write!(f, "@{ms:.1}ms inbox message {type_id}") + } Self::RequestFreeze { notice, .. } => { write!(f, "@{ms:.1}ms request freeze {}", notice.reason) } @@ -375,7 +412,7 @@ pub struct Recording { /// File decode failure. #[derive(Debug, Clone, PartialEq, Eq)] pub enum RecordingError { - /// Not a `DGR2` file. + /// Not a `DGR3` file. BadMagic, /// Malformed content. Decode(String), @@ -552,6 +589,8 @@ fn put_config(w: &mut Writer, c: &HostConfig) { StalePolicy::SafeRoom { roam: true } => 2, }); put_u64(w, c.boot_epoch); + w.string(&c.config_version); + w.bool(c.lifecycle_facts); } fn put_claims(w: &mut Writer, claims: &TicketClaims) { @@ -567,6 +606,7 @@ fn put_claims(w: &mut Writer, claims: &TicketClaims) { Some(name) => w.u8(1).string(name), None => w.u8(0), }; + w.string(&claims.extensions.to_json().to_string()); } fn get_claims(r: &mut Reader<'_>) -> Result { @@ -582,6 +622,10 @@ fn get_claims(r: &mut Reader<'_>) -> Result { 0 => None, _ => Some(r.string()?), }; + let extensions = serde_json::from_str(&r.string()?) + .ok() + .and_then(|value| crate::claims::TicketExtensions::from_json(value).ok()) + .ok_or_else(|| bad("extension claims"))?; Ok(TicketClaims { principal, zone, @@ -589,6 +633,7 @@ fn get_claims(r: &mut Reader<'_>) -> Result { nonce, role, name, + extensions, }) } @@ -624,12 +669,14 @@ fn get_config(r: &mut Reader<'_>) -> Result { _ => StalePolicy::SafeRoom { roam: true }, }, boot_epoch: get_u64(r)?, + config_version: r.string()?, + lifecycle_facts: r.bool()?, }) } impl Recording { /// Compact binary encoding: - /// `"DGR2" ‖ zone_id ‖ config ‖ created_at ‖ fact_capacity? ‖ zone_config ‖ + /// `"DGR3" ‖ zone_id ‖ config ‖ created_at ‖ fact_capacity? ‖ zone_config ‖ /// varuint(event count) ‖ events`. Integers are LEB128 varints, /// durations are nanosecond varints, clock readings are delta-coded /// against the previous event, hashes are 8 raw bytes. @@ -741,6 +788,7 @@ impl Recording { put_claims(&mut w, &transfer.claims); w.string(&transfer.source_zone).string(&transfer.issuer); put_u64(&mut w, transfer.generation); + put_u64(&mut w, transfer.departed_at_ms); w.bytes(&transfer.carried) .varuint(*capabilities) .string(content_version); @@ -789,6 +837,19 @@ impl Recording { } => { w.u8(16).bytes(state).string(saved_version); } + Event::DeliverKick { + principal, reason, .. + } => { + w.u8(18).string(principal).string(reason); + } + Event::DeliverMessage { + type_id, payload, .. + } => { + w.u8(19).varuint(*type_id).bytes(payload); + } + Event::SetConfigVersion { version, .. } => { + w.u8(17).string(version); + } } } w.into_bytes() @@ -885,6 +946,7 @@ impl Recording { let source_zone = r.string()?; let issuer = r.string()?; let generation = get_u64(&mut r)?; + let departed_at_ms = get_u64(&mut r)?; let carried = r.bytes()?.to_vec(); Event::JoinTransfer { at, @@ -894,6 +956,7 @@ impl Recording { issuer, generation, carried, + departed_at_ms, }, capabilities: r.varuint()?, content_version: r.string()?, @@ -937,6 +1000,20 @@ impl Recording { state: r.bytes()?.to_vec(), saved_version: r.string()?, }, + 17 => Event::SetConfigVersion { + at, + version: r.string()?, + }, + 18 => Event::DeliverKick { + at, + principal: r.string()?, + reason: r.string()?, + }, + 19 => Event::DeliverMessage { + at, + type_id: r.varuint()?, + payload: r.bytes()?.to_vec(), + }, _ => return Err(bad("event tag")), }; events.push(event); @@ -1241,6 +1318,39 @@ impl Recorder { }); } + /// [`ZoneHost::set_config_version`]. + pub fn set_config_version(&mut self, version: &str) { + let at = self.now(); + self.host.set_config_version(version); + self.recording.events.push(Event::SetConfigVersion { + at, + version: version.into(), + }); + } + + /// [`ZoneHost::deliver`]. + pub fn deliver( + &mut self, + inbound: crate::inbox::ZoneInbound, + ) -> crate::inbox::Delivered { + let at = self.now(); + let event = match &inbound { + crate::inbox::ZoneInbound::Kick { principal, reason } => Event::DeliverKick { + at, + principal: principal.clone(), + reason: reason.clone(), + }, + crate::inbox::ZoneInbound::Message(message) => Event::DeliverMessage { + at, + type_id: crate::GameEvent::event_id(message), + payload: crate::GameEvent::to_payload(message), + }, + }; + let delivered = self.host.deliver(inbound); + self.recording.events.push(event); + delivered + } + /// [`ZoneHost::request_freeze`]. pub fn request_freeze(&mut self, notice: Notice) -> crate::world::FreezeState { let at = self.now(); @@ -1571,6 +1681,24 @@ pub fn replay_with( } } Event::SetContentVersion { version, .. } => host.set_content_version(version.clone()), + Event::SetConfigVersion { version, .. } => host.set_config_version(version.clone()), + Event::DeliverKick { + principal, reason, .. + } => { + host.deliver(crate::inbox::ZoneInbound::Kick { + principal: principal.clone(), + reason: reason.clone(), + }); + } + Event::DeliverMessage { + type_id, payload, .. + } => { + if let Ok(Some(message)) = + ::decode_event(*type_id, payload) + { + host.deliver(crate::inbox::ZoneInbound::Message(message)); + } + } Event::RequestFreeze { notice, .. } => { host.request_freeze(notice.clone()); } diff --git a/distributed_game/src/stamp.rs b/distributed_game/src/stamp.rs index a9d7401ab..3268500db 100644 --- a/distributed_game/src/stamp.rs +++ b/distributed_game/src/stamp.rs @@ -1,12 +1,27 @@ //! Tick stamping: provenance of committed facts. //! -//! Every [`Fact`](crate::Fact) carries a [`FactStamp`] (zone id, tick, -//! content version) filled in by the host when the zone emits it. The cell -//! adapter forwards the stamp as trusted session variables; command handlers -//! copy it into `EventRecord.metadata` with [`stamp_metadata`] (or -//! `stamp_event_record` with the `cell` feature) so every durable event -//! records exactly which simulation tick produced it. +//! Every [`Fact`](crate::Fact) carries a [`FactStamp`] filled in by the host +//! when the zone emits it: zone id, activation, tick, host wall time, +//! content version and config version. The cell adapter forwards the stamp +//! as trusted session variables; command handlers copy it into +//! `EventRecord.metadata` with [`stamp_metadata`] (or `stamp_event_record` +//! with the `cell` feature) so every durable event records exactly which +//! simulation tick produced it. +//! +//! **Ordering.** Commit order is not emission order: facts travel through +//! per-principal lanes and retries, and two zones commit independently. A +//! handler that must order facts (presence: "left A" vs "joined B" after a +//! handoff) compares stamps with [`FactStamp::emission_order`]. Within one +//! zone activation the emission sequence `seq` decides (the host numbers +//! every fact it stamps, so two facts of the same tick and millisecond +//! still have a strict order); across activations of one zone the +//! activation (it grows across restarts); across zones the host wall time +//! `emitted_at_ms`. A handoff target never stamps the arriving player's +//! facts earlier than the source stamped its departure: the transfer ticket +//! carries the departure time and the target's stamps start after it, even +//! when the two hosts' clocks disagree. +use std::cmp::Ordering; use std::collections::HashMap; /// Metadata key for the zone id. @@ -15,6 +30,14 @@ pub const META_ZONE: &str = "game.zone"; pub const META_TICK: &str = "game.tick"; /// Metadata key for the content version. pub const META_CONTENT_VERSION: &str = "game.content_version"; +/// Metadata key for the host wall time at emission (Unix ms, decimal). +pub const META_EMITTED_AT_MS: &str = "game.emitted_at_ms"; +/// Metadata key for the zone activation (boot epoch, decimal). +pub const META_ACTIVATION: &str = "game.activation"; +/// Metadata key for the config version. +pub const META_CONFIG_VERSION: &str = "game.config_version"; +/// Metadata key for the per-activation emission sequence (decimal). +pub const META_SEQ: &str = "game.seq"; /// Session-variable prefix used to carry a stamp through a trusted Session. pub const SESSION_PREFIX: &str = "x-game-"; @@ -28,20 +51,43 @@ pub struct FactStamp { pub tick: u64, /// Content version the zone was running. pub content_version: String, + /// Host wall time at emission, Unix milliseconds + /// ([`Clock::unix_ms`](crate::Clock::unix_ms), never earlier than a + /// handoff source's departure stamp). + pub emitted_at_ms: u64, + /// Zone activation ([`HostConfig::boot_epoch`](crate::HostConfig::boot_epoch): + /// the ownership fencing token under the `net` server); grows across + /// restarts and hibernation. + pub activation: u64, + /// Remote config version the zone was running + /// ([`HostConfig::config_version`](crate::HostConfig::config_version)). + pub config_version: String, + /// Emission sequence within the activation: the host numbers every fact + /// it stamps 1, 2, 3, ... so facts of one tick (a join and a dropped + /// socket in the same tick and millisecond) keep a strict order. A stamp + /// taken for anything else ([`Ctx::stamp`](crate::Ctx::stamp)) carries + /// the sequence of the last fact stamped before it. + pub seq: u64, } impl FactStamp { /// Metadata entries for this stamp. - pub fn entries(&self) -> [(&'static str, String); 3] { + pub fn entries(&self) -> [(&'static str, String); 7] { [ (META_ZONE, self.zone_id.clone()), (META_TICK, self.tick.to_string()), (META_CONTENT_VERSION, self.content_version.clone()), + (META_EMITTED_AT_MS, self.emitted_at_ms.to_string()), + (META_ACTIVATION, self.activation.to_string()), + (META_CONFIG_VERSION, self.config_version.clone()), + (META_SEQ, self.seq.to_string()), ] } - /// Session variables (`x-game-zone`, `x-game-tick`, `x-game-content-version`). - pub fn session_vars(&self) -> [(String, String); 3] { + /// Session variables (`x-game-zone`, `x-game-tick`, + /// `x-game-content-version`, `x-game-emitted-at-ms`, + /// `x-game-activation`, `x-game-config-version`, `x-game-seq`). + pub fn session_vars(&self) -> [(String, String); 7] { [ (format!("{SESSION_PREFIX}zone"), self.zone_id.clone()), (format!("{SESSION_PREFIX}tick"), self.tick.to_string()), @@ -49,19 +95,78 @@ impl FactStamp { format!("{SESSION_PREFIX}content-version"), self.content_version.clone(), ), + ( + format!("{SESSION_PREFIX}emitted-at-ms"), + self.emitted_at_ms.to_string(), + ), + ( + format!("{SESSION_PREFIX}activation"), + self.activation.to_string(), + ), + ( + format!("{SESSION_PREFIX}config-version"), + self.config_version.clone(), + ), + (format!("{SESSION_PREFIX}seq"), self.seq.to_string()), ] } - /// Read a stamp back from session variables (None if absent/malformed). + /// Read a stamp back from session variables (None if any is + /// absent/malformed). pub fn from_session_vars(vars: &HashMap) -> Option { + let get = |name: &str| vars.get(&format!("{SESSION_PREFIX}{name}")); Some(Self { - zone_id: vars.get(&format!("{SESSION_PREFIX}zone"))?.clone(), - tick: vars.get(&format!("{SESSION_PREFIX}tick"))?.parse().ok()?, - content_version: vars - .get(&format!("{SESSION_PREFIX}content-version"))? - .clone(), + zone_id: get("zone")?.clone(), + tick: get("tick")?.parse().ok()?, + content_version: get("content-version")?.clone(), + emitted_at_ms: get("emitted-at-ms")?.parse().ok()?, + activation: get("activation")?.parse().ok()?, + config_version: get("config-version")?.clone(), + seq: get("seq")?.parse().ok()?, }) } + + /// Read a stamp back from event metadata written by [`stamp_metadata`] + /// (None if any entry is absent/malformed). + pub fn from_metadata(metadata: &HashMap) -> Option { + Some(Self { + zone_id: metadata.get(META_ZONE)?.clone(), + tick: metadata.get(META_TICK)?.parse().ok()?, + content_version: metadata.get(META_CONTENT_VERSION)?.clone(), + emitted_at_ms: metadata.get(META_EMITTED_AT_MS)?.parse().ok()?, + activation: metadata.get(META_ACTIVATION)?.parse().ok()?, + config_version: metadata.get(META_CONFIG_VERSION)?.clone(), + seq: metadata.get(META_SEQ)?.parse().ok()?, + }) + } + + /// Emission order of two stamps: same zone and activation by emission + /// sequence (then tick and wall time); same zone across activations by + /// activation; different + /// zones by wall time (then zone id, activation and tick, so the order + /// is total and deterministic). Commit order says nothing about this. + pub fn emission_order(&self, other: &Self) -> Ordering { + if self.zone_id == other.zone_id { + self.activation + .cmp(&other.activation) + .then(self.seq.cmp(&other.seq)) + .then(self.tick.cmp(&other.tick)) + .then(self.emitted_at_ms.cmp(&other.emitted_at_ms)) + } else { + self.emitted_at_ms + .cmp(&other.emitted_at_ms) + .then_with(|| self.zone_id.cmp(&other.zone_id)) + .then(self.activation.cmp(&other.activation)) + .then(self.seq.cmp(&other.seq)) + .then(self.tick.cmp(&other.tick)) + } + } + + /// True when this stamp was emitted after `other` + /// ([`emission_order`](Self::emission_order)). + pub fn emitted_after(&self, other: &Self) -> bool { + self.emission_order(other) == Ordering::Greater + } } /// Insert the stamp into an event metadata map. @@ -81,18 +186,74 @@ pub fn stamp_event_record(record: &mut distributed::EventRecord, stamp: &FactSta mod tests { use super::*; + fn stamp(zone: &str, activation: u64, tick: u64, at: u64) -> FactStamp { + FactStamp { + zone_id: zone.into(), + tick, + content_version: "c7".into(), + emitted_at_ms: at, + activation, + config_version: "combat@12".into(), + // Ticks only grow within an activation, so neither does seq. + seq: tick, + } + } + #[test] fn stamp_round_trips_through_session_vars_and_metadata() { - let stamp = FactStamp { - zone_id: "z1".into(), - tick: 42, - content_version: "c7".into(), - }; + let stamp = stamp("z1", 9, 42, 1_700_000_000_123); let vars: HashMap<_, _> = stamp.session_vars().into_iter().collect(); assert_eq!(FactStamp::from_session_vars(&vars), Some(stamp.clone())); let mut meta = HashMap::new(); stamp_metadata(&mut meta, &stamp); assert_eq!(meta[META_TICK], "42"); assert_eq!(meta[META_ZONE], "z1"); + assert_eq!(meta[META_EMITTED_AT_MS], "1700000000123"); + assert_eq!(meta[META_ACTIVATION], "9"); + assert_eq!(meta[META_CONFIG_VERSION], "combat@12"); + assert_eq!(meta[META_SEQ], "42"); + assert_eq!(FactStamp::from_metadata(&meta), Some(stamp)); + } + + #[test] + fn missing_or_malformed_fields_are_rejected() { + let stamp = stamp("z1", 9, 42, 5); + let mut vars: HashMap<_, _> = stamp.session_vars().into_iter().collect(); + vars.remove("x-game-activation"); + assert_eq!(FactStamp::from_session_vars(&vars), None); + let mut vars: HashMap<_, _> = stamp.session_vars().into_iter().collect(); + vars.insert("x-game-emitted-at-ms".into(), "soon".into()); + assert_eq!(FactStamp::from_session_vars(&vars), None); + let mut meta = HashMap::new(); + stamp_metadata(&mut meta, &stamp); + meta.remove(META_CONFIG_VERSION); + assert_eq!(FactStamp::from_metadata(&meta), None); + } + + #[test] + fn emission_order_by_tick_activation_then_wall_time() { + // One activation: the tick wins over a wall clock read out of order. + assert!(stamp("a", 1, 11, 100).emitted_after(&stamp("a", 1, 10, 200))); + // A later activation of the same zone is later, ticks restart. + assert!(stamp("a", 2, 1, 50).emitted_after(&stamp("a", 1, 900, 60))); + // Different zones: wall time. + assert!(stamp("b", 1, 1, 201).emitted_after(&stamp("a", 7, 99, 200))); + assert!(!stamp("a", 7, 99, 200).emitted_after(&stamp("b", 1, 1, 201))); + // Ties are broken deterministically. + let (x, y) = (stamp("a", 1, 1, 5), stamp("b", 1, 1, 5)); + assert_eq!(x.emission_order(&y), y.emission_order(&x).reverse()); + // Same tick and millisecond: the emission sequence decides. + let (first, second) = ( + FactStamp { + seq: 7, + ..stamp("a", 1, 3, 9) + }, + FactStamp { + seq: 8, + ..stamp("a", 1, 3, 9) + }, + ); + assert!(second.emitted_after(&first)); + assert!(!first.emitted_after(&second)); } } diff --git a/distributed_game/src/surface.rs b/distributed_game/src/surface.rs index 5e744068a..3b787da0b 100644 --- a/distributed_game/src/surface.rs +++ b/distributed_game/src/surface.rs @@ -67,6 +67,11 @@ pub struct GameSurface { /// (`encode` / `decode`), by name. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub codecs: Vec, + /// Zone inbox messages ([`Zone::Message`], [`crate::inbox`]) by id: + /// what services may send into a running zone. Same payload encoding + /// as events. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub messages: Vec, } /// A pure function compiled to WebAssembly ([`crate::pure::GamePure`]). @@ -242,16 +247,42 @@ impl GameSurface { pures: Vec::new(), constants: Vec::new(), codecs: Vec::new(), + messages: Vec::new(), } } - /// Kinds from the zone's [`Zone::schema`] and its typed input. + /// Kinds from the zone's [`Zone::schema`], its typed input and its + /// inbox messages ([`Zone::Message`]). pub fn for_zone(name: impl Into) -> Self where Z: Zone, Z::Input: GameInput, { - Self::new(name).schema(&Z::schema()).input::() + Self::new(name) + .schema(&Z::schema()) + .input::() + .messages::() + } + + /// Add the zone inbox message type (done by [`for_zone`](Self::for_zone)). + /// Message ids and names must be unique among messages. + pub fn messages(mut self) -> Self { + let mut records = Vec::new(); + for message in M::event_surfaces(&mut records) { + assert!( + self.messages + .iter() + .all(|m| m.id != message.id && m.name != message.name), + "duplicate message id {} / name {} in game surface {}", + message.id, + message.name, + self.name + ); + self.messages.push(message); + } + self.messages.sort_by_key(|m| m.id); + self.add_records(records); + self } /// Add every kind of `schema` (derived or hand-written). diff --git a/distributed_game/src/world/allocation.rs b/distributed_game/src/world/allocation.rs new file mode 100644 index 000000000..156b185aa --- /dev/null +++ b/distributed_game/src/world/allocation.rs @@ -0,0 +1,204 @@ +//! Zone allocation: what a zone is for, decided before it starts. +//! +//! A service that allocates a zone (game-services `Match`, a party +//! finder, an instance spawner) writes a [`ZoneAllocation`] record for the +//! zone id: roster, mode, config version, seed. When the zone activates, +//! the server loads the record and hands it to the async zone factory in a +//! [`ZoneActivation`] together with the activation's fencing token, so the +//! zone's `on_create` sees it before the first player joins. Joins that +//! arrive while the zone is being created wait; the tick never awaits. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Mutex; + +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json::Value; + +use crate::facts::BoxFuture; + +/// The record an allocating service writes for a zone. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZoneAllocation { + /// Who allocated it (service name, e.g. `"game-services.match"`). + pub allocated_by: String, + /// Monotonic revision per zone id (a re-allocation bumps it). + pub revision: u64, + /// Game-defined payload (roster, mode, config version…). + pub payload: Value, +} + +impl ZoneAllocation { + /// Record with a typed payload. + pub fn new( + allocated_by: impl Into, + revision: u64, + payload: &impl Serialize, + ) -> Result { + Ok(Self { + allocated_by: allocated_by.into(), + revision, + payload: serde_json::to_value(payload)?, + }) + } + + /// Decode the payload. + pub fn payload_as(&self) -> Result { + serde_json::from_value(self.payload.clone()) + } +} + +/// Why an allocation store failed (the activation is refused with +/// `ZONE_UNAVAILABLE`, never admitted half-configured). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AllocationError(pub String); + +impl fmt::Display for AllocationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "zone allocation store: {}", self.0) + } +} + +impl std::error::Error for AllocationError {} + +/// Where allocation records live. Async: typically a service's database or +/// read model, read once per activation. +pub trait ZoneAllocationStore: Send + Sync + 'static { + /// The record for `zone_id`, if any. + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, AllocationError>>; + + /// Write a record. Refused when a record with a higher revision exists. + fn put<'a>( + &'a self, + zone_id: &'a str, + allocation: ZoneAllocation, + ) -> BoxFuture<'a, Result<(), AllocationError>>; + + /// Drop the record (the zone was released). + fn remove<'a>(&'a self, zone_id: &'a str) -> BoxFuture<'a, Result<(), AllocationError>>; +} + +/// In-memory allocation store (tests, single process). +#[derive(Debug, Default)] +pub struct MemoryZoneAllocationStore { + records: Mutex>, +} + +impl MemoryZoneAllocationStore { + /// Empty. + pub fn new() -> Self { + Self::default() + } +} + +impl ZoneAllocationStore for MemoryZoneAllocationStore { + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, AllocationError>> { + let found = self.records.lock().expect("records").get(zone_id).cloned(); + Box::pin(async move { Ok(found) }) + } + + fn put<'a>( + &'a self, + zone_id: &'a str, + allocation: ZoneAllocation, + ) -> BoxFuture<'a, Result<(), AllocationError>> { + let mut records = self.records.lock().expect("records"); + let result = match records.get(zone_id) { + Some(current) if current.revision > allocation.revision => { + Err(AllocationError(format!( + "revision {} is older than the stored {}", + allocation.revision, current.revision + ))) + } + _ => { + records.insert(zone_id.to_string(), allocation); + Ok(()) + } + }; + Box::pin(async move { result }) + } + + fn remove<'a>(&'a self, zone_id: &'a str) -> BoxFuture<'a, Result<(), AllocationError>> { + self.records.lock().expect("records").remove(zone_id); + Box::pin(async { Ok(()) }) + } +} + +/// What the zone factory is told about the activation it configures. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZoneActivation { + /// Zone id. + pub zone_id: String, + /// Activation epoch: the ownership fencing token, also the host's + /// `boot_epoch` and every fact stamp's `activation`. + pub activation: u64, + /// The allocation record, when an allocation store is configured and + /// holds one for this zone. + pub allocation: Option, +} + +/// The factory refused to create the zone because something it needs +/// failed (storage down, invalid allocation). Clients waiting to join get +/// `Error(ZONE_UNAVAILABLE)` with the message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZoneCreateError(pub String); + +impl ZoneCreateError { + /// Error with a message for the joining clients. + pub fn new(message: impl Into) -> Self { + Self(message.into()) + } +} + +impl fmt::Display for ZoneCreateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ZoneCreateError {} + +impl From for ZoneCreateError { + fn from(error: AllocationError) -> Self { + Self(error.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn block(f: impl std::future::Future) -> T { + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + let mut f = std::pin::pin!(f); + loop { + if let std::task::Poll::Ready(v) = f.as_mut().poll(&mut cx) { + return v; + } + } + } + + #[test] + fn memory_store_keeps_the_newest_revision() { + let store = MemoryZoneAllocationStore::new(); + let record = |revision| { + ZoneAllocation::new("match", revision, &serde_json::json!({"mode": "duel"})).unwrap() + }; + block(store.put("m1", record(2))).unwrap(); + assert!(block(store.put("m1", record(1))).is_err()); + let loaded = block(store.load("m1")).unwrap().unwrap(); + assert_eq!(loaded.revision, 2); + let payload: serde_json::Value = loaded.payload_as().unwrap(); + assert_eq!(payload["mode"], "duel"); + block(store.remove("m1")).unwrap(); + assert_eq!(block(store.load("m1")).unwrap(), None); + } +} diff --git a/distributed_game/src/world/mod.rs b/distributed_game/src/world/mod.rs index d325cccaa..2183010e6 100644 --- a/distributed_game/src/world/mod.rs +++ b/distributed_game/src/world/mod.rs @@ -7,25 +7,50 @@ //! the same zone ([`LocalOwnership`]; `FencedLeaseOwnership` over //! `distributed::lease` with the `lease` feature). //! - [`state`]: [`ZoneStateStore`] for hibernated zone state -//! ([`MemoryZoneStateStore`]; `SqliteZoneStateStore` with `sqlite`). +//! ([`MemoryZoneStateStore`] for unit tests; on celld, +//! `celld::CellZoneStateStore` over [`zone_state`] cells). //! - [`topology`]: [`ZoneTopology`] parent links (province > city > borough //! > neighborhood) and [`ChildEvent`] tier-up propagation. +//! - [`allocation`]: [`ZoneAllocation`] records written by the service that +//! allocates a zone, read before the zone is created +//! ([`ZoneActivation`], [`ZoneAllocationStore`]). +//! - `zone_status` (feature `cell`): the read model of every zone's lease +//! and hibernation slot, projected from the `zone_lease` / `zone_state` +//! cells' outward events (active / hibernated / cold without opening a +//! cell). //! - [`barrier`]: content-version policy types shared by the host and the //! world barrier ([`StalePolicy`], [`BarrierMode`]). +pub mod allocation; pub mod barrier; pub mod directory; pub mod ownership; pub mod state; pub mod topology; +#[cfg(feature = "cell")] +// The `sourced` and `Snapshot` derives generate undocumented public items. +#[allow(missing_docs)] +pub mod zone_lease; +#[cfg(feature = "cell")] +#[allow(missing_docs)] +pub mod zone_state; +#[cfg(feature = "cell")] +#[allow(missing_docs)] +pub mod zone_status; pub use crate::facts::BoxFuture; +pub use allocation::{ + AllocationError, MemoryZoneAllocationStore, ZoneActivation, ZoneAllocation, + ZoneAllocationStore, ZoneCreateError, +}; pub use barrier::{BarrierMode, FreezeState, StalePolicy}; pub use directory::{HashRing, HostAddr, StaticDirectory, ZoneDirectory}; +#[doc(hidden)] +pub use ownership::ownership_conformance; #[cfg(feature = "lease")] pub use ownership::FencedLeaseOwnership; pub use ownership::{LocalOwnership, OwnershipError, ZoneClaim, ZoneOwnership}; -#[cfg(feature = "sqlite")] -pub use state::SqliteZoneStateStore; +#[doc(hidden)] +pub use state::state_store_conformance; pub use state::{MemoryZoneStateStore, StateError, StoredZoneState, ZoneStateStore}; pub use topology::{ChildEvent, DottedTopology, FlatTopology, ZoneTopology}; diff --git a/distributed_game/src/world/mutations/save_zone_lease.mutation.graphql b/distributed_game/src/world/mutations/save_zone_lease.mutation.graphql new file mode 100644 index 000000000..a09fb8d2b --- /dev/null +++ b/distributed_game/src/world/mutations/save_zone_lease.mutation.graphql @@ -0,0 +1,5 @@ +# Syntax-only read-model mutation → MutationProgram IR (not a public field). +# The row is the lease's post-transition `ZoneLeaseStatus`. +mutation SaveZoneLease { + upsert_zone_leases(object: $input.lease) +} diff --git a/distributed_game/src/world/mutations/save_zone_snapshot.mutation.graphql b/distributed_game/src/world/mutations/save_zone_snapshot.mutation.graphql new file mode 100644 index 000000000..3683b586d --- /dev/null +++ b/distributed_game/src/world/mutations/save_zone_snapshot.mutation.graphql @@ -0,0 +1,5 @@ +# Syntax-only read-model mutation → MutationProgram IR (not a public field). +# The row is the hibernation slot's post-command `ZoneSnapshotStatus`. +mutation SaveZoneSnapshot { + upsert_zone_snapshots(object: $input.snapshot) +} diff --git a/distributed_game/src/world/ownership.rs b/distributed_game/src/world/ownership.rs index 719e9af1a..00be9b711 100644 --- a/distributed_game/src/world/ownership.rs +++ b/distributed_game/src/world/ownership.rs @@ -267,6 +267,60 @@ where } } +/// Behaviour every [`ZoneOwnership`] must show (exclusive claims, `Held`, +/// idempotent re-acquire, `Lost` after release, strictly increasing +/// fencing tokens). `zone_id` must be unused. Panics on a violation. +#[doc(hidden)] +pub async fn ownership_conformance(ownership: &dyn ZoneOwnership, zone_id: &str) { + let ttl = Duration::from_secs(30); + let first = ownership + .acquire(zone_id, "h1#1", ttl) + .await + .expect("first acquire"); + assert_eq!(first.zone_id, zone_id); + assert_eq!(first.holder, "h1#1"); + assert_eq!( + ownership.acquire(zone_id, "h2#1", ttl).await, + Err(OwnershipError::Held { + holder: "h1#1".into() + }), + "a live claim is exclusive" + ); + let again = ownership + .acquire(zone_id, "h1#1", ttl) + .await + .expect("re-acquire"); + assert_eq!( + again.token, first.token, + "the holder gets its own claim back" + ); + let renewed = ownership.renew(&first, ttl).await.expect("renew"); + assert_eq!(renewed.token, first.token, "renewal keeps the token"); + ownership.release(&first).await; + assert_eq!( + ownership.renew(&first, ttl).await, + Err(OwnershipError::Lost), + "a released claim is lost" + ); + let second = ownership + .acquire(zone_id, "h2#1", ttl) + .await + .expect("second acquire"); + assert!(second.token > first.token, "tokens strictly increase"); + assert_eq!( + ownership.renew(&first, ttl).await, + Err(OwnershipError::Lost), + "an old claim stays lost" + ); + ownership.release(&second).await; + let third = ownership + .acquire(zone_id, "h1#2", ttl) + .await + .expect("third acquire"); + assert!(third.token > second.token, "tokens survive release"); + ownership.release(&third).await; +} + #[cfg(test)] mod tests { use super::*; @@ -300,6 +354,11 @@ mod tests { assert!(b.token > a.token); } + #[test] + fn local_ownership_passes_the_conformance_suite() { + block(ownership_conformance(&LocalOwnership::new(), "conformance")); + } + #[test] fn local_tokens_outrank_a_previous_process() { let ttl = Duration::from_secs(1); diff --git a/distributed_game/src/world/state.rs b/distributed_game/src/world/state.rs index e0667a870..098c2c1b3 100644 --- a/distributed_game/src/world/state.rs +++ b/distributed_game/src/world/state.rs @@ -5,22 +5,21 @@ //! the zone's ownership token; the next activation (on any host) loads it //! and hands it to [`Zone::on_restore`](crate::Zone::on_restore). //! -//! Writes are **fenced**: `save` with a token older than the stored one -//! fails with [`StateError::Stale`], so a host that lost ownership while -//! paused can never overwrite the state written by its successor. +//! Writes are **fenced**: `save` (and `delete`) with a token older than the +//! stored one fails with [`StateError::Stale`], so a host that lost +//! ownership while paused can never overwrite the state written by its +//! successor. An equal token is accepted (the same owner saving again). //! -//! The trait is synchronous (snapshots are small and rare); async drivers -//! call it on a blocking thread. -//! -//! Why not an `AggregateCell`: a hibernation snapshot is a whole-value -//! overwrite keyed by zone, not a stream of domain events; storing it as -//! events would grow without bound and buy nothing. Durable *outcomes* (XP, -//! loot) go through facts into cells; the snapshot only holds ephemeral -//! simulation state worth keeping across a cold start. +//! Durable snapshots live in celld: one `ZoneState:{zone_id}` cell per zone +//! ([`super::zone_state`], `celld::CellZoneStateStore`), so a sleeping zone +//! is a sleeping Durable Object. [`MemoryZoneStateStore`] is the in-process +//! store for unit tests. There is no native file store. use std::collections::HashMap; use std::sync::Mutex; +use crate::facts::BoxFuture; + /// A stored snapshot. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StoredZoneState { @@ -61,17 +60,29 @@ impl std::error::Error for StateError {} /// Fenced snapshot store. pub trait ZoneStateStore: Send + Sync + 'static { - /// Latest snapshot of `zone_id`. - fn load(&self, zone_id: &str) -> Result, StateError>; + /// Latest snapshot of `zone_id` (`None` when none, or deleted). + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, StateError>>; /// Store a snapshot if `state.token` >= the stored token. - fn save(&self, zone_id: &str, state: &StoredZoneState) -> Result<(), StateError>; + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a StoredZoneState, + ) -> BoxFuture<'a, Result<(), StateError>>; + + /// Drop the snapshot if `token` >= the stored token. The token stays + /// (a later save under an older token is still stale). + fn delete<'a>(&'a self, zone_id: &'a str, token: u64) -> BoxFuture<'a, Result<(), StateError>>; } -/// In-memory store (tests / single process). +/// In-memory store (unit tests / single process). #[derive(Debug, Default)] pub struct MemoryZoneStateStore { - states: Mutex>, + /// zone -> (highest token seen, snapshot). + states: Mutex)>>, } impl MemoryZoneStateStore { @@ -79,242 +90,123 @@ impl MemoryZoneStateStore { pub fn new() -> Self { Self::default() } -} - -impl ZoneStateStore for MemoryZoneStateStore { - fn load(&self, zone_id: &str) -> Result, StateError> { - Ok(self.states.lock().expect("states").get(zone_id).cloned()) - } - fn save(&self, zone_id: &str, state: &StoredZoneState) -> Result<(), StateError> { + fn fenced( + &self, + zone_id: &str, + token: u64, + value: Option, + ) -> Result<(), StateError> { let mut states = self.states.lock().expect("states"); - if let Some(current) = states.get(zone_id) { - if current.token > state.token { - return Err(StateError::Stale { - presented: state.token, - current: current.token, - }); - } + let entry = states.entry(zone_id.to_string()).or_insert((0, None)); + if entry.0 > token { + return Err(StateError::Stale { + presented: token, + current: entry.0, + }); } - states.insert(zone_id.to_string(), state.clone()); + *entry = (token, value); Ok(()) } } -/// Native SQLite store (feature `sqlite`): one table, one fenced upsert. -#[cfg(feature = "sqlite")] -pub struct SqliteZoneStateStore { - conn: Mutex, -} - -#[cfg(feature = "sqlite")] -impl std::fmt::Debug for SqliteZoneStateStore { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("SqliteZoneStateStore") +impl ZoneStateStore for MemoryZoneStateStore { + fn load<'a>( + &'a self, + zone_id: &'a str, + ) -> BoxFuture<'a, Result, StateError>> { + let state = self + .states + .lock() + .expect("states") + .get(zone_id) + .and_then(|(_, state)| state.clone()); + Box::pin(async move { Ok(state) }) } -} - -#[cfg(feature = "sqlite")] -fn backend(e: rusqlite::Error) -> StateError { - StateError::Backend(e.to_string()) -} -#[cfg(feature = "sqlite")] -impl SqliteZoneStateStore { - /// Open (or create) the database file. - pub fn open(path: impl AsRef) -> Result { - Self::from_connection(rusqlite::Connection::open(path).map_err(backend)?) + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a StoredZoneState, + ) -> BoxFuture<'a, Result<(), StateError>> { + let result = self.fenced(zone_id, state.token, Some(state.clone())); + Box::pin(async move { result }) } - /// In-memory database. - pub fn open_in_memory() -> Result { - Self::from_connection(rusqlite::Connection::open_in_memory().map_err(backend)?) - } - - fn from_connection(conn: rusqlite::Connection) -> Result { - // Hosts sharing one file collide on writes; wait for the other writer - // instead of failing at once with "database is locked". - const BUSY: std::time::Duration = std::time::Duration::from_secs(5); - conn.busy_timeout(BUSY).map_err(backend)?; - // The WAL switch at open does not use the busy handler, so two hosts - // opening the file together retry it until the same deadline. - let deadline = std::time::Instant::now() + BUSY; - loop { - match conn.execute_batch( - "PRAGMA journal_mode=WAL; - CREATE TABLE IF NOT EXISTS game_zone_state ( - zone_id TEXT PRIMARY KEY, - token INTEGER NOT NULL, - content_version TEXT NOT NULL, - state BLOB NOT NULL - );", - ) { - Ok(()) => break, - Err(rusqlite::Error::SqliteFailure(e, _)) - if matches!( - e.code, - rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked - ) && std::time::Instant::now() < deadline => - { - std::thread::sleep(std::time::Duration::from_millis(10)); - } - Err(e) => return Err(backend(e)), - } - } - Ok(Self { - conn: Mutex::new(conn), - }) + fn delete<'a>(&'a self, zone_id: &'a str, token: u64) -> BoxFuture<'a, Result<(), StateError>> { + let result = self.fenced(zone_id, token, None); + Box::pin(async move { result }) } } -#[cfg(feature = "sqlite")] -impl ZoneStateStore for SqliteZoneStateStore { - fn load(&self, zone_id: &str) -> Result, StateError> { - use rusqlite::OptionalExtension; - let conn = self.conn.lock().expect("sqlite"); - conn.query_row( - "SELECT token, content_version, state FROM game_zone_state WHERE zone_id = ?1", - [zone_id], - |row| { - Ok(StoredZoneState { - token: row.get::<_, i64>(0)? as u64, - content_version: row.get(1)?, - state: row.get(2)?, - }) - }, - ) - .optional() - .map_err(backend) - } - - fn save(&self, zone_id: &str, state: &StoredZoneState) -> Result<(), StateError> { - let conn = self.conn.lock().expect("sqlite"); - let token = i64::try_from(state.token).map_err(|_| StateError::Backend("token".into()))?; - let changed = conn - .execute( - "INSERT INTO game_zone_state (zone_id, token, content_version, state) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(zone_id) DO UPDATE SET - token = excluded.token, - content_version = excluded.content_version, - state = excluded.state - WHERE excluded.token >= game_zone_state.token", - rusqlite::params![zone_id, token, state.content_version, state.state], - ) - .map_err(backend)?; - if changed == 0 { - let current: i64 = conn - .query_row( - "SELECT token FROM game_zone_state WHERE zone_id = ?1", - [zone_id], - |row| row.get(0), - ) - .map_err(backend)?; - return Err(StateError::Stale { - presented: state.token, - current: current as u64, - }); - } - Ok(()) - } +/// The fencing contract every [`ZoneStateStore`] must meet (memory, celld). +#[doc(hidden)] +pub async fn state_store_conformance(store: &dyn ZoneStateStore, zone: &str, payload: Vec) { + let snap = |token: u64, bytes: Vec| StoredZoneState { + token, + content_version: "c1".into(), + state: bytes, + }; + assert_eq!(store.load(zone).await.unwrap(), None); + store.save(zone, &snap(2, b"a".to_vec())).await.unwrap(); + store.save(zone, &snap(2, payload.clone())).await.unwrap(); + assert_eq!( + store.load(zone).await.unwrap(), + Some(snap(2, payload.clone())) + ); + assert_eq!( + store.save(zone, &snap(1, b"old".to_vec())).await, + Err(StateError::Stale { + presented: 1, + current: 2 + }) + ); + assert_eq!( + store.delete(zone, 1).await, + Err(StateError::Stale { + presented: 1, + current: 2 + }) + ); + store.save(zone, &snap(3, b"c".to_vec())).await.unwrap(); + assert_eq!( + store.load(zone).await.unwrap(), + Some(snap(3, b"c".to_vec())) + ); + store.save(zone, &snap(4, payload.clone())).await.unwrap(); + store.delete(zone, 4).await.unwrap(); + assert_eq!(store.load(zone).await.unwrap(), None); + assert_eq!( + store.save(zone, &snap(3, b"late".to_vec())).await, + Err(StateError::Stale { + presented: 3, + current: 4 + }), + "the token survives a delete" + ); + store.save(zone, &snap(5, payload.clone())).await.unwrap(); + assert_eq!(store.load(zone).await.unwrap(), Some(snap(5, payload))); } #[cfg(test)] mod tests { use super::*; - fn snap(token: u64, bytes: &[u8]) -> StoredZoneState { - StoredZoneState { - token, - content_version: "c1".into(), - state: bytes.to_vec(), + fn block_on(future: F) -> F::Output { + use std::task::{Context, Poll, Waker}; + let mut future = std::pin::pin!(future); + match future + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + { + Poll::Ready(output) => output, + Poll::Pending => panic!("memory store futures are ready"), } } - fn exercise(store: &dyn ZoneStateStore) { - assert_eq!(store.load("z").unwrap(), None); - store.save("z", &snap(2, b"a")).unwrap(); - store.save("z", &snap(2, b"b")).unwrap(); - assert_eq!( - store.save("z", &snap(1, b"old")), - Err(StateError::Stale { - presented: 1, - current: 2 - }) - ); - store.save("z", &snap(3, b"c")).unwrap(); - assert_eq!(store.load("z").unwrap(), Some(snap(3, b"c"))); - } - #[test] fn memory_store_is_fenced() { - exercise(&MemoryZoneStateStore::new()); - } - - #[cfg(feature = "sqlite")] - #[test] - fn sqlite_stores_sharing_a_file_wait_for_each_other() { - // Hosts sharing one file (the multi-host demos) open and write at the - // same time; a colliding writer must wait, not fail "database is locked". - let path = std::env::temp_dir().join(format!( - "dg-zone-state-shared-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let threads: Vec<_> = (0..8u64) - .map(|host| { - let path = path.clone(); - std::thread::spawn(move || { - let store = SqliteZoneStateStore::open(&path)?; - for token in 1..=50 { - store.save(&format!("zone.{host}"), &snap(token, b"s"))?; - } - Ok::<_, StateError>(()) - }) - }) - .collect(); - for thread in threads { - thread.join().unwrap().unwrap(); - } - let store = SqliteZoneStateStore::open(&path).unwrap(); - for host in 0..8 { - assert_eq!( - store.load(&format!("zone.{host}")).unwrap(), - Some(snap(50, b"s")) - ); - } - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } - } - - #[cfg(feature = "sqlite")] - #[test] - fn sqlite_store_is_fenced_and_durable() { - exercise(&SqliteZoneStateStore::open_in_memory().unwrap()); - let path = std::env::temp_dir().join(format!( - "dg-zone-state-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - { - let store = SqliteZoneStateStore::open(&path).unwrap(); - store.save("realm.east", &snap(7, b"boss:dead")).unwrap(); - } - let reopened = SqliteZoneStateStore::open(&path).unwrap(); - assert_eq!( - reopened.load("realm.east").unwrap(), - Some(snap(7, b"boss:dead")) - ); - drop(reopened); - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } + let store = MemoryZoneStateStore::new(); + block_on(state_store_conformance(&store, "z", vec![7; 300_000])); } } diff --git a/distributed_game/src/world/zone_lease.rs b/distributed_game/src/world/zone_lease.rs new file mode 100644 index 000000000..626b4a807 --- /dev/null +++ b/distributed_game/src/world/zone_lease.rs @@ -0,0 +1,459 @@ +//! Zone ownership as a cell: `ZoneLease:{zone_id}`. +//! +//! The lease of a zone lives in its own aggregate cell (celld in deployment, +//! the in-process reference host in tests) instead of a store private to the +//! zone server, so services learn when a zone server dies: +//! +//! | Command | Effect | Outward event | +//! |---|---|---| +//! | `zone_lease.acquire` | new holder, token + 1, expiry scheduled | `zone.activated` | +//! | `zone_lease.renew` | later expiry, expiry rescheduled | none (no heartbeat events) | +//! | `zone_lease.release` | no holder, expiry cancelled | `zone.released` | +//! | `zone_lease.expire` (schedule only) | no holder, if still expired | `zone.lease_expired` | +//! +//! Outward events carry the lease's post-transition [`ZoneLeaseStatus`] +//! (zone, holder, token, `active` / `released` / `expired`); the +//! [`zone_status`](super::zone_status) read model projects them. +//! +//! Fencing tokens increase on every acquisition and are never reset. Expiry +//! is declared by a schedule ([`distributed::schedule::Schedule`]): the +//! expiry command re-reads the lease and expires it only when the holder's +//! expiry has passed by the host clock, so a renewal that races it never +//! produces a false expiry. All times are the host's clock (`ctx.now()`), +//! never the caller's. The commands declare `Capability::Scheduling` and +//! `Capability::Clock`: they mount on aggregate cells (celld; the reference +//! host in tests) and on any backend that dispatches schedules. +//! +//! Callers must hold the `zone-host` role (an internal identity; the +//! commands are not routed through the public GraphQL edge). + +use std::time::{Duration, SystemTime}; + +use distributed::cell_host::AggregateCell; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, +}; +use distributed::schedule::{schedule_guard, Capability, Schedule}; +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; + +/// Aggregate type (cell `ZoneLease:{zone_id}`). +pub const ZONE_LEASE: &str = "ZoneLease"; +/// Role a zone server's internal session carries. +pub const ZONE_HOST_ROLE: &str = "zone-host"; +/// Acquire command. +pub const ACQUIRE: &str = "zone_lease.acquire"; +/// Renew command. +pub const RENEW: &str = "zone_lease.renew"; +/// Release command. +pub const RELEASE: &str = "zone_lease.release"; +/// Alarm-only expiry command. +pub const EXPIRE: &str = "zone_lease.expire"; +/// Rejection message prefix when another holder has the lease. +pub const HELD_PREFIX: &str = "held:"; +/// Rejection message when a claim is no longer the live one. +pub const LOST: &str = "lost"; +/// The expiry schedule's name. +pub const EXPIRY_ALARM: &str = "expiry"; + +/// [`ZoneLeaseStatus::status`] of a held lease. +pub const STATUS_ACTIVE: &str = "active"; +/// [`ZoneLeaseStatus::status`] after `zone.released`. +pub const STATUS_RELEASED: &str = "released"; +/// [`ZoneLeaseStatus::status`] after `zone.lease_expired`. +pub const STATUS_EXPIRED: &str = "expired"; + +/// The lease of one zone. +#[derive(Clone, Debug, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct ZoneLease { + entity: Entity, + holder: Option, + token: u64, + expires_at_ms: u64, + /// Holder of the last claim (kept after release and expiry). + #[serde(default)] + last_holder: String, + /// How the last claim ended (`released` / `expired`; empty while held + /// or never claimed). + #[serde(default)] + ended: String, +} + +/// The public state of a lease carried by its outward events (versioned +/// independently of the replay events): one `zone_leases` read-model row. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, distributed::DomainState)] +#[domain_state(version = 1)] +pub struct ZoneLeaseStatus { + /// Zone (the cell's stream id). + pub zone_id: String, + /// Holder of the current or, once ended, the last claim. + pub holder: String, + /// Fencing token of that claim. + pub token: u64, + /// [`STATUS_ACTIVE`], [`STATUS_RELEASED`] or [`STATUS_EXPIRED`]. + pub status: String, +} + +impl From<&ZoneLease> for ZoneLeaseStatus { + fn from(lease: &ZoneLease) -> Self { + Self { + zone_id: lease.entity.id().to_string(), + holder: lease.last_holder.clone(), + token: lease.token, + status: if lease.holder.is_some() { + STATUS_ACTIVE.to_string() + } else { + lease.ended.clone() + }, + } + } +} + +#[sourced( + entity, + aggregate_type = "ZoneLease", + events = "ZoneLeaseEvent", + domain_state = ZoneLeaseStatus +)] +impl ZoneLease { + #[event("zone.activated", version = 1, domain)] + fn activated(&mut self, zone_id: String, holder: String, token: u64, expires_at_ms: u64) { + if self.entity.id().is_empty() { + self.entity.set_id(zone_id); + } + self.last_holder = holder.clone(); + self.holder = Some(holder); + self.token = token; + self.expires_at_ms = expires_at_ms; + self.ended.clear(); + } + + #[event("zone_lease.renewed", version = 1)] + fn renewed(&mut self, expires_at_ms: u64) { + self.expires_at_ms = expires_at_ms; + } + + #[event("zone.released", version = 1, domain)] + fn released(&mut self, zone_id: String, holder: String, token: u64) { + let _ = (zone_id, holder, token); + self.holder = None; + self.ended = STATUS_RELEASED.to_string(); + } + + #[event("zone.lease_expired", version = 1, domain)] + fn lease_expired(&mut self, zone_id: String, holder: String, token: u64) { + let _ = (zone_id, holder, token); + self.holder = None; + self.ended = STATUS_EXPIRED.to_string(); + } +} + +impl ZoneLease { + /// Current holder, if any. + pub fn holder(&self) -> Option<&str> { + self.holder.as_deref() + } + + /// Latest fencing token (kept after release and expiry). + pub fn token(&self) -> u64 { + self.token + } + + /// Expiry of the current holder (epoch ms, cell clock). + pub fn expires_at_ms(&self) -> u64 { + self.expires_at_ms + } +} + +/// `zone_lease.acquire` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct AcquireInput { + pub zone_id: String, + pub holder: String, + pub ttl_ms: u64, +} + +/// `zone_lease.renew` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct RenewInput { + pub zone_id: String, + pub holder: String, + pub token: u64, + pub ttl_ms: u64, +} + +/// `zone_lease.release` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct ReleaseInput { + pub zone_id: String, + pub holder: String, + pub token: u64, +} + +/// `zone_lease.expire` input (scheduled by the lease itself). +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct ExpireInput { + pub zone_id: String, +} + +/// A held (or released) claim. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, distributed::CommandOutput)] +pub struct ClaimPayload { + pub zone_id: String, + pub holder: String, + pub token: u64, + pub expires_at_ms: u64, +} + +/// What an expiry schedule did. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, distributed::CommandOutput)] +pub struct ExpiryPayload { + pub expired: bool, +} + +/// Map a rejected lease command's message to an +/// [`OwnershipError`](super::OwnershipError). +pub fn ownership_error(message: &str) -> super::OwnershipError { + let message = message.strip_prefix("rejected: ").unwrap_or(message); + if let Some(holder) = message.strip_prefix(HELD_PREFIX) { + super::OwnershipError::Held { + holder: holder.to_string(), + } + } else if message == LOST { + super::OwnershipError::Lost + } else { + super::OwnershipError::Backend(message.to_string()) + } +} + +fn rejected(message: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(message.to_string()) +} + +fn is_zone_host(ctx: &CausalCommandContext<'_, ZoneLease>) -> bool { + ctx.session().has_role(ZONE_HOST_ROLE) +} + +fn now_ms(ctx: &CausalCommandContext<'_, ZoneLease>) -> Result { + ctx.now()? + .duration_since(SystemTime::UNIX_EPOCH) + .map(|now| now.as_millis() as u64) + .map_err(|_| rejected("host clock is before the Unix epoch")) +} + +fn ttl(ttl_ms: u64) -> Result { + if ttl_ms == 0 || ttl_ms > 24 * 60 * 60 * 1_000 { + return Err(rejected("ttl_ms must be within 1..=86400000")); + } + Ok(ttl_ms) +} + +fn validate(zone_id: &str, holder: &str) -> Result<(), HandlerError> { + if zone_id.is_empty() || holder.is_empty() { + return Err(rejected("zone_id and holder must not be empty")); + } + Ok(()) +} + +fn expiry_schedule(zone_id: &str, expires_at_ms: u64) -> Result { + Schedule::at( + EXPIRY_ALARM, + SystemTime::UNIX_EPOCH + Duration::from_millis(expires_at_ms), + EXPIRE, + &ExpireInput { + zone_id: zone_id.to_string(), + }, + ) + .map_err(rejected) +} + +async fn handle_acquire( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: AcquireInput, +) -> Result>, HandlerError> { + validate(&input.zone_id, &input.holder)?; + let ttl = ttl(input.ttl_ms)?; + let now = now_ms(ctx)?; + let repo = ctx.repo(); + let mut lease = match repo.get(&input.zone_id).await? { + Some(lease) => lease, + None => repo.create(), + }; + match lease.holder() { + // The holder re-acquiring its live claim gets it back unchanged. + Some(holder) if holder == input.holder => { + let claim = ClaimPayload { + zone_id: input.zone_id, + holder: input.holder, + token: lease.token, + expires_at_ms: lease.expires_at_ms, + }; + return repo.commit(lease)?.succeeded(claim); + } + // Held until the expiry schedule declares otherwise. + Some(holder) => return Err(rejected(format!("{HELD_PREFIX}{holder}"))), + None => {} + } + let token = lease.token + 1; + let expires_at_ms = now + ttl; + lease + .activated( + input.zone_id.clone(), + input.holder.clone(), + token, + expires_at_ms, + ) + .map_err(rejected)?; + ctx.schedule(expiry_schedule(&input.zone_id, expires_at_ms)?)?; + repo.publish_events() + .commit(lease)? + .succeeded(ClaimPayload { + zone_id: input.zone_id, + holder: input.holder, + token, + expires_at_ms, + }) +} + +async fn handle_renew( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: RenewInput, +) -> Result>, HandlerError> { + validate(&input.zone_id, &input.holder)?; + let ttl = ttl(input.ttl_ms)?; + let now = now_ms(ctx)?; + let repo = ctx.repo(); + let mut lease = repo + .get(&input.zone_id) + .await? + .ok_or_else(|| rejected(LOST))?; + if lease.holder() != Some(input.holder.as_str()) || lease.token != input.token { + return Err(rejected(LOST)); + } + let expires_at_ms = now + ttl; + lease.renewed(expires_at_ms).map_err(rejected)?; + ctx.schedule(expiry_schedule(&input.zone_id, expires_at_ms)?)?; + // Internal state only: no outward event, so no outbox row. + repo.commit(lease)?.succeeded(ClaimPayload { + zone_id: input.zone_id, + holder: input.holder, + token: input.token, + expires_at_ms, + }) +} + +async fn handle_release( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: ReleaseInput, +) -> Result>, HandlerError> { + validate(&input.zone_id, &input.holder)?; + let repo = ctx.repo(); + let mut lease = repo + .get(&input.zone_id) + .await? + .ok_or_else(|| rejected(LOST))?; + let claim = ClaimPayload { + zone_id: input.zone_id.clone(), + holder: input.holder.clone(), + token: input.token, + expires_at_ms: lease.expires_at_ms, + }; + if lease.holder() != Some(input.holder.as_str()) || lease.token != input.token { + // Releasing a claim that is already gone is a no-op. + return repo.commit(lease)?.succeeded(claim); + } + lease + .released(input.zone_id, input.holder, input.token) + .map_err(rejected)?; + ctx.cancel_schedule(EXPIRY_ALARM)?; + repo.publish_events().commit(lease)?.succeeded(claim) +} + +async fn handle_expire( + ctx: &CausalCommandContext<'_, ZoneLease>, + input: ExpireInput, +) -> Result>, HandlerError> { + let now = now_ms(ctx)?; + let repo = ctx.repo(); + let Some(mut lease) = repo.get(&input.zone_id).await? else { + return Err(rejected("no lease")); + }; + let expired = match lease.holder.clone() { + Some(holder) if now >= lease.expires_at_ms => { + let token = lease.token; + lease + .lease_expired(input.zone_id, holder, token) + .map_err(rejected)?; + true + } + // Renewed (or released) after this schedule was read: not expired. + _ => false, + }; + if expired { + repo.publish_events() + .commit(lease)? + .succeeded(ExpiryPayload { expired }) + } else { + repo.commit(lease)?.succeeded(ExpiryPayload { expired }) + } +} + +macro_rules! lease_command { + ($name:ident, $command:expr, $input:ty, $output:ty, $guard:expr, $handler:expr) => { + struct $name; + impl PortableCommand for $name + where + D: CausalRouteDependencies + Send + Sync + 'static, + { + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::<$input, Succeeded<$output>>($command)) + .requires(Capability::Scheduling) + .requires(Capability::Clock) + .guarded($guard, $handler) + } + } + }; +} + +lease_command!( + Acquire, + ACQUIRE, + AcquireInput, + ClaimPayload, + is_zone_host, + handle_acquire +); +lease_command!( + Renew, + RENEW, + RenewInput, + ClaimPayload, + is_zone_host, + handle_renew +); +lease_command!( + Release, + RELEASE, + ReleaseInput, + ClaimPayload, + is_zone_host, + handle_release +); +lease_command!( + Expire, + EXPIRE, + ExpireInput, + ExpiryPayload, + schedule_guard::, + handle_expire +); + +/// Mount every zone-lease command on a `ZoneLease` cell. +pub fn mount_zone_lease(cell: AggregateCell) -> AggregateCell { + cell.mount(Acquire) + .mount(Renew) + .mount(Release) + .mount(Expire) +} diff --git a/distributed_game/src/world/zone_state.rs b/distributed_game/src/world/zone_state.rs new file mode 100644 index 000000000..f6a72dbdb --- /dev/null +++ b/distributed_game/src/world/zone_state.rs @@ -0,0 +1,608 @@ +//! Hibernated zone state as a cell: `ZoneState:{zone_id}`. +//! +//! A sleeping zone is a sleeping Durable Object: its last snapshot lives in +//! its own aggregate cell on celld (the in-process reference host in unit +//! tests), written and read by zone servers through +//! `celld::CellZoneStateStore`. +//! +//! | Command | Effect | +//! |---|---| +//! | `zone_state.save` | store the snapshot if `token` >= the stored token | +//! | `zone_state.delete` | drop the snapshot if `token` >= the stored token | +//! +//! Fencing equals the [`ZoneStateStore`](super::ZoneStateStore) contract: an +//! older token is rejected with `stale:{current}`, an equal one is accepted, +//! and the highest token survives a delete. +//! +//! **Size.** A snapshot up to [`ZONE_STATE_INLINE_MAX`] bytes is stored in +//! the cell itself (in the `zone_state.saved` event and the aggregate +//! snapshot). A larger one goes to object storage: the worker route +//! (`celld_worker::fetch`, feature `workers-rs`) writes the bytes to an R2 bucket +//! binding of celld (the fleet bucket; RustFS in the Hops stack) under +//! `zone-state/{zone}/{token}-{sha256}` *before* the command, and the cell +//! records only `{key, sha256, len}`. The blob a save or delete supersedes +//! is deleted after the commit; a save refused as stale deletes its own +//! upload. Loads verify the length and digest. +//! +//! Every save is one small event (inline bytes or a blob reference); cells +//! load from the aggregate snapshot, which is refreshed on every commit. +//! +//! Both commands publish the post-command [`ZoneSnapshotStatus`] (zone, +//! token, whether a snapshot is stored, its content version and size, never +//! the bytes) as `zone_state.saved` / `zone_state.deleted` domain events; +//! the [`zone_status`](super::zone_status) read model projects them, so +//! "is this zone asleep with state?" is answered without opening its cell. +//! +//! Callers must hold the `zone-host` role (an internal identity on the +//! authenticated internal path). + +use distributed::cell_host::AggregateCell; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, +}; +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; + +pub use super::zone_lease::ZONE_HOST_ROLE; + +/// Aggregate type (cell `ZoneState:{zone_id}`). +pub const ZONE_STATE: &str = "ZoneState"; +/// Save command. +pub const SAVE: &str = "zone_state.save"; +/// Delete command. +pub const DELETE: &str = "zone_state.delete"; +/// Rejection prefix of a stale token (`stale:{current}`). +pub const STALE_PREFIX: &str = "stale:"; +/// Largest snapshot kept inline in the cell; larger ones go to object +/// storage. +pub const ZONE_STATE_INLINE_MAX: usize = 64 * 1024; + +/// A snapshot stored in object storage. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, distributed::CommandInput)] +pub struct BlobRef { + /// Object key. + pub key: String, + /// Lowercase hex SHA-256 of the bytes. + pub sha256: String, + /// Length in bytes. + pub len: u64, +} + +/// Where a snapshot's bytes are. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum SnapshotBytes { + /// In the cell. + Inline(Vec), + /// In object storage. + Blob(BlobRef), +} + +impl SnapshotBytes { + fn blob_key(&self) -> Option<&str> { + match self { + Self::Blob(blob) => Some(&blob.key), + Self::Inline(_) => None, + } + } +} + +/// The hibernated state of one zone. +#[derive(Clone, Debug, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct ZoneState { + entity: Entity, + token: u64, + content_version: String, + bytes: Option, +} + +/// The public state of a zone's hibernation slot carried by its outward +/// events: one `zone_snapshots` read-model row (no snapshot bytes). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, distributed::DomainState)] +#[domain_state(version = 1)] +pub struct ZoneSnapshotStatus { + /// Zone (the cell's stream id). + pub zone_id: String, + /// Highest token seen. + pub token: u64, + /// A snapshot is stored. + pub stored: bool, + /// Content version of the stored snapshot (empty when none). + pub content_version: String, + /// Size of the stored snapshot in bytes (0 when none). + pub bytes: u64, +} + +impl From<&ZoneState> for ZoneSnapshotStatus { + fn from(state: &ZoneState) -> Self { + let bytes = match &state.bytes { + Some(SnapshotBytes::Inline(bytes)) => bytes.len() as u64, + Some(SnapshotBytes::Blob(blob)) => blob.len, + None => 0, + }; + Self { + zone_id: state.entity.id().to_string(), + token: state.token, + stored: state.bytes.is_some(), + content_version: if state.bytes.is_some() { + state.content_version.clone() + } else { + String::new() + }, + bytes, + } + } +} + +#[sourced( + entity, + aggregate_type = "ZoneState", + events = "ZoneStateEvent", + domain_state = ZoneSnapshotStatus +)] +impl ZoneState { + #[event("zone_state.saved", version = 1, domain)] + fn saved( + &mut self, + zone_id: String, + token: u64, + content_version: String, + bytes: SnapshotBytes, + ) { + if self.entity.id().is_empty() { + self.entity.set_id(zone_id); + } + self.token = token; + self.content_version = content_version; + self.bytes = Some(bytes); + } + + #[event("zone_state.deleted", version = 1, domain)] + fn deleted(&mut self, zone_id: String, token: u64) { + if self.entity.id().is_empty() { + self.entity.set_id(zone_id); + } + self.token = token; + self.bytes = None; + } +} + +impl ZoneState { + /// Highest token seen (kept after a delete). + pub fn token(&self) -> u64 { + self.token + } + + /// Content version of the stored snapshot. + pub fn content_version(&self) -> &str { + &self.content_version + } + + /// The stored snapshot, if any. + pub fn bytes(&self) -> Option<&SnapshotBytes> { + self.bytes.as_ref() + } +} + +/// `zone_state.save` input: exactly one of `inline_b64` (standard base64, +/// at most [`ZONE_STATE_INLINE_MAX`] bytes decoded) and `blob`. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct SaveInput { + pub zone_id: String, + pub token: u64, + pub content_version: String, + pub inline_b64: Option, + pub blob: Option, +} + +/// `zone_state.delete` input. +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct DeleteInput { + pub zone_id: String, + pub token: u64, +} + +/// Outcome of a save or delete. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, distributed::CommandOutput)] +pub struct StatePayload { + pub zone_id: String, + pub token: u64, + /// Object key the command superseded (delete it after the commit). + pub superseded_blob: Option, +} + +/// Map a rejected command's message to a stale token, if it is one. +pub fn stale_current(message: &str) -> Option { + let message = message.strip_prefix("rejected: ").unwrap_or(message); + message.strip_prefix(STALE_PREFIX)?.parse().ok() +} + +fn rejected(message: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(message.to_string()) +} + +fn is_zone_host(ctx: &CausalCommandContext<'_, ZoneState>) -> bool { + ctx.session().has_role(ZONE_HOST_ROLE) +} + +async fn fenced( + ctx: &CausalCommandContext<'_, ZoneState>, + zone_id: &str, + token: u64, +) -> Result, HandlerError> { + if zone_id.is_empty() { + return Err(rejected("zone_id must not be empty")); + } + let repo = ctx.repo(); + let state = match repo.get(zone_id).await? { + Some(state) => state, + None => repo.create(), + }; + if state.token > token { + return Err(rejected(format!("{STALE_PREFIX}{}", state.token))); + } + Ok(state) +} + +async fn handle_save( + ctx: &CausalCommandContext<'_, ZoneState>, + input: SaveInput, +) -> Result>, HandlerError> { + let bytes = match (input.inline_b64, input.blob) { + (Some(inline), None) => { + use base64::Engine as _; + let bytes = base64::engine::general_purpose::STANDARD + .decode(inline) + .map_err(|_| rejected("inline_b64 is not base64"))?; + if bytes.len() > ZONE_STATE_INLINE_MAX { + return Err(rejected(format!( + "inline snapshots are at most {ZONE_STATE_INLINE_MAX} bytes" + ))); + } + SnapshotBytes::Inline(bytes) + } + (None, Some(blob)) => { + let hex = blob.sha256.len() == 64 && blob.sha256.bytes().all(|b| b.is_ascii_hexdigit()); + if blob.key.is_empty() || !hex { + return Err(rejected("blob needs a key and a SHA-256")); + } + SnapshotBytes::Blob(blob) + } + _ => return Err(rejected("exactly one of inline_b64 and blob")), + }; + let mut state = fenced(ctx, &input.zone_id, input.token).await?; + let previous = state + .bytes + .as_ref() + .and_then(|b| b.blob_key()) + .map(str::to_string); + let superseded = previous.filter(|key| Some(key.as_str()) != bytes.blob_key()); + state + .saved( + input.zone_id.clone(), + input.token, + input.content_version, + bytes, + ) + .map_err(rejected)?; + ctx.repo() + .publish_events() + .commit(state)? + .succeeded(StatePayload { + zone_id: input.zone_id, + token: input.token, + superseded_blob: superseded, + }) +} + +async fn handle_delete( + ctx: &CausalCommandContext<'_, ZoneState>, + input: DeleteInput, +) -> Result>, HandlerError> { + let mut state = fenced(ctx, &input.zone_id, input.token).await?; + let superseded = state + .bytes + .as_ref() + .and_then(|b| b.blob_key()) + .map(str::to_string); + state + .deleted(input.zone_id.clone(), input.token) + .map_err(rejected)?; + ctx.repo() + .publish_events() + .commit(state)? + .succeeded(StatePayload { + zone_id: input.zone_id, + token: input.token, + superseded_blob: superseded, + }) +} + +struct Save; +impl PortableCommand for Save +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(SAVE)) + .guarded(is_zone_host, handle_save) + } +} + +struct Delete; +impl PortableCommand for Delete +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + DELETE, + )) + .guarded(is_zone_host, handle_delete) + } +} + +/// Mount both zone-state commands on a `ZoneState` cell. +pub fn mount_zone_state(cell: AggregateCell) -> AggregateCell { + cell.mount(Save).mount(Delete) +} + +/// Lowercase hex SHA-256. +pub fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// Object key of a large snapshot. +pub fn blob_key(zone_id: &str, token: u64, sha256: &str) -> String { + format!("zone-state/{zone_id}/{token}-{sha256}") +} + +/// celld worker glue: the `zone-state/{zone}` HTTP surface over a +/// `ZoneState` cell and an R2 bucket binding (see the module docs). The +/// caller authenticates the internal secret first. +#[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] +pub mod celld_worker { + use super::*; + use ::worker::{Bucket, Method, Request, Response, Result}; + use base64::Engine as _; + use distributed::cell_host::{ + CellCommandIdentity, CellDispatchError, CellWaitPathRequest, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, + }; + use distributed::microsvc::Session; + use serde_json::{json, Value}; + + /// Wire input of a save from `celld::CellZoneStateStore`. + #[derive(Deserialize)] + struct WireSave { + zone_id: String, + token: u64, + content_version: String, + state_b64: String, + } + + fn json_status(body: Value, status: u16) -> Result { + Ok(Response::from_json(&body)?.with_status(status)) + } + + fn error_response(error: CellDispatchError) -> Result { + json_status( + json!({ "error": error.client_message(), "code": error.code() }), + error.status_code(), + ) + } + + /// The R2 bucket binding `name`. celld's R2 binding is not an instance + /// of the Workers runtime's `R2Bucket` class, which `Env::bucket` + /// checks for, but it has the same methods; take it unchecked. + pub fn bucket(env: &::worker::Env, name: &str) -> Result { + use ::worker::wasm_bindgen::{JsCast, JsValue}; + let env: &JsValue = env.as_ref(); + let binding = ::worker::js_sys::Reflect::get(env, &JsValue::from_str(name)) + .map_err(|_| ::worker::Error::RustError(format!("no binding `{name}`")))?; + if binding.is_undefined() { + return Err(::worker::Error::RustError(format!( + "R2 bucket binding `{name}` is not configured" + ))); + } + Ok(binding.unchecked_into::()) + } + + /// `GET /zone-state/{zone}` → `{state: null | {token, content_version, + /// state_b64}}`; `POST /zone-state/{zone}/zone_state.save|delete`. + pub async fn fetch( + cell: &AggregateCell, + bucket: &Bucket, + mut req: Request, + ) -> Result { + let url = req.url()?; + let command = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .nth(2) + .map(str::to_string); + match (req.method(), command.as_deref()) { + (Method::Get, None) => load(cell, bucket).await, + (Method::Post, Some(SAVE)) | (Method::Post, Some(DELETE)) => { + let command = command.unwrap(); + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => { + return error_response(CellDispatchError::BadRequest("invalid JSON".into())) + } + }; + let parsed = match CellWaitPathRequest::parse(body) { + Ok(parsed) => parsed, + Err(error) => return error_response(CellDispatchError::BadRequest(error)), + }; + let header = |name: &str| req.headers().get(name).ok().flatten(); + let identity = match ( + header(CELL_SERVICE_ID_HEADER), + header(CELL_PRINCIPAL_PARTITION_HEADER), + ) { + (Some(service), Some(partition)) => { + match CellCommandIdentity::new(service, partition, &parsed.command_id) { + Ok(identity) => identity, + Err(error) => return error_response(error), + } + } + _ => return error_response(CellDispatchError::Unauthorized), + }; + let mut session = Session::new(); + for key in [distributed::USER_ID_KEY, distributed::microsvc::ROLE_KEY] { + if let Some(value) = header(key) { + session.set(key, value); + } + } + if command == SAVE { + save(cell, bucket, &identity, parsed.input, session).await + } else { + dispatch(cell, bucket, DELETE, &identity, parsed.input, session, None).await + } + } + _ => json_status(json!({ "error": "not found" }), 404), + } + } + + async fn load(cell: &AggregateCell, bucket: &Bucket) -> Result { + let state = cell + .load() + .await + .map_err(|error| ::worker::Error::RustError(error.to_string()))?; + let Some(state) = state else { + return json_status(json!({ "state": null }), 200); + }; + let bytes = match state.bytes() { + None => return json_status(json!({ "state": null, "token": state.token() }), 200), + Some(SnapshotBytes::Inline(bytes)) => bytes.clone(), + Some(SnapshotBytes::Blob(blob)) => { + let object = bucket.get(blob.key.clone()).execute().await?; + let Some(body) = object.as_ref().and_then(|object| object.body()) else { + return error_response(CellDispatchError::Internal(format!( + "zone-state blob {} is missing", + blob.key + ))); + }; + let bytes = body.bytes().await?; + if bytes.len() as u64 != blob.len || sha256_hex(&bytes) != blob.sha256 { + return error_response(CellDispatchError::Internal(format!( + "zone-state blob {} does not match its digest", + blob.key + ))); + } + bytes + } + }; + json_status( + json!({ "state": { + "token": state.token(), + "content_version": state.content_version(), + "state_b64": base64::engine::general_purpose::STANDARD.encode(bytes), + }}), + 200, + ) + } + + async fn save( + cell: &AggregateCell, + bucket: &Bucket, + identity: &CellCommandIdentity, + input: Value, + session: Session, + ) -> Result { + let wire: WireSave = match serde_json::from_value(input) { + Ok(wire) => wire, + Err(error) => { + return error_response(CellDispatchError::BadRequest(format!( + "zone_state.save input: {error}" + ))) + } + }; + let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(&wire.state_b64) else { + return error_response(CellDispatchError::BadRequest( + "state_b64 is not base64".into(), + )); + }; + let (command_input, uploaded) = if bytes.len() <= ZONE_STATE_INLINE_MAX { + ( + json!({ + "zone_id": wire.zone_id, "token": wire.token, + "content_version": wire.content_version, + "inline_b64": wire.state_b64, "blob": null, + }), + None, + ) + } else { + let sha256 = sha256_hex(&bytes); + let key = blob_key(&wire.zone_id, wire.token, &sha256); + let len = bytes.len() as u64; + bucket.put(key.clone(), bytes).execute().await?; + ( + json!({ + "zone_id": wire.zone_id, "token": wire.token, + "content_version": wire.content_version, "inline_b64": null, + "blob": { "key": key, "sha256": sha256, "len": len }, + }), + Some(key), + ) + }; + dispatch( + cell, + bucket, + SAVE, + identity, + command_input, + session, + uploaded, + ) + .await + } + + async fn dispatch( + cell: &AggregateCell, + bucket: &Bucket, + command: &str, + identity: &CellCommandIdentity, + input: Value, + session: Session, + uploaded: Option, + ) -> Result { + match cell + .dispatch_idempotent(command, identity, input, session) + .await + { + Ok(dispatch) => { + let superseded = dispatch + .payload() + .get("superseded_blob") + .and_then(Value::as_str) + .map(str::to_string); + if let Some(key) = superseded { + if let Err(error) = bucket.delete(key.clone()).await { + ::worker::console_error!("superseded zone-state blob {key}: {error}"); + } + } + json_status(dispatch.wait_path_wire(), 200) + } + Err(error) => { + if let CellDispatchError::Internal(message) = &error { + ::worker::console_error!("zone-state {command}: {message}"); + } + // Refused (stale or invalid): the upload is nobody's. + if let Some(key) = uploaded { + if let Err(error) = bucket.delete(key.clone()).await { + ::worker::console_error!("refused zone-state blob {key}: {error}"); + } + } + error_response(error) + } + } + } +} diff --git a/distributed_game/src/world/zone_status.rs b/distributed_game/src/world/zone_status.rs new file mode 100644 index 000000000..2d029ec22 --- /dev/null +++ b/distributed_game/src/world/zone_status.rs @@ -0,0 +1,230 @@ +//! Zone status read model: which zones are active (and on which holder), +//! which are asleep with a hibernation snapshot, and which never ran — +//! answered from SQL rows instead of opening one cell per zone. +//! +//! Projected from the outward events of the per-zone cells: +//! +//! | Source cell | Events | Row | +//! |---|---|---| +//! | [`ZoneLease`](super::zone_lease::ZoneLease) | `zone.activated`, `zone.released`, `zone.lease_expired` | [`ZoneLeases`] (`zone_leases`) | +//! | [`ZoneState`](super::zone_state::ZoneState) | `zone_state.saved`, `zone_state.deleted` | [`ZoneSnapshots`] (`zone_snapshots`) | +//! +//! Each event carries its cell's post-transition state, so a row is an +//! upsert of the body ([`ZONE_LEASES`], [`ZONE_SNAPSHOTS`]); the projector +//! runtime orders and deduplicates per stream. [`ZoneStatus::of`] combines +//! a zone's two rows. Mount [`projectors`] on a consumer service (any read +//! model store: SQLite, Postgres) fed by the bus the cells' outbox reaches. +//! +//! The read model is eventually consistent: it is for views (world maps, +//! minimaps, dashboards), never for decisions a zone server must fence — +//! ownership and restore always go to the cells. + +use distributed::graphql::SurfaceProjector; +use distributed::projection::lower::{EventualOnly, ProjectionDescriptor}; +use distributed::{ + mutation_file, LocalProjectionMountsBuilder, Mutation, ReadModel, ReadModelCatalog, + RelationalReadModel, +}; +use serde::{Deserialize, Serialize}; + +use super::zone_lease::{ + ZoneLeaseActivatedDomainEvent, ZoneLeaseLeaseExpiredDomainEvent, ZoneLeaseReleasedDomainEvent, + STATUS_ACTIVE, +}; +use super::zone_state::{ZoneStateDeletedDomainEvent, ZoneStateSavedDomainEvent}; + +/// Application of the read model (catalog and projector owner; the +/// consumer service is named after it). +pub const APPLICATION: &str = "zone-status"; +/// Projector of [`ZoneLeases`]. +pub const LEASES_PROJECTOR: &str = "project_zone_leases"; +/// Projector of [`ZoneSnapshots`]. +pub const SNAPSHOTS_PROJECTOR: &str = "project_zone_snapshots"; +/// Epoch of [`ZONE_LEASES`] (bump to rebuild from the bus log). +pub const LEASES_EPOCH: &str = "zone-leases-v1"; +/// Epoch of [`ZONE_SNAPSHOTS`]. +pub const SNAPSHOTS_EPOCH: &str = "zone-snapshots-v1"; + +/// One `zone_leases` row: the last lease transition of a zone +/// ([`ZoneLeaseStatus`](super::zone_lease::ZoneLeaseStatus); the type is +/// named after its table, as `mutation_file!` requires). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ReadModel)] +#[table("zone_leases")] +pub struct ZoneLeases { + /// Zone id. + #[id("zone_id")] + pub zone_id: String, + /// Holder of the current (or last) claim. + pub holder: String, + /// Its fencing token. + pub token: i64, + /// `active`, `released` or `expired`. + pub status: String, +} + +/// One `zone_snapshots` row: the hibernation slot of a zone +/// ([`ZoneSnapshotStatus`](super::zone_state::ZoneSnapshotStatus)). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ReadModel)] +#[table("zone_snapshots")] +pub struct ZoneSnapshots { + /// Zone id. + #[id("zone_id")] + pub zone_id: String, + /// Highest token seen. + pub token: i64, + /// A snapshot is stored. + pub stored: bool, + /// Content version of the stored snapshot. + pub content_version: String, + /// Snapshot size in bytes. + pub bytes: i64, +} + +/// Upsert a [`ZoneLeases`] from a lease event's state body. +#[allow(non_snake_case)] +pub fn SaveZoneLease() -> Mutation<()> { + mutation_file!("src/world/mutations/save_zone_lease.mutation.graphql") +} + +/// Upsert a [`ZoneSnapshots`] from a zone-state event's state body. +#[allow(non_snake_case)] +pub fn SaveZoneSnapshot() -> Mutation<()> { + mutation_file!("src/world/mutations/save_zone_snapshot.mutation.graphql") +} + +distributed::projection! { + pub const ZONE_LEASES: ProjectionDescriptor = { + name: "project_zone_leases", + version: 1, + epoch: "zone-leases-v1", + model: ZoneLeases, + on { + events: [ + ZoneLeaseActivatedDomainEvent, + ZoneLeaseReleasedDomainEvent, + ZoneLeaseLeaseExpiredDomainEvent, + ], + mutation: SaveZoneLease, + input: { lease: body }, + }, + }; +} + +distributed::projection! { + pub const ZONE_SNAPSHOTS: ProjectionDescriptor = { + name: "project_zone_snapshots", + version: 1, + epoch: "zone-snapshots-v1", + model: ZoneSnapshots, + on { + events: [ZoneStateSavedDomainEvent, ZoneStateDeletedDomainEvent], + mutation: SaveZoneSnapshot, + input: { snapshot: body }, + }, + }; +} + +/// Physical read-model catalog (`zone_leases`, `zone_snapshots`). +pub fn read_model_catalog() -> ReadModelCatalog { + ReadModelCatalog::new(APPLICATION) + .table_schema(ZoneLeases::schema().clone()) + .table_schema(ZoneSnapshots::schema().clone()) +} + +/// The modeled projector mounts `(leases, snapshots)`: mount both on one +/// consumer service named [`APPLICATION`], applying [`ZONE_LEASES`] and +/// [`ZONE_SNAPSHOTS`] respectively. +pub fn projectors() -> (SurfaceProjector, SurfaceProjector) { + let mounts = LocalProjectionMountsBuilder::new(APPLICATION, "ordered-domain-events") + .expect("projection source") + .eventual_model::(LEASES_PROJECTOR, ZONE_LEASES, LEASES_EPOCH) + .expect("zone leases mount") + .eventual_model::(SNAPSHOTS_PROJECTOR, ZONE_SNAPSHOTS, SNAPSHOTS_EPOCH) + .expect("zone snapshots mount") + .build() + .expect("projection catalog"); + ( + mounts + .projector(LEASES_PROJECTOR) + .expect("zone leases projector"), + mounts + .projector(SNAPSHOTS_PROJECTOR) + .expect("zone snapshots projector"), + ) +} + +/// What the read model knows about a zone. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "camelCase")] +pub enum ZoneStatus { + /// A holder has the lease. + Active { + /// The lease holder. + holder: String, + }, + /// No holder; a hibernation snapshot is stored. + Hibernated { + /// Snapshot size in bytes. + bytes: u64, + }, + /// No holder and no snapshot: never ran, or ran without hibernating. + Cold, +} + +impl ZoneStatus { + /// Combine a zone's rows (either may be missing). + pub fn of(lease: Option<&ZoneLeases>, snapshot: Option<&ZoneSnapshots>) -> Self { + if let Some(lease) = lease.filter(|lease| lease.status == STATUS_ACTIVE) { + return Self::Active { + holder: lease.holder.clone(), + }; + } + match snapshot.filter(|snapshot| snapshot.stored) { + Some(snapshot) => Self::Hibernated { + bytes: u64::try_from(snapshot.bytes).unwrap_or(0), + }, + None => Self::Cold, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_prefers_an_active_lease_then_a_stored_snapshot() { + let lease = |status: &str| ZoneLeases { + zone_id: "z".into(), + holder: "A#1".into(), + token: 3, + status: status.into(), + }; + let snapshot = |stored| ZoneSnapshots { + zone_id: "z".into(), + token: 3, + stored, + content_version: "v".into(), + bytes: if stored { 42 } else { 0 }, + }; + assert_eq!( + ZoneStatus::of(Some(&lease("active")), Some(&snapshot(true))), + ZoneStatus::Active { + holder: "A#1".into() + } + ); + assert_eq!( + ZoneStatus::of(Some(&lease("released")), Some(&snapshot(true))), + ZoneStatus::Hibernated { bytes: 42 } + ); + assert_eq!( + ZoneStatus::of(Some(&lease("expired")), Some(&snapshot(false))), + ZoneStatus::Cold + ); + assert_eq!(ZoneStatus::of(None, None), ZoneStatus::Cold); + // The projections compile against the cells' outward events. + let (leases, snapshots) = projectors(); + drop((leases, snapshots)); + assert_eq!(read_model_catalog().tables.len(), 2); + } +} diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 166a06ebc..a6e2710bd 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -10,6 +10,7 @@ //! `on_tick`/`on_timer`/`on_fact_settled` interleaved → `on_leave` / //! `on_unobserve` → `on_dispose`. +use crate::claims::TicketExtensions; use crate::facts::FactSettlement; use crate::host::Ctx; use crate::protocol::Role; @@ -74,6 +75,10 @@ pub struct TicketClaims { /// identity: use `principal` for that). Exposed as /// [`PlayerInfo::name`] / [`ObserverInfo::name`]. pub name: Option, + /// Game-declared claims the issuer vouches for (match id, team…; the + /// ticket's `ext`), checked by the `net` server against + /// `ServerConfig::ticket_extensions`. Never identity or rewards. + pub extensions: TicketExtensions, } impl TicketClaims { @@ -92,9 +97,16 @@ impl TicketClaims { nonce: nonce.into(), role, name: None, + extensions: TicketExtensions::new(), } } + /// Attach extension claims. + pub fn with_extensions(mut self, extensions: TicketExtensions) -> Self { + self.extensions = extensions; + self + } + /// Attach a display name. pub fn with_name(mut self, name: impl Into) -> Self { self.name = Some(name.into()); @@ -154,6 +166,9 @@ pub struct PlayerInfo { pub principal: String, /// Display name from the ticket ([`TicketClaims::name`]), if any. pub name: Option, + /// Extension claims from the ticket ([`TicketClaims::extensions`]); + /// replaced by a fresh ticket's on rejoin, carried across handoffs. + pub extensions: TicketExtensions, /// Hello capabilities. pub capabilities: u32, /// False while inside the reconnect grace window. @@ -169,6 +184,8 @@ pub struct ObserverInfo { pub principal: String, /// Display name from the ticket, if any. pub name: Option, + /// Extension claims from the ticket. + pub extensions: TicketExtensions, /// Hello capabilities. pub capabilities: u32, } @@ -192,6 +209,10 @@ pub trait Zone: Sized + Send + 'static { type Input: Send + 'static; /// Timer payload. type Timer: Clone + Send + 'static; + /// Messages services deliver into the running zone + /// ([`on_message`](Self::on_message), [`crate::inbox`]): a + /// `#[derive(GameEvent)]` struct or enum, or [`NoMessages`](crate::NoMessages). + type Message: crate::GameEvent + Send + 'static; /// Replicated entity schema (called once at creation). fn schema() -> EntitySchema; @@ -207,9 +228,19 @@ pub trait Zone: Sized + Send + 'static { /// A player joined (not called on reconnect). fn on_join(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId) {} - /// A player left for `reason` (not called on a disconnect still inside the grace window). + /// A player left for `reason` (not called on a disconnect still inside + /// the grace window: see [`on_connection_lost`](Self::on_connection_lost)). fn on_leave(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId, _reason: LeaveReason) {} + /// A player's connection dropped (or it was told to reload) and its + /// reconnect grace window started; the slot and its entity stay. + /// `on_leave(Disconnected)` follows if it does not come back in time. + fn on_connection_lost(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId) {} + + /// A player resumed its slot with a reconnect token within the grace + /// window. + fn on_reconnected(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId) {} + /// A fresh ticket (not a reconnect token) arrived for a principal that /// already holds a player slot here, e.g. the page was reloaded or opened /// in another tab with the same identity. The old connection was @@ -245,6 +276,11 @@ pub trait Zone: Sized + Send + 'static { /// One fixed simulation step (`ctx.dt()` is the step length of the tier). fn on_tick(&mut self, ctx: &mut Ctx<'_, Self>); + /// A message from a service ([`crate::inbox`]), delivered on the zone's + /// task between ticks (never during one). The built-in kick is handled + /// by the host and never reaches this hook. + fn on_message(&mut self, _ctx: &mut Ctx<'_, Self>, _message: Self::Message) {} + /// A timer fired (before `on_tick` of the same tick). fn on_timer(&mut self, _ctx: &mut Ctx<'_, Self>, _id: TimerId, _timer: Self::Timer) {} @@ -264,8 +300,18 @@ pub trait Zone: Sized + Send + 'static { false } - /// Describe replicated entities for this patch. - fn replicate(&self, out: &mut WorldView); + /// The zone's entity collections, registered once at creation + /// ([`entity`](crate::entity)): every patch the host pushes each + /// entity's [`Project`](crate::Project)ed view (in registration order), + /// then calls [`replicate`](Self::replicate). Default: none. + fn entities() -> crate::Entities { + crate::Entities::new() + } + + /// Describe replicated entities for this patch that are not projected + /// from a registered collection ([`entities`](Self::entities)). + /// Default: nothing. + fn replicate(&self, _out: &mut WorldView) {} /// Record hit volumes for lag compensation after each tick /// ([`Ctx::rewind`](crate::Ctx::rewind)). Default: nothing recorded. diff --git a/distributed_game/tests/authoring/main.rs b/distributed_game/tests/authoring/main.rs index 21b18449f..c4e4c5163 100644 --- a/distributed_game/tests/authoring/main.rs +++ b/distributed_game/tests/authoring/main.rs @@ -127,6 +127,19 @@ pub enum SampleEvent { }, } +/// What services may send into the sample zone (zone inbox). +#[derive(Debug, Clone, PartialEq, GameEvent)] +pub enum SampleMessage { + #[game_event(id = 1)] + Announce { + #[long] + text: String, + urgent: bool, + }, + #[game_event(id = 2)] + SetWind(SampleWind), +} + #[derive(Debug, Clone, PartialEq, GameEvent)] #[game_event(id = 9)] pub struct Ping { @@ -243,6 +256,7 @@ impl Zone for Sample { type Config = (); type Input = SampleInput; type Timer = (); + type Message = SampleMessage; fn schema() -> EntitySchema { schema![PlayerView, MarkerView, StatsView] @@ -753,6 +767,21 @@ fn golden_fixture_is_current() { "sanitized": {"mx": sanitized.mx, "my": sanitized.my, "buttons": sanitized.buttons, "aim": aim_wire, "power": 0.25, "viewTick": 123_456}, }, "events": events, + // Zone inbox messages (`/admin/zone-message` payloads). + "messages": [ + { + "id": 1, + "name": "announce", + "value": { "text": "server restart in 5", "urgent": true }, + "hex": hex(&SampleMessage::Announce { text: "server restart in 5".into(), urgent: true }.to_payload()), + }, + { + "id": 2, + "name": "set_wind", + "value": { "speed": 2.5, "gusts": 7 }, + "hex": hex(&SampleMessage::SetWind(SampleWind { speed: 2.5, gusts: 7 }).to_payload()), + }, + ], // Each constant's encoding: the generated literal must decode-equal it. "constants": { "sample_board": hex(&distributed_game::codec::to_bytes(&sample_board())), diff --git a/distributed_game/tests/bots/main.rs b/distributed_game/tests/bots/main.rs index e8543d42c..6a673838d 100644 --- a/distributed_game/tests/bots/main.rs +++ b/distributed_game/tests/bots/main.rs @@ -9,7 +9,7 @@ use std::time::Duration; use arena::{Arena, ArenaConfig}; use distributed_game::bots::{run_bots, BehaviourKind, BotConfig}; -use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; +use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey, ZoneFactory}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; const SECRET: &str = "bots-secret"; @@ -34,9 +34,10 @@ async fn fifty_bots_join_play_and_receive_deltas() { Duration::from_millis(5), Duration::from_millis(5), )); - let server = GameServer::::new(config, |zone| { - zone.starts_with("load").then(ArenaConfig::default) - }); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with("load").then(ArenaConfig::default)), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let serving = server.clone(); diff --git a/distributed_game/tests/cell_sink/main.rs b/distributed_game/tests/cell_sink/main.rs index 9cf78fc0f..2bb062127 100644 --- a/distributed_game/tests/cell_sink/main.rs +++ b/distributed_game/tests/cell_sink/main.rs @@ -10,9 +10,11 @@ use distributed::microsvc::{ CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, }; use distributed::{sourced, Entity}; -use distributed_game::cell::{command_id_for, CellFactSink}; +use std::collections::HashMap; + +use distributed_game::cell::{command_id_for, session_stamp, CellFactSink}; use distributed_game::{ - fact_channel, spawn_fact_worker, Fact, FactOutcome, FactStamp, RetryPolicy, + fact_channel, spawn_fact_worker, stamp_metadata, Fact, FactOutcome, FactStamp, RetryPolicy, }; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -42,6 +44,11 @@ struct CreditPayload { hero: String, xp: u32, tick: String, + /// The stamp the handler read from the trusted session, after a trip + /// through event metadata. + emitted_at_ms: u64, + activation: u64, + config_version: String, } struct CreditXp; @@ -69,6 +76,11 @@ async fn handle_credit( // Identity comes from the trusted session, never from the input. let principal = ctx.session().user_id().unwrap_or_default().to_string(); let tick = ctx.session().get("x-game-tick").unwrap_or("").to_string(); + let stamp = session_stamp(ctx.session()) + .ok_or_else(|| HandlerError::Rejected("fact sessions carry a stamp".into()))?; + let mut metadata = HashMap::new(); + stamp_metadata(&mut metadata, &stamp); + let stamp = FactStamp::from_metadata(&metadata).expect("metadata round trip"); let repo = ctx.repo(); let mut hero = match repo.get(&principal).await? { Some(hero) => hero, @@ -81,6 +93,9 @@ async fn handle_credit( hero: principal, xp, tick, + emitted_at_ms: stamp.emitted_at_ms, + activation: stamp.activation, + config_version: stamp.config_version, }) } @@ -104,6 +119,10 @@ async fn facts_dispatch_idempotently_with_trusted_principal() { zone_id: "z1".into(), tick: 99, content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_456, + activation: 77, + config_version: "combat@12".into(), + seq: 12, }; let fact = Fact::trusted( "coin:z1:7", @@ -131,6 +150,9 @@ async fn facts_dispatch_idempotently_with_trusted_principal() { assert_eq!(first.payload["hero"], "alice"); assert_eq!(first.payload["xp"], 10); assert_eq!(first.payload["tick"], "99"); + assert_eq!(first.payload["emitted_at_ms"], 1_700_000_000_456u64); + assert_eq!(first.payload["activation"], 77); + assert_eq!(first.payload["config_version"], "combat@12"); assert_eq!(cell.load().await.unwrap().unwrap().xp, 10, "credited once"); // Payload-supplied identity is not a declared input: permanent failure. diff --git a/distributed_game/tests/celld_facts/main.rs b/distributed_game/tests/celld_facts/main.rs new file mode 100644 index 000000000..2e61cedc8 --- /dev/null +++ b/distributed_game/tests/celld_facts/main.rs @@ -0,0 +1,358 @@ +//! Zone facts over the network into celld cells (`CelldFactSink`). +//! +//! Owns an isolated `celld dev` process (no Docker) serving +//! `tests/celld/fact-worker` (Kills, Match and Presence cells), and kills only +//! the processes it starts. Build the worker first: +//! +//! (cd tests/celld/fact-worker && worker-build --release) +//! cargo test -p distributed_game --features celld,runtime --test celld_facts +//! +//! `CELLD` selects the binary (default `celld` on PATH); `celld dev` needs +//! esbuild (`CELLD_ESBUILD`). Without a built worker the test skips, unless +//! `CELLD_FACTS_REQUIRED=1` (CI), which turns that into a failure. + +use std::time::{Duration, Instant}; + +use distributed::cell_host::InternalHttpSecret; +use distributed_game::celld::{CelldFactSink, FactRoute}; +use distributed_game::lifecycle::{JoinVia, LeftReason, PlayerJoined, PlayerLeft}; +use distributed_game::{ + fact_channel, spawn_fact_worker, EmitError, Fact, FactCommand, FactError, FactOutcome, + FactQueue, FactSettlement, FactSink, FactStamp, RetryPolicy, +}; +use serde_json::{json, Value}; + +#[path = "../support/celld.rs"] +mod celld_support; +use celld_support::{Celld, SECRET}; +const SERVICE: &str = "zone-server-test"; + +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "kills.record")] +struct Kill { + mob: String, +} + +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "match.record_result")] +struct MatchResult { + match_id: String, + winner_slot: u32, +} + +fn sink(base: &str, timeout: Duration) -> CelldFactSink { + CelldFactSink::celld_with_timeout( + base, + InternalHttpSecret::new(SECRET).unwrap(), + SERVICE, + "zone:pit-1", + timeout, + ) + .unwrap() + .route(FactRoute::by_principal::("kills")) + .route(FactRoute::by_input_field::( + "match", "match_id", + )) + .lifecycle_route("presence") +} + +fn stamp(tick: u64) -> FactStamp { + FactStamp { + zone_id: "pit-1".into(), + tick, + content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_000 + tick, + activation: 1, + config_version: "cfg-1".into(), + seq: tick, + } +} + +fn fact(key: &str, input: &C, principal: Option<&str>, tick: u64) -> Fact { + Fact::trusted( + key, + C::COMMAND, + serde_json::to_value(input).unwrap(), + principal.map(str::to_string), + stamp(tick), + ) +} + +async fn state(base: &str, kind: &str, shard: &str) -> Value { + reqwest::Client::new() + .get(format!("{base}/{kind}/{shard}")) + .header("x-distributed-internal-secret", SECRET) + .send() + .await + .unwrap() + .json() + .await + .unwrap() +} + +/// Drain settlements until `count` arrive (or time out). +async fn settle(queue: &mut FactQueue, count: usize, within: Duration) -> Vec { + let deadline = Instant::now() + within; + let mut settled = Vec::new(); + while settled.len() < count && Instant::now() < deadline { + settled.extend(queue.drain()); + tokio::time::sleep(Duration::from_millis(50)).await; + } + settled +} + +fn committed(settlement: &FactSettlement) -> bool { + matches!(settlement.outcome, FactOutcome::Committed(_)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn zone_facts_route_to_three_cell_kinds_and_survive_a_celld_outage() { + let Some(mut celld) = Celld::start_new() else { + return; + }; + let base = celld.base(); + let policy = RetryPolicy::until_settled(Duration::from_millis(100), Duration::from_millis(500)); + let (mut queue, feed) = fact_channel(4, policy); + let worker = spawn_fact_worker( + feed.with_max_concurrent_commits(2), + sink(&base, Duration::from_secs(10)), + ); + + // 1. One zone, one sink, three aggregate kinds (lifecycle -> presence). + let joined = PlayerJoined { + zone: "pit-1".into(), + via: JoinVia::Ticket, + from_zone: None, + }; + queue + .try_send(fact( + "zone.player_joined:pit-1:1:p0:0", + &joined, + Some("alice"), + 1, + )) + .unwrap(); + queue + .try_send(fact( + "kills.record:pit-1:1:k1", + &Kill { mob: "m1".into() }, + Some("alice"), + 2, + )) + .unwrap(); + queue + .try_send(fact( + "kills.record:pit-1:1:k2", + &Kill { mob: "m2".into() }, + Some("bob"), + 3, + )) + .unwrap(); + queue + .try_send(fact( + "match.record_result:pit-1:1:m-7", + &MatchResult { + match_id: "m-7".into(), + winner_slot: 2, + }, + None, + 4, + )) + .unwrap(); + let settled = settle(&mut queue, 4, Duration::from_secs(30)).await; + assert_eq!(settled.len(), 4, "{settled:?}"); + assert!(settled.iter().all(committed), "{settled:?}"); + let left = PlayerLeft { + zone: "pit-1".into(), + reason: LeftReason::Left, + }; + queue + .try_send(fact( + "zone.player_left:pit-1:1:p0:1", + &left, + Some("alice"), + 5, + )) + .unwrap(); + assert!(settle(&mut queue, 1, Duration::from_secs(30)) + .await + .iter() + .all(committed)); + + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1"]) + ); + assert_eq!( + state(&base, "kills", "bob").await["state"]["mobs"], + json!(["m2"]) + ); + assert_eq!( + state(&base, "match", "m-7").await["state"]["results"], + json!([2]) + ); + assert_eq!( + state(&base, "presence", "alice").await["state"]["transitions"], + json!(["joined:pit-1:ticket", "left:pit-1:left"]), + "lifecycle facts go to presence, not to the player's kill cell" + ); + + // 2. The same key twice (a retry after a lost response) commits once and + // replays the ledger receipt. + let direct = sink(&base, Duration::from_secs(10)); + let again = fact( + "kills.record:pit-1:1:k1", + &Kill { mob: "m1".into() }, + Some("alice"), + 2, + ); + let replay = direct.commit(&again).await.unwrap(); + assert!(replay.replayed); + assert_eq!(state(&base, "kills", "alice").await["events"], 1); + // Forced timeout after commit: the first attempt gives up before the + // response; the retry finds the commit in the ledger. + let hasty = sink(&base, Duration::from_millis(1)); + let late = fact( + "kills.record:pit-1:1:k3", + &Kill { mob: "m3".into() }, + Some("alice"), + 6, + ); + assert!(matches!( + hasty.commit(&late).await, + Err(FactError::Retryable(_)) + )); + tokio::time::sleep(Duration::from_millis(500)).await; + let receipt = direct.commit(&late).await.unwrap(); + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1", "m3"]), + "exactly one commit for the key (replayed: {})", + receipt.replayed + ); + + // 3. An unknown command settles Permanent without a network call. + queue + .try_send(Fact::trusted( + "guild.promote:pit-1:1:x", + "guild.promote", + json!({}), + Some("alice".into()), + stamp(7), + )) + .unwrap(); + let unknown = settle(&mut queue, 1, Duration::from_secs(10)).await; + assert!( + matches!( + &unknown[0].outcome, + FactOutcome::Failed { + error: FactError::Permanent(_), + attempts: 1 + } + ), + "{unknown:?}" + ); + + // 4. The cell worker rejects requests without the internal secret. + let forged = reqwest::Client::new() + .post(format!("{base}/kills/alice/kills.record")) + .header("x-distributed-service-id", SERVICE) + .header("x-distributed-principal-partition", "alice") + .header("x-user-id", "alice") + .json(&json!({ "commandId": "0190a000-0000-7000-8000-000000000001", "input": { "mob": "x" } })) + .send() + .await + .unwrap(); + assert_eq!(forged.status(), 401); + let forged_secret = reqwest::Client::new() + .post(format!("{base}/kills/alice/kills.record")) + .header("x-distributed-internal-secret", "forged-secret-for-red-team-request-0000") + .json(&json!({ "commandId": "0190a000-0000-7000-8000-000000000002", "input": { "mob": "x" } })) + .send() + .await + .unwrap(); + assert_eq!(forged_secret.status(), 401); + // With the secret but without the fact headers, handlers refuse. + let not_a_fact = reqwest::Client::new() + .post(format!("{base}/kills/alice/kills.record")) + .header("x-distributed-internal-secret", SECRET) + .header("x-distributed-service-id", SERVICE) + .header("x-distributed-principal-partition", "alice") + .header("x-user-id", "alice") + .json(&json!({ "commandId": "0190a000-0000-7000-8000-000000000003", "input": { "mob": "x" } })) + .send() + .await + .unwrap(); + assert!( + not_a_fact.status().is_client_error(), + "{}", + not_a_fact.status() + ); + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1", "m3"]) + ); + + // 5. celld down: facts stay pending, the zone is never blocked, the + // queue reports Backpressure at capacity. + celld.crash(); + let outage = [ + ("alice", "a1", 10), + ("bob", "b1", 11), + ("alice", "a2", 12), + ("bob", "b2", 13), + ]; + for (principal, mob, tick) in outage { + let started = Instant::now(); + queue + .try_send(fact( + &format!("kills.record:pit-1:1:{mob}"), + &Kill { mob: mob.into() }, + Some(principal), + tick, + )) + .unwrap(); + assert!( + started.elapsed() < Duration::from_millis(50), + "emitting never waits on celld" + ); + } + assert_eq!( + queue.try_send(fact( + "kills.record:pit-1:1:a3", + &Kill { mob: "a3".into() }, + Some("alice"), + 14 + )), + Err(EmitError::Backpressure) + ); + assert!( + settle(&mut queue, 1, Duration::from_secs(3)) + .await + .is_empty(), + "nothing settles while celld is down" + ); + assert_eq!(queue.in_flight(), 4); + + // celld back: the backlog drains, each principal in emission order. + celld.start().expect("celld restarts"); + let drained = settle(&mut queue, 4, Duration::from_secs(60)).await; + assert_eq!(drained.len(), 4, "{drained:?}"); + assert!(drained.iter().all(committed), "{drained:?}"); + assert_eq!( + state(&base, "kills", "alice").await["state"]["mobs"], + json!(["m1", "m3", "a1", "a2"]) + ); + assert_eq!( + state(&base, "kills", "bob").await["state"]["mobs"], + json!(["m2", "b1", "b2"]) + ); + let counts = drained + .iter() + .map(|settlement| settlement.key.clone()) + .collect::>(); + assert_eq!(counts.len(), 4); + + drop(queue); + let _ = tokio::time::timeout(Duration::from_secs(5), worker).await; +} diff --git a/distributed_game/tests/celld_zone_lease/main.rs b/distributed_game/tests/celld_zone_lease/main.rs new file mode 100644 index 000000000..cd51235ed --- /dev/null +++ b/distributed_game/tests/celld_zone_lease/main.rs @@ -0,0 +1,184 @@ +//! Zone ownership as celld cells (`CellZoneOwnership` → `ZoneLease:{zone}`) +//! on an isolated `celld dev`: the ownership conformance suite, expiry +//! declared by the cell alarm when a holder stops renewing, renewals that +//! keep the lease, and a `GameServer` running a zone on it. +//! +//! (cd tests/celld/fact-worker && worker-build --release) +//! cargo test -p distributed_game --features celld,runtime,net --test celld_zone_lease + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use distributed::cell_host::InternalHttpSecret; +use distributed_game::celld::CellZoneOwnership; +use distributed_game::net::{GameServer, ServerConfig, TicketKey, ZoneFactory}; +use distributed_game::world::{ownership_conformance, OwnershipError, ZoneOwnership}; +use distributed_game::*; +use serde_json::Value; + +#[path = "../support/celld.rs"] +mod celld_support; +use celld_support::{Celld, SECRET}; + +fn ownership(base: &str, host: &str) -> CellZoneOwnership { + CellZoneOwnership::new( + base, + InternalHttpSecret::new(SECRET).unwrap(), + "zone-lease-test", + host, + ) + .unwrap() +} + +async fn lease(base: &str, zone: &str) -> Value { + reqwest::Client::new() + .get(format!("{base}/zone-lease/{zone}")) + .header("x-distributed-internal-secret", SECRET) + .send() + .await + .unwrap() + .json() + .await + .unwrap() +} + +fn count(state: &Value, event: &str) -> usize { + state["eventNames"].as_array().map_or(0, |names| { + names.iter().filter(|name| *name == event).count() + }) +} + +async fn until(what: &str, within: Duration, mut check: impl AsyncFnMut() -> bool) { + let deadline = Instant::now() + within; + while Instant::now() < deadline { + if check().await { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("timed out: {what}"); +} + +/// A zone with no players or state: enough to run under a lease. +struct Beacon; + +impl Zone for Beacon { + type Config = (); + type Input = u8; + type Timer = (); + type Message = NoMessages; + + fn schema() -> EntitySchema { + EntitySchema::new() + } + + fn on_create(_ctx: &mut Ctx<'_, Self>, _config: ()) -> Self { + Beacon + } + + fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option { + payload.first().copied() + } + + fn on_tick(&mut self, _ctx: &mut Ctx<'_, Self>) {} + + fn replicate(&self, _view: &mut WorldView) {} +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn zone_leases_live_in_celld_cells() { + let Some(celld) = Celld::start_new() else { + return; + }; + let base = celld.base(); + let own = ownership(&base, "host-a"); + + // 1. Fenced, exclusive, strictly increasing tokens; Held and Lost. + ownership_conformance(&own, "zl-conformance").await; + let conformance = lease(&base, "zl-conformance").await; + assert_eq!(count(&conformance, "zone.activated"), 3); + assert_eq!(count(&conformance, "zone.released"), 3); + assert_eq!(count(&conformance, "zone.lease_expired"), 0); + + // 2. A holder that stops renewing: exactly one zone.lease_expired after + // the TTL, declared by the cell's own alarm (nobody calls the cell). + let dead = own + .acquire("zl-expire", "a#1", Duration::from_millis(1_500)) + .await + .unwrap(); + until("lease expiry", Duration::from_secs(20), async || { + lease(&base, "zl-expire").await["state"]["holder"].is_null() + }) + .await; + tokio::time::sleep(Duration::from_secs(2)).await; + let expired = lease(&base, "zl-expire").await; + assert_eq!(count(&expired, "zone.lease_expired"), 1, "{expired}"); + assert_eq!(expired["armedAlarms"], 0); + assert_eq!( + own.renew(&dead, Duration::from_secs(1)).await, + Err(OwnershipError::Lost) + ); + let next = own + .acquire("zl-expire", "b#1", Duration::from_secs(5)) + .await + .unwrap(); + assert!(next.token > dead.token); + own.release(&next).await; + + // 3. Renewals keep the lease past many TTLs, without expiry or outward + // events. + let mut live = own + .acquire("zl-renew", "a#1", Duration::from_millis(1_500)) + .await + .unwrap(); + for _ in 0..10 { + tokio::time::sleep(Duration::from_millis(400)).await; + live = own + .renew(&live, Duration::from_millis(1_500)) + .await + .unwrap(); + } + let renewed = lease(&base, "zl-renew").await; + assert_eq!(renewed["state"]["holder"], "a#1"); + assert_eq!(count(&renewed, "zone.lease_expired"), 0); + assert_eq!(count(&renewed, "zone_lease.renewed"), 10); + own.release(&live).await; + + // 4. A zone server runs a zone under the celld lease: the cell sees the + // holder while the server renews, and the lease ends when it stops. + let mut config = ServerConfig::new(TicketKey::new("celld-lease-test-ticket-secret")); + config.ownership_ttl = Duration::from_secs(3); + config.ownership_renew_every = Duration::from_millis(500); + let server = GameServer::::new(config, ZoneFactory::sync(|_| Some(()))) + .with_host_id("A") + .with_ownership(Arc::new(ownership(&base, "host-a"))); + assert!(server.activate("zl-game").await, "zone activated"); + let held = lease(&base, "zl-game").await; + let holder = held["state"]["holder"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!(holder.starts_with("A#"), "{held}"); + tokio::time::sleep(Duration::from_secs(5)).await; + let kept = lease(&base, "zl-game").await; + assert_eq!( + kept["state"]["holder"], + holder.as_str(), + "renewed past the TTL" + ); + assert_eq!(count(&kept, "zone.lease_expired"), 0); + server.shutdown(); + until( + "lease end after shutdown", + Duration::from_secs(20), + async || lease(&base, "zl-game").await["state"]["holder"].is_null(), + ) + .await; + let ended = lease(&base, "zl-game").await; + assert_eq!( + count(&ended, "zone.released") + count(&ended, "zone.lease_expired"), + 1, + "exactly one terminal event: {ended}" + ); + drop(celld); +} diff --git a/distributed_game/tests/celld_zone_state/main.rs b/distributed_game/tests/celld_zone_state/main.rs new file mode 100644 index 000000000..bf1d2b1b8 --- /dev/null +++ b/distributed_game/tests/celld_zone_state/main.rs @@ -0,0 +1,61 @@ +//! Hibernation snapshots in celld cells (`CellZoneStateStore` → +//! `ZoneState:{zone}`) on an isolated `celld dev`: the zone-state fencing +//! contract for an inline snapshot and for one above the inline limit (in +//! celld's R2 object storage), and both surviving a celld crash. +//! +//! (cd tests/celld/fact-worker && worker-build --release) +//! cargo test -p distributed_game --features celld,runtime --test celld_zone_state + +use distributed::cell_host::InternalHttpSecret; +use distributed_game::celld::CellZoneStateStore; +use distributed_game::world::zone_state::ZONE_STATE_INLINE_MAX; +use distributed_game::world::{state_store_conformance, StoredZoneState, ZoneStateStore}; + +#[path = "../support/celld.rs"] +mod celld_support; +use celld_support::{Celld, SECRET}; + +fn store(base: &str) -> CellZoneStateStore { + CellZoneStateStore::new( + base, + InternalHttpSecret::new(SECRET).unwrap(), + "zone-state-test", + "host-a", + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn snapshots_are_fenced_inline_and_in_object_storage_and_survive_a_crash() { + let Some(mut celld) = Celld::start_new() else { + return; + }; + let states = store(&celld.base()); + state_store_conformance(&states, "grid.0.0", vec![1; 512]).await; + let big: Vec = (0..ZONE_STATE_INLINE_MAX * 3) + .map(|i| (i % 253) as u8) + .collect(); + state_store_conformance(&states, "grid.0.1", big.clone()).await; + + // celld crashes (process tree killed) and restarts on its state. + celld.crash(); + celld.start().expect("celld restarts"); + let states = store(&celld.base()); + let small = states.load("grid.0.0").await.unwrap().unwrap(); + assert_eq!((small.token, small.state.len()), (5, 512)); + let large = states.load("grid.0.1").await.unwrap().unwrap(); + assert_eq!(large.token, 5); + assert_eq!(large.state, big, "the object came back byte for byte"); + // Fencing state survived too. + assert!(states + .save( + "grid.0.1", + &StoredZoneState { + token: 4, + content_version: "c1".into(), + state: vec![0; 10], + }, + ) + .await + .is_err()); +} diff --git a/distributed_game/tests/command_facts/main.rs b/distributed_game/tests/command_facts/main.rs new file mode 100644 index 000000000..da31577ac --- /dev/null +++ b/distributed_game/tests/command_facts/main.rs @@ -0,0 +1,286 @@ +//! The fact routes a zone server uses with celld (`tests/celld_facts`), on +//! an ordinary service instead: `CommandFactSink` over an +//! in-memory repository. Kills land in the killer's stream, match results in +//! the match's stream, lifecycle facts in the player's — no cell involved — +//! with the trusted principal and stamp, exactly once per fact key. + +use std::sync::Arc; + +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, Service, +}; +use distributed::{sourced, AggregateRepository, Entity, InMemoryRepository, LocalCommandHost}; +use distributed_game::cell::session_stamp; +use distributed_game::command_sink::{CommandFactSink, FactRoute}; +use distributed_game::lifecycle::{JoinVia, PlayerJoined, LIFECYCLE_COMMANDS}; +use distributed_game::{Fact, FactCommand, FactError, FactSink, FactStamp}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Serialize, FactCommand)] +#[fact(command = "kills.record")] +struct Kill { + mob: String, +} + +#[derive(Serialize, FactCommand)] +#[fact(command = "match.record_result")] +struct MatchResult { + match_id: String, + winner_slot: u32, +} + +/// One stream per routed shard: a player's kills and lifecycle, a match. +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +struct Log { + entity: Entity, + entries: Vec, +} + +#[sourced(entity, aggregate_type = "Log", events = "LogEvent")] +impl Log { + #[event("log.recorded", version = 1)] + fn recorded(&mut self, id: String, entry: String) { + if self.entity.id().is_empty() { + self.entity.set_id(id); + } + self.entries.push(entry); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +struct KillInput { + mob: String, +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +struct ResultInput { + match_id: String, + winner_slot: u32, +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +#[allow(dead_code)] +struct LifecycleInput { + zone: String, + via: Option, + from_zone: Option, + reason: Option, + grace_ms: Option, + cause: Option, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +struct Recorded { + stream: String, + entries: usize, +} + +fn rejected(error: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(error.to_string()) +} + +/// Append `entry` to stream `id` (the shard the route would address). +async fn record( + ctx: &CausalCommandContext<'_, Log>, + id: String, + entry: String, +) -> Result>, HandlerError> { + let repo = ctx.repo(); + let mut log = match repo.get(&id).await? { + Some(log) => log, + None => repo.create(), + }; + log.recorded(id.clone(), entry).map_err(rejected)?; + let entries = log.entries.len(); + repo.commit(log)?.succeeded(Recorded { + stream: id, + entries, + }) +} + +async fn handle_kill( + ctx: &CausalCommandContext<'_, Log>, + input: KillInput, +) -> Result>, HandlerError> { + // Same shard as `FactRoute::by_principal`: the trusted session's user. + let principal = ctx.user_id()?.to_string(); + let zone = session_stamp(ctx.session()) + .ok_or_else(|| rejected("no stamp"))? + .zone_id; + record( + ctx, + format!("kills/{principal}"), + format!("{}@{zone}", input.mob), + ) + .await +} + +async fn handle_result( + ctx: &CausalCommandContext<'_, Log>, + input: ResultInput, +) -> Result>, HandlerError> { + // Same shard as `FactRoute::by_input_field("match_id")`. + record( + ctx, + format!("match/{}", input.match_id), + format!("winner {}", input.winner_slot), + ) + .await +} + +async fn handle_joined( + ctx: &CausalCommandContext<'_, Log>, + input: LifecycleInput, +) -> Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + record( + ctx, + format!("presence/{principal}"), + format!("joined {}", input.zone), + ) + .await +} + +struct Commands; +impl PortableCommand for Commands +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + Kill::COMMAND, + )) + .handle(handle_kill) + .typed_command(typed_command::>( + MatchResult::COMMAND, + )) + .handle(handle_result) + .typed_command(typed_command::>( + LIFECYCLE_COMMANDS[0], + )) + .handle(handle_joined) + } +} + +fn stamp(tick: u64) -> FactStamp { + FactStamp { + zone_id: "pit-1".into(), + tick, + content_version: "c1".into(), + emitted_at_ms: 1_700_000_000_000 + tick, + activation: 1, + config_version: "cfg-1".into(), + seq: tick, + } +} + +fn fact( + key: &str, + input: &C, + principal: Option<&str>, + tick: u64, +) -> Fact { + Fact::trusted( + key, + C::COMMAND, + serde_json::to_value(input).unwrap(), + principal.map(str::to_string), + stamp(tick), + ) +} + +#[tokio::test] +async fn the_celld_fact_routes_land_on_a_service_over_any_repository() { + let repo = InMemoryRepository::new(); + let routes = Commands.install(Routes::new().with_repo(AggregateRepository::new(repo.clone()))); + let service = Arc::new(Service::new().named("zone-server-test").routes(routes)); + // The same routes as the celld fact-routing test. + let sink = CommandFactSink::new( + LocalCommandHost::new(service), + "zone-server-test", + "zone:pit-1", + ) + .unwrap() + .route(FactRoute::by_principal::("kills")) + .route(FactRoute::by_input_field::( + "match", "match_id", + )) + .lifecycle_route("presence"); + + let kill = fact("kill:1", &Kill { mob: "m1".into() }, Some("alice"), 1); + let first = sink.commit(&kill).await.unwrap(); + assert!(!first.replayed); + assert_eq!( + first.payload, + json!({ "stream": "kills/alice", "entries": 1 }) + ); + let again = sink.commit(&kill).await.unwrap(); + assert!(again.replayed, "same fact key replays through the ledger"); + assert_eq!(again.payload, first.payload); + let second = sink + .commit(&fact( + "kill:2", + &Kill { mob: "m2".into() }, + Some("alice"), + 2, + )) + .await + .unwrap(); + assert_eq!(second.payload["entries"], 2, "credited once per key"); + + let result = sink + .commit(&fact( + "result:1", + &MatchResult { + match_id: "m-7".into(), + winner_slot: 1, + }, + None, + 3, + )) + .await + .unwrap(); + assert_eq!(result.payload["stream"], "match/m-7"); + + let joined = PlayerJoined { + zone: "pit-1".into(), + via: JoinVia::Ticket, + from_zone: None, + }; + let lifecycle = Fact::trusted( + "joined:1", + LIFECYCLE_COMMANDS[0], + serde_json::to_value(&joined).unwrap(), + Some("alice".into()), + stamp(4), + ); + let presence = sink.commit(&lifecycle).await.unwrap(); + assert_eq!(presence.payload["stream"], "presence/alice"); + + // Unrouted commands are permanent; so is a command the service rejects. + let unrouted = Fact::trusted( + "x", + "guild.whatever", + json!({}), + Some("alice".into()), + stamp(5), + ); + assert!(matches!( + sink.commit(&unrouted).await, + Err(FactError::Permanent(_)) + )); + let forged = Fact::trusted( + "kill:3", + Kill::COMMAND, + json!({ "mob": "m3", "xp": 9999 }), + Some("alice".into()), + stamp(6), + ); + assert!(matches!( + sink.commit(&forged).await, + Err(FactError::Permanent(_)) + )); +} diff --git a/distributed_game/tests/entities/main.rs b/distributed_game/tests/entities/main.rs new file mode 100644 index 000000000..3b9c1ccf3 --- /dev/null +++ b/distributed_game/tests/entities/main.rs @@ -0,0 +1,247 @@ +//! Zone entities projected into views (`distributed_game::entity`): +//! `#[replicated(.., of = T)]` + `#[from]`, `Project`, and the host +//! projecting registered collections before `Zone::replicate`. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use distributed_game::{ + schema, Ctx, Entities, Entity, EntityId, EntitySchema, FieldValue, HostConfig, ManualClock, + PlayerId, Project, Replicated, WorldView, Zone, ZoneHost, +}; + +#[derive(Debug, Clone, Copy, Default)] +struct Body { + x: f32, + y: f32, +} + +/// Server state: `home` and `hp` never leave the zone as such. +struct Mob { + id: EntityId, + body: Body, + home: (f32, f32), + hp: i32, + max_hp: i32, +} + +impl Entity for Mob { + fn entity_id(&self) -> EntityId { + self.id + } +} + +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "mob", id = 1, of = Mob)] +struct MobView { + #[from(body.x)] + x: f32, + #[from(body.y)] + y: f32, + /// HP percent. + hp: u8, +} + +impl Project for Mob { + fn view(&self) -> Option { + (self.hp > 0).then(|| { + MobView::project( + self, + MobViewRest { + hp: (self.hp * 100 / self.max_hp) as u8, + }, + ) + }) + } +} + +/// Every field `#[from]`: the `Project` impl is generated. +#[derive(Debug, Clone, Copy)] +struct Rock { + id: EntityId, + x: f32, + y: f32, + r: f32, +} + +impl Entity for Rock { + fn entity_id(&self) -> EntityId { + self.id + } +} + +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "rock", id = 2, global, of = Rock)] +struct RockView { + #[from] + x: f32, + #[from] + y: f32, + #[from] + r: f32, +} + +/// The zone projects its own HUD numbers. +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "hud", id = 3, global, of = Field)] +struct HudView { + alive: u8, +} + +/// A view without `of`, pushed by hand in `replicate`. +#[derive(Debug, Clone, PartialEq, Replicated)] +#[replicated(kind = "marker", id = 4)] +struct MarkerView { + x: f32, + y: f32, +} + +struct Field { + mobs: Vec, + rocks: BTreeMap, + boss: Option, +} + +impl Entity for Field { + fn entity_id(&self) -> EntityId { + EntityId(900) + } +} + +impl Project for Field { + fn view(&self) -> Option { + let alive = self.mobs.iter().filter(|m| m.hp > 0).count(); + Some(HudView::project(self, HudViewRest { alive: alive as u8 })) + } +} + +fn mob(id: u32, x: f32, hp: i32) -> Mob { + Mob { + id: EntityId(id), + body: Body { x, y: -x }, + home: (x, x), + hp, + max_hp: 40, + } +} + +impl Zone for Field { + type Config = (); + type Input = (); + type Timer = (); + type Message = distributed_game::NoMessages; + + fn schema() -> EntitySchema { + schema![MobView, RockView, HudView, MarkerView] + } + + fn entities() -> Entities { + Entities::new() + .each(|z: &Self| &z.mobs) + .each(|z: &Self| &z.rocks) + .each(|z: &Self| &z.boss) + .one(|z: &Self| z) + } + + fn on_create(_ctx: &mut Ctx<'_, Self>, _config: ()) -> Self { + Self { + mobs: vec![mob(1, 1.0, 40), mob(2, 2.0, 0), mob(3, 3.0, 10)], + rocks: [(7u8, 5.0f32), (3, 4.0)] + .into_iter() + .map(|(k, x)| { + ( + k, + Rock { + id: EntityId(100 + u32::from(k)), + x, + y: 0.0, + r: 0.5, + }, + ) + }) + .collect(), + boss: Some(mob(50, 9.0, 20)), + } + } + + fn decode_input(&self, _player: PlayerId, _payload: &[u8]) -> Option<()> { + None + } + + fn on_tick(&mut self, _ctx: &mut Ctx<'_, Self>) {} + + fn replicate(&self, out: &mut WorldView) { + out.add(EntityId(77), &MarkerView { x: 0.5, y: 0.5 }); + } +} + +#[test] +fn registered_collections_are_projected_in_order_then_replicate_runs() { + let host = ZoneHost::::new( + "field", + HostConfig::default(), + Arc::new(ManualClock::new()), + (), + ); + let view = host.world_view(); + let ids: Vec<(u32, u16)> = view.entities().iter().map(|e| (e.id, e.kind)).collect(); + assert_eq!( + ids, + [ + (1, 1), // mobs in Vec order; mob 2 is dead: no view + (3, 1), // + (103, 2), // rocks in BTreeMap key order (3, 7) + (107, 2), // + (50, 1), // the Option + (900, 3), // the zone itself + (77, 4), // replicate() last + ] + ); + let mob3 = &view.entities()[1]; + assert_eq!( + mob3.fields, + [FieldValue::F(3.0), FieldValue::F(-3.0), FieldValue::U(25)], + "#[from(body.x)] copied, hp derived" + ); + assert_eq!((mob3.x, mob3.y), (3.0, -3.0)); + assert!(view.entities()[2].global, "kind flags carry over"); + assert_eq!(view.entities()[5].fields, [FieldValue::U(2)]); + // Unreplicated server state stays put. + assert_eq!(host.zone().mobs[0].home, (1.0, 1.0)); +} + +#[test] +fn hand_written_replicate_can_project_entities_too() { + let mut out = WorldView::default(); + let dead = mob(9, 0.0, 0); + assert!(out.project::(&dead).is_none(), "None omits"); + out.project::(&mob(8, 1.0, 40)) + .expect("alive") + .priority(3.0); + let rock = Rock { + id: EntityId(5), + x: 1.0, + y: 2.0, + r: 0.25, + }; + out.project::(&rock); + assert_eq!(out.entities().len(), 2); + assert_eq!(out.entities()[0].priority, 3.0); + assert_eq!( + RockView::project(&rock), + RockView { + x: 1.0, + y: 2.0, + r: 0.25 + } + ); + assert_eq!(rock.view(), Some(RockView::project(&rock))); +} + +#[test] +fn entity_ids_convert_and_display_as_the_raw_id() { + let id = EntityId::from(1002); + assert_eq!(u32::from(id), 1002); + assert_eq!(id.get(), 1002); + assert_eq!(id.to_string(), "1002"); +} diff --git a/distributed_game/tests/fixtures/authoring.json b/distributed_game/tests/fixtures/authoring.json index 5983f862c..492d1eb2d 100644 --- a/distributed_game/tests/fixtures/authoring.json +++ b/distributed_game/tests/fixtures/authoring.json @@ -107,6 +107,26 @@ "viewTick": 123456 } }, + "messages": [ + { + "hex": "13736572766572207265737461727420696e203501", + "id": 1, + "name": "announce", + "value": { + "text": "server restart in 5", + "urgent": true + } + }, + { + "hex": "0000204007", + "id": 2, + "name": "set_wind", + "value": { + "gusts": 7, + "speed": 2.5 + } + } + ], "pures": [ { "arg": "0000c03fffff03", @@ -547,6 +567,40 @@ "priority": 1.0 } ], + "messages": [ + { + "id": 1, + "name": "announce", + "payload": { + "fields": [ + { + "name": "text", + "type": { + "type": "str" + } + }, + { + "name": "urgent", + "type": { + "type": "bool" + } + } + ], + "kind": "fields" + } + }, + { + "id": 2, + "name": "set_wind", + "payload": { + "kind": "value", + "type": { + "name": "SampleWind", + "type": "record" + } + } + } + ], "name": "sample", "predict": { "abi": 1, diff --git a/distributed_game/tests/fixtures/sveltekit_tickets.json b/distributed_game/tests/fixtures/sveltekit_tickets.json index ca6fbe151..3c231db41 100644 --- a/distributed_game/tests/fixtures/sveltekit_tickets.json +++ b/distributed_game/tests/fixtures/sveltekit_tickets.json @@ -12,6 +12,14 @@ "ticket": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMiIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLW9ic2VydmVyIiwicm9sZSI6Im9ic2VydmVyIn0.1j2Ql0XePnslZ1kJf64VhjUeDYkdl_nAt6Vep2zQ1bs", "principal": "session-user-2" }, + "extended": { + "ticket": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMyIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLWV4dGVuZGVkIiwicm9sZSI6InBsYXllciIsImV4dCI6eyJtYXRjaF9pZCI6Im0tNyIsInRlYW0iOjJ9fQ.sQwdQFAEEsYZcCy2qB3szKm8bIBRWqmngIdE-FGwfwo", + "principal": "session-user-3", + "ext": { + "match_id": "m-7", + "team": 2 + } + }, "expired": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMSIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6MzAwLCJub25jZSI6ImZpeHR1cmUtZXhwaXJlZCIsInJvbGUiOiJwbGF5ZXIifQ.pJED6dt-O-JNtFi-fqzUFNPtEWYljUs0pKR9gJrX380", "wrongZone": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJzZXNzaW9uLXVzZXItMSIsInpvbmUiOiJ6LWVsc2V3aGVyZSIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLXdyb25nLXpvbmUiLCJyb2xlIjoicGxheWVyIn0.s4mdpfnnVke8Ntt_LjHAgonARO5RRt_KVgKH5v96k7A", "tampered": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhZG1pbiIsInpvbmUiOiJ6LXN2ZWx0ZWtpdCIsImV4cCI6NDAwMDAwMDAwMCwibm9uY2UiOiJmaXh0dXJlLXRhbXBlcmVkIiwicm9sZSI6InBsYXllciJ9.k0UNks88hcYz9M18yZdUA2OPH-Xwgf6jBwi6snvkoJE", diff --git a/distributed_game/tests/fixtures/trusted_field_denylist.json b/distributed_game/tests/fixtures/trusted_field_denylist.json new file mode 100644 index 000000000..36b210253 --- /dev/null +++ b/distributed_game/tests/fixtures/trusted_field_denylist.json @@ -0,0 +1,6 @@ +{ + "comment": "Names a zone fact (FactCommand) or a ticket extension claim (TicketExtension, createGameTicketHandler claims) must never use. Matching ignores '_', '-' and case. Checked against distributed_game_macros/src/trusted.rs and js/src/sveltekit/game/ticket.ts by their tests.", + "identity": ["principal", "user", "user_id", "owner", "owner_id", "account", "account_id", "subject", "sub"], + "reward": ["xp", "experience", "reward", "rewards"], + "economy": ["amount", "coins", "gold", "currency", "price", "cost", "balance", "grant", "payout", "credit", "debit"] +} diff --git a/distributed_game/tests/inbox/main.rs b/distributed_game/tests/inbox/main.rs new file mode 100644 index 000000000..9d3c08437 --- /dev/null +++ b/distributed_game/tests/inbox/main.rs @@ -0,0 +1,398 @@ +//! The zone inbox: typed messages and the built-in kick, delivered on the +//! zone task between ticks; `GameServer::send`, `/admin/zone-message` and +//! `ZoneInboxClient` (auth, inactive zones, wrong owner). + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use distributed_game::lifecycle::{LeftReason, Lifecycle}; +use distributed_game::net::peer::post_json; +use distributed_game::net::{ + GameServer, InboxError, ServerConfig, TicketKey, ZoneFactory, ZoneInboxClient, + ZONE_MESSAGE_PATH, +}; +use distributed_game::protocol::kick; +use distributed_game::world::{HostAddr, StaticDirectory}; +use distributed_game::*; +use serde_json::json; + +const SECRET: &str = "inbox-secret"; +const ADMIN: &str = "inbox-admin"; + +/// Messages a service may send into a hall. +#[derive(Debug, Clone, PartialEq, GameEvent)] +pub enum HallMessage { + /// Change the message of the day. + #[game_event(id = 1)] + SetMotd { + /// New text. + #[long] + motd: String, + }, + /// Numbered probe (ordering / tick checks). + #[game_event(id = 2)] + Probe { + /// Sequence. + n: u32, + }, +} + +#[derive(Default)] +struct Shared { + /// `(n, tick seen, delivered inside a tick?)` per probe. + probes: Vec<(u32, u64, bool)>, +} + +struct Hall { + motd: String, + in_tick: bool, + ticks: u64, + left: Vec, + shared: Arc>, +} + +impl Zone for Hall { + type Config = Arc>; + type Input = u8; + type Timer = (); + type Message = HallMessage; + + fn schema() -> EntitySchema { + EntitySchema::new() + } + + fn on_create(_ctx: &mut Ctx<'_, Self>, shared: Arc>) -> Self { + Self { + motd: String::new(), + in_tick: false, + ticks: 0, + left: Vec::new(), + shared, + } + } + + fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option { + payload.first().copied() + } + + fn on_tick(&mut self, _ctx: &mut Ctx<'_, Self>) { + self.in_tick = true; + self.ticks += 1; + // A little work so a concurrent delivery would have a window. + std::thread::sleep(Duration::from_micros(200)); + self.in_tick = false; + } + + fn on_message(&mut self, ctx: &mut Ctx<'_, Self>, message: HallMessage) { + match message { + HallMessage::SetMotd { motd } => self.motd = motd, + HallMessage::Probe { n } => { + self.shared + .lock() + .unwrap() + .probes + .push((n, ctx.tick(), self.in_tick)); + } + } + } + + fn on_leave(&mut self, _ctx: &mut Ctx<'_, Self>, _player: PlayerId, reason: LeaveReason) { + self.left.push(reason); + } + + /// Keep ticking at the Hot rate with nobody connected. + fn has_pending_work(&self) -> bool { + true + } + + fn replicate(&self, _out: &mut WorldView) {} +} + +fn claims(principal: &str, nonce: &str) -> TicketClaims { + TicketClaims::new(principal, "hall", u64::MAX, nonce, Role::Player) +} + +#[test] +fn kick_removes_the_principal_with_a_lifecycle_fact_and_messages_reach_on_message() { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "hall", + HostConfig { + lifecycle_facts: true, + ..HostConfig::default() + }, + Arc::new(clock.clone()), + Arc::default(), + ) + .with_facts(queue); + let ana = h.join(&claims("ana", "a"), 0, "").unwrap(); + h.join(&claims("bo", "b"), 0, "").unwrap(); + h.drain_outbound(); + + let delivered = h.deliver(ZoneInbound::Kick { + principal: "ana".into(), + reason: "banned".into(), + }); + assert_eq!(delivered.players_kicked, 1); + assert_eq!(h.zone().left, [LeaveReason::Kicked]); + assert_eq!(h.player_count(), 1); + let out = h.drain_outbound(); + let kicked = out.iter().any(|o| match o { + Outbound::Frame { client, frame } if *client == ana.client => matches!( + ServerMessage::decode(frame, &()), + Ok(ServerMessage::Kick { code, ref message }) if code == kick::BY_ZONE && message == "banned" + ), + _ => false, + }); + assert!(kicked, "the client is told why"); + // Kicking someone absent is a no-op. + assert_eq!( + h.deliver(ZoneInbound::Kick { + principal: "nobody".into(), + reason: String::new(), + }), + Delivered::default() + ); + h.deliver(ZoneInbound::Message(HallMessage::SetMotd { + motd: "welcome".into(), + })); + assert_eq!(h.zone().motd, "welcome"); + + drop(h); + let sink = MemoryFactSink::new(); + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + let mut run = std::pin::pin!(feed.run(sink.clone(), |_| async {})); + while std::future::Future::poll(run.as_mut(), &mut cx).is_pending() {} + let ana: Vec<_> = sink + .committed() + .iter() + .filter(|f| f.principal() == Some("ana")) + .filter_map(Lifecycle::of) + .collect(); + assert!( + matches!(ana.last(), Some(Lifecycle::Left(l)) if l.reason == LeftReason::Kicked), + "{ana:?}" + ); +} + +#[test] +fn messages_are_in_the_surface_with_their_codec() { + let surface = GameSurface::new("hall").messages::<::Message>(); + let names: Vec<_> = surface.messages.iter().map(|m| m.name.as_str()).collect(); + assert_eq!(names, ["set_motd", "probe"]); + assert!(surface.to_json().contains("\"messages\"")); + // No messages: nothing in the JSON (older surfaces stay identical). + let quiet = GameSurface::new("quiet").messages::(); + assert!(!quiet.to_json().contains("messages")); + let message = HallMessage::Probe { n: 7 }; + let back = HallMessage::decode_event(message.event_id(), &message.to_payload()).unwrap(); + assert_eq!(back, Some(message)); +} + +async fn serve( + server: &GameServer, + listener: tokio::net::TcpListener, +) -> tokio::task::JoinHandle<()> { + let serving = server.clone(); + tokio::spawn(async move { + serving + .serve(listener, std::future::pending()) + .await + .unwrap(); + }) +} + +fn server(shared: Arc>) -> GameServer { + let mut config = ServerConfig::new(TicketKey::new(SECRET)); + config.admin_secret = Some(ADMIN.into()); + config.host.tick_rate = 120; + GameServer::new( + config, + ZoneFactory::sync(move |a| a.zone_id.starts_with("hall").then(|| shared.clone())), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn on_message_runs_on_the_zone_task_between_ticks() { + let shared = Arc::new(Mutex::new(Shared::default())); + let server = server(shared.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let handle = serve(&server, listener).await; + assert!(server.activate("hall").await); + let senders: Vec<_> = (0..64u32) + .map(|n| { + let server = server.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(u64::from(n % 16) * 3)).await; + server + .send("hall", ZoneInbound::Message(HallMessage::Probe { n })) + .await + }) + }) + .collect(); + for sender in senders { + sender.await.unwrap().unwrap(); + } + let probes = shared.lock().unwrap().probes.clone(); + assert_eq!(probes.len(), 64); + assert!( + probes.iter().all(|(_, _, in_tick)| !in_tick), + "never inside a tick" + ); + let ticks: std::collections::BTreeSet<_> = probes.iter().map(|(_, t, _)| *t).collect(); + assert!(ticks.len() > 1, "delivered across several ticks: {ticks:?}"); + + // A message for a zone that is not running is refused, never activates it. + assert_eq!( + server + .send( + "hall-cold", + ZoneInbound::Message(HallMessage::Probe { n: 0 }) + ) + .await, + Err(InboxError::Inactive) + ); + assert!(!server.is_active("hall-cold")); + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .unwrap() + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn admin_route_authenticates_refuses_inactive_and_redirects_to_the_owner() { + // Two hosts; the shared directory places `hall.a` on A. + let (la, lb) = ( + tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(), + tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(), + ); + let a_addr = HostAddr::new("A", la.local_addr().unwrap().to_string()); + let b_addr = HostAddr::new("B", lb.local_addr().unwrap().to_string()); + let directory = Arc::new(StaticDirectory::new()); + directory.add_host(a_addr.clone()).add_host(b_addr.clone()); + directory.assign("hall.a", "A").assign("hall.cold", "A"); + let shared = Arc::new(Mutex::new(Shared::default())); + let a = server(shared.clone()) + .with_host_id("A") + .with_directory(directory.clone()); + let b = server(shared.clone()) + .with_host_id("B") + .with_directory(directory.clone()); + let (ha, hb) = (serve(&a, la).await, serve(&b, lb).await); + assert!(a.activate("hall.a").await); + let probe = |n: u32| { + let m = HallMessage::Probe { n }; + json!({ + "zone": "hall.a", + "message": { + "type_id": m.event_id(), + "payload": base64_url(&m.to_payload()), + } + }) + }; + let post = |url: String, secret: &'static str, body: serde_json::Value| async move { + post_json( + &url, + ZONE_MESSAGE_PATH, + secret, + &body, + Duration::from_secs(3), + ) + .await + .unwrap() + }; + + // Unauthenticated (wrong or empty secret) is refused before anything. + for secret in ["", "wrong"] { + let (status, body) = post(a_addr.admin_url.clone(), secret, probe(1)).await; + assert_eq!( + (status, body["error"].as_str()), + (401, Some("unauthorized")) + ); + } + // The owner delivers. + let (status, _) = post(a_addr.admin_url.clone(), ADMIN, probe(2)).await; + assert_eq!(status, 200); + // The wrong host answers with the owner instead of delivering. + let (status, body) = post(b_addr.admin_url.clone(), ADMIN, probe(3)).await; + assert_eq!(status, 421, "{body}"); + assert_eq!(body["error"], "wrong_owner"); + assert_eq!(body["owner"], "A"); + assert_eq!(body["admin_url"], a_addr.admin_url); + // Inactive zones: 404, and still inactive. + let mut cold = probe(4); + cold["zone"] = json!("hall.cold"); + let (status, body) = post(a_addr.admin_url.clone(), ADMIN, cold).await; + assert_eq!((status, body["error"].as_str()), (404, Some("inactive"))); + assert!(!a.is_active("hall.cold")); + // Unknown message ids and malformed payloads: 400. + let bad = json!({"zone": "hall.a", "message": {"type_id": 99, "payload": ""}}); + let (status, _) = post(a_addr.admin_url.clone(), ADMIN, bad).await; + assert_eq!(status, 400); + let short = json!({"zone": "hall.a", "message": {"type_id": 2, "payload": ""}}); + let (status, _) = post(a_addr.admin_url.clone(), ADMIN, short).await; + assert_eq!(status, 400); + + // A service whose directory is stale (says B) follows the redirect. + let stale = Arc::new(StaticDirectory::new()); + stale.add_host(b_addr.clone()).assign("hall.a", "B"); + let client = ZoneInboxClient::new(stale, ADMIN); + client + .send("hall.a", &HallMessage::SetMotd { motd: "hi".into() }) + .await + .unwrap(); + assert_eq!( + a.inspect("hall.a", |h| h.zone().motd.clone()) + .await + .unwrap(), + "hi" + ); + client + .send("hall.a", &HallMessage::Probe { n: 5 }) + .await + .unwrap(); + assert_eq!( + client.kick("hall.a", "nobody", "bye").await.unwrap(), + Delivered::default() + ); + assert_eq!( + client.send("hall.cold", &HallMessage::Probe { n: 6 }).await, + Err(InboxError::Inactive) + ); + let unauthorized = ZoneInboxClient::new(directory.clone(), "wrong"); + assert_eq!( + unauthorized + .send("hall.a", &HallMessage::Probe { n: 7 }) + .await, + Err(InboxError::Unauthorized) + ); + let ns: Vec<_> = shared.lock().unwrap().probes.iter().map(|p| p.0).collect(); + assert_eq!(ns, [2, 5], "only the authorized, owned deliveries ran"); + + a.shutdown(); + b.shutdown(); + for handle in [ha, hb] { + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .unwrap() + .unwrap(); + } +} + +fn base64_url(bytes: &[u8]) -> String { + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + let mut out = String::new(); + for chunk in bytes.chunks(3) { + let n = chunk + .iter() + .enumerate() + .fold(0u32, |acc, (i, b)| acc | (u32::from(*b) << (16 - 8 * i))); + for i in 0..=chunk.len() { + out.push(ALPHABET[((n >> (18 - 6 * i)) & 63) as usize] as char); + } + } + out +} diff --git a/distributed_game/tests/lifecycle/main.rs b/distributed_game/tests/lifecycle/main.rs new file mode 100644 index 000000000..1b1a5c67d --- /dev/null +++ b/distributed_game/tests/lifecycle/main.rs @@ -0,0 +1,372 @@ +//! Host-emitted player lifecycle facts (`HostConfig::lifecycle_facts`): +//! the five transitions, restart-safe keys, per-principal order under +//! concurrent commits, backpressure backlog and the connection hooks. + +#[path = "../zone_runtime/arena.rs"] +mod arena; + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use arena::{Arena, ArenaConfig}; +use distributed_game::lifecycle::{ + JoinVia, LeftReason, Lifecycle, LostCause, CONNECTION_LOST, PLAYER_DISCONNECTED, PLAYER_JOINED, + PLAYER_RECONNECTED, +}; +use distributed_game::protocol::ServerMessage; +use distributed_game::{ + fact_channel, spawn_fact_worker, BoxFuture, ClientId, Fact, FactError, FactFeed, FactReceipt, + FactSink, HostConfig, ManualClock, MemoryFactSink, Outbound, RetryPolicy, Role, TicketClaims, + ZoneHost, +}; + +const MS: fn(u64) -> Duration = Duration::from_millis; + +fn claims(principal: &str, nonce: &str) -> TicketClaims { + TicketClaims::new(principal, "z1", u64::MAX, nonce, Role::Player) +} + +fn config(boot_epoch: u64) -> HostConfig { + HostConfig { + lifecycle_facts: true, + boot_epoch, + reconnect_grace: MS(500), + ..HostConfig::default() + } +} + +fn token(out: &[Outbound], client: ClientId) -> String { + out.iter() + .find_map(|o| match o { + Outbound::Frame { client: c, frame } if *c == client => { + match ServerMessage::decode(frame, &()).ok()? { + ServerMessage::Welcome(w) => Some(w.reconnect_token), + _ => None, + } + } + _ => None, + }) + .expect("welcome") +} + +/// Drive a whole scripted session and return the committed facts. +fn run_session(boot_epoch: u64) -> (Vec, Vec) { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(64, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "z1", + config(boot_epoch), + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + + // ana: join, drop, reconnect, reload the page (fresh ticket), drop, + // grace expires. + let ana = h.join(&claims("ana", "a1"), 0, "").unwrap(); + let ana_token = token(&h.drain_outbound(), ana.client); + h.disconnect(ana.client, false); + let back = h.reconnect(&ana_token, 0).unwrap(); + h.drain_outbound(); + let again = h.join(&claims("ana", "a2"), 0, "").unwrap(); + assert_ne!(back.client, again.client); + h.disconnect(again.client, false); + clock.advance(MS(600)); + h.advance(); + // bo: kicked. cy: leaves gracefully. dee: still here at dispose. + let bo = h.join(&claims("bo", "b1"), 0, "").unwrap(); + let Some(distributed_game::Viewer::Player(bo_id)) = h.viewer_of(bo.client) else { + panic!("bo is a player") + }; + h.kick(bo_id, "cheating"); + let cy = h.join(&claims("cy", "c1"), 0, "").unwrap(); + h.disconnect(cy.client, true); + h.join(&claims("dee", "d1"), 0, "").unwrap(); + h.dispose(); + let hooks = h.zone().connection_events.clone(); + drop(h); + let sink = MemoryFactSink::new(); + block_on(feed.run(sink.clone(), |_| async {})); + (sink.committed(), hooks) +} + +fn block_on(f: impl std::future::Future) { + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + let mut f = std::pin::pin!(f); + while f.as_mut().poll(&mut cx).is_pending() {} +} + +fn of(facts: &[Fact], principal: &str) -> Vec { + facts + .iter() + .filter(|f| f.principal() == Some(principal)) + .map(|f| Lifecycle::of(f).expect("only lifecycle facts")) + .collect() +} + +#[test] +fn the_host_emits_every_transition_bound_to_the_verified_principal() { + let (facts, hooks) = run_session(7); + assert!(facts.iter().all(|f| f.stamp().zone_id == "z1")); + // Every stamp is strictly later than the previous one, even for + // transitions in the same tick and millisecond (emission sequence). + for pair in facts.windows(2) { + assert!( + pair[1].stamp().emitted_after(pair[0].stamp()), + "{:?} then {:?}", + pair[0].stamp(), + pair[1].stamp() + ); + } + let ana = of(&facts, "ana"); + let commands: Vec<_> = facts + .iter() + .filter(|f| f.principal() == Some("ana")) + .map(Fact::command) + .collect(); + assert_eq!( + commands, + [ + PLAYER_JOINED, + CONNECTION_LOST, + PLAYER_RECONNECTED, + PLAYER_JOINED, + CONNECTION_LOST, + PLAYER_DISCONNECTED + ] + ); + let Lifecycle::Joined(first) = &ana[0] else { + panic!() + }; + assert_eq!(first.via, JoinVia::Ticket); + let Lifecycle::ConnectionLost(lost) = &ana[1] else { + panic!() + }; + assert_eq!((lost.grace_ms, lost.cause), (500, LostCause::Dropped)); + let Lifecycle::Joined(rejoin) = &ana[3] else { + panic!() + }; + assert_eq!(rejoin.via, JoinVia::Rejoin); + assert!(ana[5].is_terminal()); + + let reasons: Vec<_> = ["bo", "cy", "dee"] + .iter() + .map(|p| match of(&facts, p).as_slice() { + [Lifecycle::Joined(_), Lifecycle::Left(left)] => left.reason, + other => panic!("{p}: {other:?}"), + }) + .collect(); + assert_eq!( + reasons, + [LeftReason::Kicked, LeftReason::Left, LeftReason::Disposed] + ); + + // Keys: command, zone, boot epoch, slot, transition counter (ana's + // slot is PlayerId(2): ids are shared with connection ids). + let keys: Vec<_> = facts + .iter() + .filter(|f| f.principal() == Some("ana")) + .map(|f| f.key().to_string()) + .collect(); + assert_eq!(keys[0], format!("{PLAYER_JOINED}:z1:7:p2:1")); + assert_eq!(keys[5], format!("{PLAYER_DISCONNECTED}:z1:7:p2:6")); + // The payload names no identity; the principal is the binding. + assert!(facts.iter().all(|f| f.input().get("principal").is_none())); + + // Connection hooks: grace start and reconnect. + assert_eq!(hooks, ["lost:2", "back:2", "lost:2"]); +} + +#[test] +fn keys_are_stable_per_run_and_distinct_after_a_restart() { + let (first, _) = run_session(100); + let (same, _) = run_session(100); + let (restarted, _) = run_session(101); + let keys = |facts: &[Fact]| { + facts + .iter() + .map(|f| f.key().to_string()) + .collect::>() + }; + assert_eq!(keys(&first), keys(&same), "deterministic per activation"); + let before: std::collections::HashSet<_> = keys(&first).into_iter().collect(); + assert_eq!(before.len(), first.len(), "unique within a run"); + assert!( + keys(&restarted).iter().all(|k| !before.contains(k)), + "a restart never reuses a key" + ); +} + +#[test] +fn off_by_default() { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "z1", + HostConfig::default(), + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + let ana = h.join(&claims("ana", "a1"), 0, "").unwrap(); + h.disconnect(ana.client, false); + let bo = h.join(&claims("bo", "b1"), 0, "").unwrap(); + h.disconnect(bo.client, true); + assert_eq!(h.zone().connection_events, ["lost:2"], "hooks still run"); + drop(h); + let sink = MemoryFactSink::new(); + block_on(feed.run(sink.clone(), |_| async {})); + assert!(sink.committed().is_empty()); +} + +/// Commits slowly for "joined" (so a later fact of the same principal would +/// overtake it if lanes were not respected) and fails the first attempt of +/// every reconnect (retries keep the key). +#[derive(Clone, Default)] +struct SlowSink { + order: Arc>>, + attempts: Arc>>, +} + +impl FactSink for SlowSink { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + let attempt = { + let mut attempts = self.attempts.lock().unwrap(); + let n = attempts.entry(fact.key().to_string()).or_default(); + *n += 1; + *n + }; + let delay = match fact.command() { + PLAYER_JOINED => 40, + CONNECTION_LOST => 5, + _ => 1, + }; + tokio::time::sleep(MS(delay)).await; + if fact.command() == PLAYER_RECONNECTED && attempt == 1 { + return Err(FactError::Retryable("flaky".into())); + } + self.order.lock().unwrap().push(( + fact.principal().unwrap_or_default().to_string(), + fact.command().to_string(), + fact.key().to_string(), + )); + Ok(FactReceipt { + payload: serde_json::Value::Null, + replayed: false, + }) + }) + } +} + +fn concurrent(feed: FactFeed) -> FactFeed { + feed.with_max_concurrent_commits(4) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn per_principal_order_holds_with_concurrent_commits_and_backpressure() { + let clock = ManualClock::new(); + // Capacity 2: most lifecycle facts wait in the host's backlog. + let policy = RetryPolicy { + base_backoff: MS(2), + ..RetryPolicy::default() + }; + let (queue, feed) = fact_channel(2, policy); + let sink = SlowSink::default(); + let worker = spawn_fact_worker(concurrent(feed), sink.clone()); + let mut h = ZoneHost::::new( + "z1", + config(9), + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + + let mut emitted: BTreeMap> = BTreeMap::new(); + for principal in ["ana", "bo", "cy"] { + let joined = h.join(&claims(principal, principal), 0, "").unwrap(); + let token = token(&h.drain_outbound(), joined.client); + h.disconnect(joined.client, false); + h.reconnect(&token, 0).unwrap(); + h.drain_outbound(); + emitted.insert( + principal.into(), + vec![PLAYER_JOINED, CONNECTION_LOST, PLAYER_RECONNECTED], + ); + } + assert!(h.facts_in_flight() > 2, "backlog beyond the queue capacity"); + // The host keeps flushing its backlog as the queue drains. + for _ in 0..400 { + if h.facts_in_flight() == 0 { + break; + } + tokio::time::sleep(MS(5)).await; + clock.advance(MS(5)); + h.advance(); + } + assert_eq!(h.facts_in_flight(), 0, "every lifecycle fact settled"); + drop(h); + worker.await.unwrap(); + + let order = sink.order.lock().unwrap().clone(); + assert_eq!(order.len(), 9, "nothing dropped: {order:?}"); + for (principal, expected) in &emitted { + let committed: Vec<_> = order + .iter() + .filter(|(p, _, _)| p == principal) + .map(|(_, command, _)| command.as_str()) + .collect(); + assert_eq!(&committed, expected, "{principal}"); + } + // The flaky reconnects were retried under the same key. + let attempts = sink.attempts.lock().unwrap(); + for (_, command, key) in order.iter().filter(|(_, c, _)| c == PLAYER_RECONNECTED) { + assert_eq!(attempts[key], 2, "{command} retried once with key {key}"); + } +} + +#[test] +fn handoff_and_reload_are_lifecycle_transitions_too() { + let clock = ManualClock::new(); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = ZoneHost::::new( + "z1", + HostConfig { + content_version: "v1".into(), + ..config(3) + }, + Arc::new(clock.clone()), + ArenaConfig::default(), + ) + .with_facts(queue); + let transfer = distributed_game::Transfer { + claims: claims("ana", "t1"), + source_zone: "z0".into(), + issuer: "h0".into(), + generation: 1, + carried: Vec::new(), + departed_at_ms: 0, + }; + let joined = h.join_transfer(&transfer, 0, "v1").unwrap(); + let Some(distributed_game::Viewer::Player(ana)) = h.viewer_of(joined.client) else { + panic!() + }; + h.reload_player(ana); + h.handoff(ana, "z2", "ticket"); + drop(h); + let sink = MemoryFactSink::new(); + block_on(feed.run(sink.clone(), |_| async {})); + match of(&sink.committed(), "ana").as_slice() { + [Lifecycle::Joined(j), Lifecycle::ConnectionLost(l), Lifecycle::Left(left)] => { + assert_eq!( + (j.via, j.from_zone.as_deref()), + (JoinVia::Handoff, Some("z0")) + ); + assert_eq!(l.cause, LostCause::Reload); + assert_eq!(left.reason, LeftReason::Handoff); + } + other => panic!("{other:?}"), + } +} diff --git a/distributed_game/tests/net_e2e/main.rs b/distributed_game/tests/net_e2e/main.rs index f4f1c13b6..68a376b48 100644 --- a/distributed_game/tests/net_e2e/main.rs +++ b/distributed_game/tests/net_e2e/main.rs @@ -7,13 +7,14 @@ mod arena; use std::time::Duration; use arena::{Arena, ArenaConfig, AVATAR}; -use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey}; +use distributed_game::net::{GameServer, NetSim, ServerConfig, TicketKey, ZoneFactory}; use distributed_game::protocol::{error, kick, Hello}; use distributed_game::{ ClientMessage, ClientMirror, FieldValue, MemoryFactSink, RetryPolicy, Role, ServerMessage, - TicketClaims, PROTOCOL_VERSION, + TicketClaims, TicketExtension, TicketExtensions, PROTOCOL_VERSION, }; use futures_util::{SinkExt, StreamExt}; +use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; @@ -30,6 +31,7 @@ fn ticket(principal: &str, zone: &str, nonce: &str, role: Role) -> String { nonce: nonce.into(), role, name: None, + extensions: Default::default(), }) } @@ -47,9 +49,10 @@ async fn start_keyed( config.host.tick_rate = 30; config.host.patch_rate = 20; config.net_sim = net_sim; - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }) + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ) .with_fact_sink(MemoryFactSink::new(), 64, RetryPolicy::default()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -228,6 +231,7 @@ async fn client_joins_sends_inputs_receives_deltas_and_reconnects() { nonce: "n4".into(), role: Role::Player, name: None, + extensions: Default::default(), }); expect_error(&url, "z1", forged, error::UNAUTHORIZED).await; @@ -290,6 +294,21 @@ async fn admits_tickets_minted_by_the_sveltekit_handler() { assert_eq!(claims.nonce, text("/valid/nonce")); assert_eq!(claims.role, Role::Player); assert_eq!(claims.name.as_deref(), Some("Ana")); + assert!(claims.extensions.is_empty()); + + // Extension claims from the handler's server-side `claims()` callback. + let extended = TicketKey::new(secret.as_str()) + .verify(&text("/extended/ticket"), 1_700_000_000) + .unwrap(); + assert_eq!(extended.principal, text("/extended/principal")); + assert_eq!( + extended.extensions.typed::().unwrap(), + MatchClaims { + match_id: "m-7".into(), + side: 2, + party: None, + } + ); let (url, server, handle) = start_keyed(&secret, None).await; @@ -401,9 +420,10 @@ async fn http_get(addr: std::net::SocketAddr, path: &str) -> String { async fn serve_with_merges_extra_routes_and_shuts_down_gracefully() { let mut config = ServerConfig::new(TicketKey::new(SECRET)); config.host.tick_rate = 30; - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let extra = axum::Router::new().route("/hello", axum::routing::get(|| async { "hi" })); @@ -530,9 +550,10 @@ fn bff_ticket(jti: &str, exp_in: u64, zone_route: Option<&str>) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn custom_verifier_admits_web_bff_tickets_with_replay_protection() { let config = ServerConfig::new(TicketKey::new(SECRET)).with_ticket_verifier(bff_verifier); - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }); + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("ws://{}", listener.local_addr().unwrap()); let serving = server.clone(); @@ -613,9 +634,10 @@ async fn host_config_hook_sets_per_zone_rates() { let mut config = ServerConfig::new(TicketKey::new(SECRET)); config.host.tick_rate = 30; config.host.content_version = "c1".into(); - let server = GameServer::::new(config, |zone_id| { - zone_id.starts_with('z').then(ArenaConfig::default) - }) + let server = GameServer::::new( + config, + ZoneFactory::sync(|a| a.zone_id.starts_with('z').then(ArenaConfig::default)), + ) .with_host_config(|zone_id, host| { if zone_id == "z-slow" { host.tick_rate = 5; @@ -623,8 +645,8 @@ async fn host_config_hook_sets_per_zone_rates() { host.content_version = "ignored".into(); } }); - assert!(server.activate("z-slow")); - assert!(server.activate("z-hot")); + assert!(server.activate("z-slow").await); + assert!(server.activate("z-hot").await); let slow = server .inspect("z-slow", |h| { ( @@ -639,3 +661,285 @@ async fn host_config_hook_sets_per_zone_rates() { assert_eq!(hot, Some(30)); server.shutdown(); } + +/// A game's declared extension claims. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TicketExtension)] +struct MatchClaims { + match_id: String, + #[serde(rename = "team")] + side: u8, + #[serde(default)] + party: Option, +} + +fn extended_ticket(nonce: &str, ext: serde_json::Value) -> String { + TicketKey::new(SECRET).sign(&TicketClaims { + principal: "ana".into(), + zone: "z-match".into(), + expires_at: 4_000_000_000, + nonce: nonce.into(), + role: Role::Player, + name: None, + extensions: TicketExtensions::from_json(ext).unwrap(), + }) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn extension_claims_reach_on_auth_and_player_info() { + assert_eq!(MatchClaims::CLAIMS, ["match_id", "team", "party"]); + let config = ServerConfig::new(TicketKey::new(SECRET)).with_ticket_extensions::(); + let server = + GameServer::::new(config, ZoneFactory::sync(|_| Some(ArenaConfig::default()))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}", listener.local_addr().unwrap()); + let serving = server.clone(); + let handle = tokio::spawn(async move { + serving + .serve(listener, std::future::pending()) + .await + .unwrap(); + }); + + // Undeclared names and mistyped values are refused at admission. + for (nonce, ext) in [ + ( + "x-unknown", + serde_json::json!({"match_id": "m-7", "team": 2, "gold": 50}), + ), + ( + "x-typed", + serde_json::json!({"match_id": "m-7", "team": "red"}), + ), + ("x-object", serde_json::json!(["m-7"])), + ] { + let ticket = if nonce == "x-object" { + TicketKey::new(SECRET).sign_json(&serde_json::json!({ + "sub": "ana", "zone": "z-match", "exp": 4_000_000_000u64, + "nonce": nonce, "ext": ext, + })) + } else { + extended_ticket(nonce, ext) + }; + expect_error(&url, "z-match", ticket, error::UNAUTHORIZED).await; + } + + let expected = MatchClaims { + match_id: "m-7".into(), + side: 2, + party: None, + }; + let mut ws = connect(&url, "z-match").await; + hello( + &mut ws, + extended_ticket("ok", serde_json::json!({"match_id": "m-7", "team": 2})), + String::new(), + ) + .await; + let Some(ServerMessage::Welcome(_)) = next(&mut ws, &mut ClientMirror::new()).await else { + panic!("expected Welcome") + }; + let (auth, players) = server + .inspect("z-match", |host| { + ( + host.zone().auth_extensions.clone(), + host.players().cloned().collect::>(), + ) + }) + .await + .expect("zone active"); + assert_eq!( + auth.last().unwrap().typed::().unwrap(), + expected + ); + assert_eq!( + players[0].extensions.typed::().unwrap(), + expected + ); + + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("server stops") + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn extension_claims_are_refused_without_a_declared_schema() { + let (url, server, handle) = start(None).await; + let ticket = TicketKey::new(SECRET).sign(&TicketClaims { + principal: "ana".into(), + zone: "z1".into(), + expires_at: 4_000_000_000, + nonce: "undeclared".into(), + role: Role::Player, + name: None, + extensions: TicketExtensions::from_json(serde_json::json!({"match_id": "m"})).unwrap(), + }); + expect_error(&url, "z1", ticket, error::UNAUTHORIZED).await; + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("server stops") + .unwrap(); +} + +async fn serve_arena(server: &GameServer) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}", listener.local_addr().unwrap()); + let serving = server.clone(); + let handle = tokio::spawn(async move { + serving + .serve(listener, std::future::pending()) + .await + .unwrap(); + }); + (url, handle) +} + +async fn stop(server: GameServer, handle: tokio::task::JoinHandle<()>) { + server.shutdown(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("server stops") + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_factory_reads_the_allocation_before_admitting_queued_joins() { + use distributed_game::world::{MemoryZoneAllocationStore, ZoneAllocation, ZoneAllocationStore}; + use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + use std::sync::Arc; + + let store = Arc::new(MemoryZoneAllocationStore::new()); + let roster = serde_json::json!({ "mode": "duel", "roster": ["ana", "bo"] }); + store + .put( + "z-match", + ZoneAllocation::new("game-services.match", 1, &roster).unwrap(), + ) + .await + .unwrap(); + let created = Arc::new(AtomicBool::new(false)); + let calls = Arc::new(AtomicU32::new(0)); + let (done, calls_in) = (created.clone(), calls.clone()); + let server = GameServer::::new( + ServerConfig::new(TicketKey::new(SECRET)), + ZoneFactory::new(move |activation| { + let (done, calls) = (done.clone(), calls_in.clone()); + async move { + calls.fetch_add(1, Ordering::SeqCst); + assert!(activation.activation > 0, "the fencing token"); + // Storage is slow: both joins arrive while this awaits. + tokio::time::sleep(Duration::from_millis(200)).await; + done.store(true, Ordering::SeqCst); + Ok(Some(ArenaConfig { + allocation: activation.allocation.map(|a| a.payload), + ..ArenaConfig::default() + })) + } + }), + ) + .with_allocation_store(store); + let (url, handle) = serve_arena(&server).await; + + let join = |principal: &'static str, nonce: &'static str| { + let url = url.clone(); + let created = created.clone(); + async move { + let mut ws = connect(&url, "z-match").await; + hello( + &mut ws, + ticket(principal, "z-match", nonce, Role::Player), + String::new(), + ) + .await; + let msg = next(&mut ws, &mut ClientMirror::new()).await; + assert!( + created.load(Ordering::SeqCst), + "admitted only after the factory finished" + ); + matches!(msg, Some(ServerMessage::Welcome(_))) + } + }; + let (ana, bo) = tokio::join!(join("ana", "q1"), join("bo", "q2")); + assert!(ana && bo, "both queued joins admitted"); + assert_eq!(calls.load(Ordering::SeqCst), 1, "one activation"); + let allocation = server + .inspect("z-match", |host| host.zone().cfg.allocation.clone()) + .await + .unwrap(); + assert_eq!(allocation, Some(roster), "the allocation reached on_create"); + stop(server, handle).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn factory_failures_answer_joins_with_typed_errors() { + use distributed_game::world::{ + AllocationError, BoxFuture, ZoneAllocation, ZoneAllocationStore, ZoneCreateError, + }; + use std::sync::Arc; + + struct DownStore; + impl ZoneAllocationStore for DownStore { + fn load<'a>( + &'a self, + _: &'a str, + ) -> BoxFuture<'a, Result, AllocationError>> { + Box::pin(async { Err(AllocationError("connection refused".into())) }) + } + fn put<'a>( + &'a self, + _: &'a str, + _: ZoneAllocation, + ) -> BoxFuture<'a, Result<(), AllocationError>> { + Box::pin(async { Ok(()) }) + } + fn remove<'a>(&'a self, _: &'a str) -> BoxFuture<'a, Result<(), AllocationError>> { + Box::pin(async { Ok(()) }) + } + } + + let mut config = ServerConfig::new(TicketKey::new(SECRET)); + config.activation_timeout = Duration::from_millis(300); + let server = GameServer::::new( + config, + ZoneFactory::new(|activation| async move { + match activation.zone_id.as_str() { + "z-broken" => Err(ZoneCreateError::new("roster store unavailable")), + "z-hang" => std::future::pending().await, + "z-ok" => Ok(Some(ArenaConfig::default())), + _ => Ok(None), + } + }), + ); + let (url, handle) = serve_arena(&server).await; + let started = std::time::Instant::now(); + for (zone, code, nonce) in [ + ("z-broken", error::ZONE_UNAVAILABLE, "f1"), + ("z-hang", error::ZONE_UNAVAILABLE, "f2"), + ("z-unknown", error::NOT_FOUND, "f3"), + ] { + expect_error(&url, zone, ticket("ana", zone, nonce, Role::Player), code).await; + assert!(!server.is_active(zone), "{zone} unregistered"); + } + assert!(started.elapsed() < Duration::from_secs(3), "no hang"); + assert!(!server.activate("z-broken").await); + assert!(server.activate("z-ok").await); + stop(server, handle).await; + + // An allocation store failure refuses too, without calling the factory. + let server = GameServer::::new( + ServerConfig::new(TicketKey::new(SECRET)), + ZoneFactory::new(|_| async { panic!("factory must not run without its allocation") }), + ) + .with_allocation_store(Arc::new(DownStore)); + let (url, handle) = serve_arena(&server).await; + expect_error( + &url, + "z-match", + ticket("ana", "z-match", "f4", Role::Player), + error::ZONE_UNAVAILABLE, + ) + .await; + stop(server, handle).await; +} diff --git a/distributed_game/tests/netcode/main.rs b/distributed_game/tests/netcode/main.rs index 5d3711f68..f5683d491 100644 --- a/distributed_game/tests/netcode/main.rs +++ b/distributed_game/tests/netcode/main.rs @@ -24,6 +24,7 @@ fn claims(principal: &str, zone: &str) -> TicketClaims { nonce: format!("n-{principal}"), role: Role::Player, name: None, + extensions: Default::default(), } } @@ -52,6 +53,7 @@ impl Zone for Range { type Config = (); type Input = (f32, u16); type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { EntitySchema::new().kind(KindDef::new(1, "target").field("x", FieldType::F32)) diff --git a/distributed_game/tests/support/celld.rs b/distributed_game/tests/support/celld.rs new file mode 100644 index 000000000..1efd2fffa --- /dev/null +++ b/distributed_game/tests/support/celld.rs @@ -0,0 +1,229 @@ +//! An isolated `celld dev` serving `tests/celld/fact-worker`, owned by one +//! test: its own temp project, port and process tree (killed on drop; only +//! processes this harness started). Skips (returns `None`) when the worker +//! is not built or celld cannot start, unless `CELLD_FACTS_REQUIRED=1`. + +#![allow(dead_code)] + +use std::io::Write as _; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +pub const SECRET: &str = "test-only-internal-secret-change-me-2026"; + +fn worker_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../tests/celld/fact-worker") +} + +pub struct Celld { + project: PathBuf, + port: u16, + child: Option, + generation: u32, +} + +impl Celld { + pub fn start_new() -> Option { + let build = worker_dir().join("build"); + if !build.join("worker/shim.mjs").exists() { + assert!( + std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), + "CELLD_FACTS_REQUIRED is set but the fact worker is not built" + ); + eprintln!("skipping celld fact routing: build tests/celld/fact-worker first"); + return None; + } + let project = std::env::temp_dir().join(format!( + "distributed-celld-facts-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + copy_dir(&build, &project.join("build")); + std::fs::copy( + worker_dir().join("wrangler.jsonc"), + project.join("wrangler.jsonc"), + ) + .unwrap(); + let port = TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let mut celld = Self { + project, + port, + child: None, + generation: 0, + }; + if let Err(reason) = celld.start() { + assert!( + std::env::var_os("CELLD_FACTS_REQUIRED").is_none(), + "celld dev is required but did not start: {reason}" + ); + eprintln!("skipping celld fact routing: {reason}"); + return None; + } + Some(celld) + } + + pub fn base(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Start `celld dev`; `Err` when it is unavailable or exits early (no + /// celld on PATH, or no esbuild for its bundler). + pub fn start(&mut self) -> Result<(), String> { + self.generation += 1; + let binary = std::env::var("CELLD_BIN").unwrap_or_else(|_| "celld".into()); + let help = Command::new(&binary) + .args(["dev", "--help"]) + .output() + .map_err(|error| format!("cannot run `{binary}`: {error}"))?; + let mut args = vec![ + "dev".to_string(), + self.project.display().to_string(), + "--host".into(), + "127.0.0.1".into(), + "--port".into(), + self.port.to_string(), + "--logs".into(), + ]; + if String::from_utf8_lossy(&help.stdout).contains("--no-watch") { + args.push("--no-watch".into()); + } + let log_path = self.project.join(format!("celld-{}.log", self.generation)); + let log = std::fs::File::create(&log_path).unwrap(); + let child = Command::new(&binary) + .args(&args) + .env("RUST_LOG", "warn") + .stdin(Stdio::null()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .map_err(|error| format!("cannot start `{binary} dev`: {error}"))?; + self.child = Some(child); + let deadline = Instant::now() + Duration::from_secs(90); + while Instant::now() < deadline { + if std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() + && blocking_get(&format!("{}/health", self.base())).is_some_and(|s| s == 200) + { + return Ok(()); + } + let exited = self + .child + .as_mut() + .and_then(|child| child.try_wait().ok().flatten()); + if let Some(status) = exited { + self.child = None; + let log = std::fs::read_to_string(&log_path).unwrap_or_default(); + return Err(format!( + "celld dev exited ({status}): {}", + log.lines().rev().take(3).collect::>().join(" | ") + )); + } + std::thread::sleep(Duration::from_millis(200)); + } + Err(format!( + "celld dev did not become ready; logs in {}", + self.project.display() + )) + } + + /// Kill the launcher this test started and every descendant. + pub fn crash(&mut self) { + let Some(mut child) = self.child.take() else { + return; + }; + let root = child.id(); + let _ = Command::new("kill") + .args(["-STOP", &root.to_string()]) + .status(); + let table = Command::new("ps") + .args(["-axo", "pid=,ppid="]) + .output() + .unwrap(); + let pairs = String::from_utf8_lossy(&table.stdout) + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + Some(( + parts.next()?.parse::().ok()?, + parts.next()?.parse::().ok()?, + )) + }) + .collect::>(); + let mut doomed = vec![root]; + let mut index = 0; + while index < doomed.len() { + let parent = doomed[index]; + doomed.extend( + pairs + .iter() + .filter(|(_, ppid)| *ppid == parent) + .map(|(pid, _)| *pid), + ); + index += 1; + } + for pid in doomed.iter().rev() { + let _ = Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status(); + } + let _ = child.wait(); + // The port is released once the processes are gone. + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline + && std::net::TcpStream::connect(("127.0.0.1", self.port)).is_ok() + { + std::thread::sleep(Duration::from_millis(100)); + } + } +} + +impl Drop for Celld { + fn drop(&mut self) { + self.crash(); + if !std::thread::panicking() { + let _ = std::fs::remove_dir_all(&self.project); + } else { + let _ = writeln!( + std::io::stderr(), + "celld artifacts kept in {}", + self.project.display() + ); + } + } +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), target).unwrap(); + } + } +} + +fn blocking_get(url: &str) -> Option { + let url = url.strip_prefix("http://")?; + let (host, path) = url.split_once('/')?; + let mut stream = std::net::TcpStream::connect(host).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok()?; + write!( + stream, + "GET /{path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n" + ) + .ok()?; + let mut response = String::new(); + std::io::Read::read_to_string(&mut stream, &mut response).ok()?; + response.split_whitespace().nth(1)?.parse().ok() +} diff --git a/distributed_game/tests/world/main.rs b/distributed_game/tests/world/main.rs index 6283d11f4..165419928 100644 --- a/distributed_game/tests/world/main.rs +++ b/distributed_game/tests/world/main.rs @@ -1,7 +1,12 @@ //! Two GameServers in one process on different ports: cross-host handoff //! with a real WebSocket client, tier-up events across hosts, hibernation, -//! fenced ownership and the world content barrier. +//! fenced ownership and the world content barrier. Hibernation runs on +//! celld (`CellZoneOwnership` + `CellZoneStateStore` against +//! `tests/celld/fact-worker`; skipped unless the worker is built and celld +//! starts, required with `CELLD_FACTS_REQUIRED=1`). +#[path = "../support/celld.rs"] +mod celld_support; mod realm; use std::sync::{Arc, Mutex}; @@ -10,11 +15,12 @@ use std::time::Duration; use distributed::lease::InMemoryLeaseStore; use distributed_game::net::{ AbortNotes, GameServer, RemoteHost, RolloutError, ServerConfig, TicketKey, WorldCoordinator, + ZoneFactory, }; use distributed_game::protocol::{error, kick, Hello, Notice}; use distributed_game::world::{ - FencedLeaseOwnership, HostAddr, LocalOwnership, SqliteZoneStateStore, StaticDirectory, - ZoneDirectory, ZoneOwnership, ZoneStateStore, + FencedLeaseOwnership, HostAddr, LocalOwnership, StaticDirectory, ZoneDirectory, ZoneOwnership, + ZoneStateStore, }; use distributed_game::*; use futures_util::{SinkExt, StreamExt}; @@ -36,6 +42,7 @@ fn ticket(principal: &str, zone: &str, nonce: &str) -> String { nonce: nonce.into(), role: Role::Player, name: None, + extensions: Default::default(), }) } @@ -92,7 +99,7 @@ async fn two_hosts( for (host, listener) in [(a, la), (b, lb)] { let sink = MemoryFactSink::new(); let f = factory.clone(); - let server = GameServer::::new(config(), move |zone| f(zone)) + let server = GameServer::::new(config(), ZoneFactory::sync(move |a| f(&a.zone_id))) .with_fact_sink(sink.clone(), 64, RetryPolicy::default()) .with_host_id(host.id.clone()) .with_directory(dir.clone()) @@ -243,6 +250,7 @@ fn realm_factory( reject_all: rejecting.contains(&zone), gate: zone.ends_with(".inn"), freeze_ready: !zone.ends_with(".stuck"), + move_facts: false, }) }) } @@ -326,6 +334,7 @@ async fn cross_host_handoff_follows_redirect_and_ticket_is_single_use() { nonce: "x".into(), role: Role::Player, name: None, + extensions: Default::default(), }); assert!(join_url(&b.addr.zone_url("realm.west"), &forged, "") .await @@ -499,16 +508,11 @@ async fn boss_defeat_tiers_up_to_parent_and_grandparent_across_hosts() { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn dormant_zone_hibernates_to_sqlite_and_restores() { - let path = std::env::temp_dir().join(format!( - "dg-world-hibernate-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let store = Arc::new(SqliteZoneStateStore::open(&path).unwrap()); +async fn dormant_zone_hibernates_to_celld_and_restores() { + let Some(celld) = celld_support::Celld::start_new() else { + return; + }; + let (ownership, store) = on_celld(&celld, "host-a"); let mut config = base_config(); config.host.idle_tick_rate = None; config.host.dormant_after = Duration::from_millis(50); @@ -516,8 +520,9 @@ async fn dormant_zone_hibernates_to_sqlite_and_restores() { let sink = MemoryFactSink::new(); let (listener, addr) = listen().await; let factory = realm_factory(&[], &[]); - let server = GameServer::::new(config, move |z| factory(z)) + let server = GameServer::::new(config, ZoneFactory::sync(move |a| factory(&a.zone_id))) .with_fact_sink(sink.clone(), 64, RetryPolicy::default()) + .with_ownership(ownership) .with_state_store(store.clone()); serve(&server, listener); let url = format!("ws://{addr}/zones/realm.vault"); @@ -541,7 +546,11 @@ async fn dormant_zone_hibernates_to_sqlite_and_restores() { async move { !server.is_active("realm.vault") && hibernation_outcomes(&server) >= 1 } }) .await; - let saved = store.load("realm.vault").unwrap().expect("snapshot stored"); + let saved = store + .load("realm.vault") + .await + .unwrap() + .expect("snapshot stored"); assert_eq!(saved.state, 3u32.to_le_bytes()); // Facts emitted while disposing were still committed. eventually("dispose fact", || { @@ -574,28 +583,51 @@ async fn dormant_zone_hibernates_to_sqlite_and_restores() { } }; assert_eq!(counter, 3); - drop(store); - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } +} + +/// Zone ownership and hibernation snapshots in celld cells. +fn on_celld( + celld: &celld_support::Celld, + host: &str, +) -> ( + Arc, + Arc, +) { + let secret = || distributed::cell_host::InternalHttpSecret::new(celld_support::SECRET).unwrap(); + ( + Arc::new( + distributed_game::celld::CellZoneOwnership::new( + celld.base(), + secret(), + "world-test", + host, + ) + .unwrap(), + ), + Arc::new( + distributed_game::celld::CellZoneStateStore::new( + celld.base(), + secret(), + "world-test", + host, + ) + .unwrap(), + ), + ) } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { - let path = std::env::temp_dir().join(format!( - "dg-world-restart-{}-{}.sqlite3", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); + let Some(celld) = celld_support::Celld::start_new() else { + return; + }; let url_of = |addr: &str| format!("ws://{addr}/zones/realm.keep"); let mut epochs = Vec::new(); let mut count = 0u32; for run in 1..=2 { - // A fresh process: new store handle, new (default) LocalOwnership. - let store = Arc::new(SqliteZoneStateStore::open(&path).unwrap()); + // A fresh process: new store and ownership handles (the fencing + // tokens live in the celld lease cell, so they keep growing). + let (ownership, store) = on_celld(&celld, &format!("host-{run}")); let mut config = base_config(); config.host.idle_tick_rate = None; config.host.dormant_after = Duration::from_millis(50); @@ -603,7 +635,9 @@ async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { let (listener, addr) = listen().await; let factory = realm_factory(&[], &[]); let server = - GameServer::::new(config, move |z| factory(z)).with_state_store(store.clone()); + GameServer::::new(config, ZoneFactory::sync(move |a| factory(&a.zone_id))) + .with_ownership(ownership) + .with_state_store(store.clone()); serve(&server, listener); // Two activations per run, so the first run's last token exceeds // the restarted run's first token unless tokens survive restarts. @@ -641,7 +675,7 @@ async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { cycle, "run {run}" ); - let saved = store.load("realm.keep").unwrap().expect("snapshot"); + let saved = store.load("realm.keep").await.unwrap().expect("snapshot"); assert_eq!(saved.state, count.to_le_bytes(), "run {run} saved"); } assert!(server @@ -654,9 +688,6 @@ async fn hibernation_saves_succeed_after_a_host_restart_on_the_same_store() { epochs.windows(2).all(|w| w[1] > w[0]), "epochs grow: {epochs:?}" ); - for suffix in ["", "-wal", "-shm"] { - let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); - } } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -676,7 +707,7 @@ async fn fenced_lease_keeps_zone_on_one_host_and_loss_stops_it() { for id in ["A", "B"] { let (listener, addr) = listen().await; let f = realm_factory(&[], &[]); - let server = GameServer::::new(config(), move |z| f(z)) + let server = GameServer::::new(config(), ZoneFactory::sync(move |a| f(&a.zone_id))) .with_host_id(id) .with_ownership(ownership.clone()); serve(&server, listener); @@ -1054,3 +1085,69 @@ fn hibernation_outcomes(server: &GameServer) -> u64 { let m = server.metrics(); m.hibernation_saves("saved") + m.hibernation_saves("stale") + m.hibernation_saves("error") } + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize, TicketExtension)] +struct PartyClaims { + party: String, + team: u8, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn transfer_tickets_carry_extension_claims_across_hosts() { + let (a, b, _dir) = two_hosts( + || base_config().with_ticket_extensions::(), + &[("realm.east", "A"), ("realm.west", "B")], + realm_factory(&[("realm.east", "realm.west")], &[]), + Arc::new(LocalOwnership::new()), + ) + .await; + let party = PartyClaims { + party: "p-9".into(), + team: 1, + }; + let t = TicketKey::new(SECRET).sign( + &TicketClaims::new("ana", "realm.east", 4_000_000_000, "ext-1", Role::Player) + .with_extensions(TicketExtensions::from_typed(&party).unwrap()), + ); + let mut ana = join_url(&a.addr.zone_url("realm.east"), &t, "") + .await + .unwrap(); + input(&mut ana, 1, CMD_HANDOFF).await; + let transfer = loop { + match control(&mut ana).await { + Some(ServerMessage::Handoff { + transfer_ticket, .. + }) => break transfer_ticket, + Some(_) => continue, + None => panic!("closed before Handoff"), + } + }; + // The server-minted transfer ticket signs the claims... + let distributed_game::net::VerifiedTicket::Transfer(verified) = TicketKey::new(SECRET) + .verify_ticket(&transfer, 1_700_000_000) + .unwrap() + else { + panic!("transfer ticket") + }; + assert_eq!( + verified.claims.extensions.typed::().unwrap(), + party + ); + // ...and the target zone on the other host sees them. + let _west = join_url(&a.addr.zone_url("realm.west"), &transfer, "") + .await + .unwrap(); + let players = b + .server + .inspect("realm.west", |h| h.players().cloned().collect::>()) + .await + .unwrap(); + assert_eq!(players[0].principal, "ana"); + assert_eq!(players[0].extensions.typed::().unwrap(), party); +} + +#[tokio::test] +async fn fenced_lease_ownership_passes_the_conformance_suite() { + let ownership = FencedLeaseOwnership::new(InMemoryLeaseStore::new()); + distributed_game::world::ownership_conformance(&ownership, "conformance").await; +} diff --git a/distributed_game/tests/world/realm.rs b/distributed_game/tests/world/realm.rs index 0ec711f33..36188a219 100644 --- a/distributed_game/tests/world/realm.rs +++ b/distributed_game/tests/world/realm.rs @@ -20,6 +20,8 @@ pub struct RealmConfig { pub freeze_ready: bool, pub neighbour: Option, pub reject_all: bool, + /// Emit `realm.left` / `realm.arrived` facts for handoffs. + pub move_facts: bool, } impl RealmConfig { @@ -53,6 +55,7 @@ impl Zone for Realm { type Config = RealmConfig; type Input = u8; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { EntitySchema::new().kind(KindDef::new(1, "realm").field("counter", FieldType::U32)) @@ -94,6 +97,15 @@ impl Zone for Realm { .player(player) .map(|p| p.principal.clone()) .unwrap_or_default(); + if self.cfg.move_facts { + let key = format!("arrived:{}:{principal}", ctx.zone_id()); + let fact = Fact::new( + key, + "realm.arrived", + serde_json::json!({ "from": from_zone }), + ); + let _ = ctx.emit_fact_for(player, fact); + } let mut payload = carried.clone(); payload.extend_from_slice(b"|"); payload.extend_from_slice(from_zone.as_bytes()); @@ -105,7 +117,11 @@ impl Zone for Realm { }); } - fn on_leave(&mut self, _ctx: &mut Ctx<'_, Self>, player: PlayerId, reason: LeaveReason) { + fn on_leave(&mut self, ctx: &mut Ctx<'_, Self>, player: PlayerId, reason: LeaveReason) { + if self.cfg.move_facts && reason == LeaveReason::Handoff { + let key = format!("left:{}:{}", ctx.zone_id(), player.0); + let _ = ctx.emit_fact(Fact::new(key, "realm.left", serde_json::json!({}))); + } self.left.push((player, reason)); } diff --git a/distributed_game/tests/world_host/main.rs b/distributed_game/tests/world_host/main.rs index 1699da6bc..d85e3b950 100644 --- a/distributed_game/tests/world_host/main.rs +++ b/distributed_game/tests/world_host/main.rs @@ -20,6 +20,7 @@ fn claims(principal: &str, zone: &str) -> TicketClaims { nonce: format!("n-{principal}"), role: Role::Player, name: None, + extensions: Default::default(), } } @@ -58,6 +59,7 @@ fn transfer(principal: &str, from: &str, to: &str) -> Transfer { issuer: "h1".into(), generation: 1, carried: b"gold=7".to_vec(), + departed_at_ms: 0, } } @@ -409,6 +411,7 @@ fn world_entry_points_record_and_replay_deterministically() { clock.advance(Duration::from_millis(40)); rec.advance(); rec.set_content_version("v2"); + rec.set_config_version("combat@2"); rec.resume(notice); rec.reload_stale_players(); let out = rec.drain_outbound(); @@ -421,6 +424,7 @@ fn world_entry_points_record_and_replay_deterministically() { clock.advance(Duration::from_millis(40)); rec.advance(); assert_eq!(rec.host().zone().counter, 6); + assert_eq!(rec.host().config().config_version, "combat@2"); assert_eq!(rec.host().zone().child_events.len(), 1); assert_eq!(rec.host().player_count(), 1, "bo handed off"); @@ -428,3 +432,125 @@ fn world_entry_points_record_and_replay_deterministically() { let summary = replay::(&recording, cfg).unwrap(); assert!(summary.ticks >= 2, "{summary:?}"); } + +#[test] +fn handoff_target_stamps_after_source_departure_despite_clock_skew() { + // One virtual time, two hosts whose wall clocks disagree: the target's + // reads 9 000 s *behind* the source's. + let time = ManualClock::new(); + let source_clock = time.clone().with_unix_origin_ms(10_000_000); + let target_clock = time.clone().with_unix_origin_ms(1_000_000); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let (source_queue, source_feed) = fact_channel(16, RetryPolicy::default()); + let mut source = ZoneHost::::new( + "realm.east", + HostConfig { + boot_epoch: 3, + config_version: "combat@12".into(), + ..HostConfig::default() + }, + Arc::new(source_clock), + RealmConfig { + neighbour: Some("realm.west".into()), + move_facts: true, + ..RealmConfig::new() + }, + ) + .with_facts(source_queue) + .with_transfer_issuer(move |r: &HandoffRequest| { + seen.lock().unwrap().push(r.clone()); + Some("signed".into()) + }); + let (target_queue, target_feed) = fact_channel(16, RetryPolicy::default()); + let mut target = ZoneHost::::new( + "realm.west", + HostConfig { + boot_epoch: 8, + ..HostConfig::default() + }, + Arc::new(target_clock), + RealmConfig { + move_facts: true, + ..RealmConfig::new() + }, + ) + .with_facts(target_queue); + + let ana = source.join(&claims("ana", "realm.east"), 0, "").unwrap(); + source.push_input(ana.client, 1, 0, &[CMD_HANDOFF]); + time.advance(Duration::from_millis(40)); + source.advance(); + let request = requests.lock().unwrap().pop().expect("handoff issued"); + // The ManualClock decides the stamp time: origin + virtual time. + assert_eq!(request.departed_at_ms, 10_000_040); + + // The source's leave runs later on its clock, still stamped with the + // departure time the ticket carries. + time.advance(Duration::from_millis(5)); + let transfer = Transfer { + claims: claims("ana", "realm.west"), + source_zone: request.source_zone.clone(), + issuer: "h1".into(), + generation: 3, + carried: request.carried.clone(), + departed_at_ms: request.departed_at_ms, + }; + target.join_transfer(&transfer, 0, "").unwrap(); + drop(source); + drop(target); + + // The target commits first. + let sink = MemoryFactSink::new(); + futures_executor(target_feed.run(sink.clone(), |_| async {})); + futures_executor(source_feed.run(sink.clone(), |_| async {})); + let committed = sink.committed(); + let commands: Vec<_> = committed.iter().map(Fact::command).collect(); + assert_eq!(commands, ["realm.arrived", "realm.left"]); + let (arrived, left) = (committed[0].stamp(), committed[1].stamp()); + + assert_eq!(left.emitted_at_ms, 10_000_040); + assert_eq!( + (left.activation, left.config_version.as_str()), + (3, "combat@12") + ); + // The target's own clock says 1 000 045; the stamp starts after the + // departure instead. + assert_eq!(arrived.emitted_at_ms, 10_000_041); + assert_eq!(arrived.activation, 8); + assert!(arrived.emitted_after(left)); + let mut ordered = committed.clone(); + ordered.sort_by(|a, b| a.stamp().emission_order(b.stamp())); + let ordered: Vec<_> = ordered.iter().map(Fact::command).collect(); + assert_eq!(ordered, ["realm.left", "realm.arrived"]); +} + +#[test] +fn stamps_follow_the_manual_clock_and_config_version() { + let clock = ManualClock::new().with_unix_origin_ms(500); + let h = host( + "z", + HostConfig { + boot_epoch: 42, + ..HostConfig::default() + }, + &clock, + RealmConfig::new(), + ); + clock.advance(Duration::from_millis(250)); + let (queue, feed) = fact_channel(16, RetryPolicy::default()); + let mut h = h.with_facts(queue); + let joined = h.join(&claims("bo", "z"), 0, "").unwrap(); + h.push_input(joined.client, 1, 0, &[CMD_COUNT]); + h.set_config_version("combat@13"); + clock.advance(Duration::from_millis(40)); + h.advance(); + drop(h); + let sink = MemoryFactSink::new(); + futures_executor(feed.run(sink.clone(), |_| async {})); + let stamp = sink.committed()[0].stamp().clone(); + assert_eq!(stamp.emitted_at_ms, 790); + assert_eq!(stamp.activation, 42); + assert_eq!(stamp.config_version, "combat@13"); + assert_eq!(stamp.zone_id, "z"); +} diff --git a/distributed_game/tests/zone_lease/main.rs b/distributed_game/tests/zone_lease/main.rs new file mode 100644 index 000000000..3c2ceaf50 --- /dev/null +++ b/distributed_game/tests/zone_lease/main.rs @@ -0,0 +1,281 @@ +//! `ZoneLease` cells on the in-process reference host, driven by a +//! `ManualClock`: ownership conformance, no outbox rows for renewals, +//! exactly one `zone.lease_expired`, and renewals racing the alarm. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use distributed::cell_host::{ + AggregateCell, CellCommandIdentity, CellDispatchError, CellStreamStore, +}; +use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; +use distributed::schedule::schedule_session; +use distributed::{Clock, ManualClock, StreamIdentity}; +use distributed_game::cell::command_id_for; +use distributed_game::world::zone_lease::{ + mount_zone_lease, ownership_error, ClaimPayload, ZoneLease, ACQUIRE, EXPIRE, RELEASE, RENEW, + ZONE_HOST_ROLE, +}; +use distributed_game::world::{ownership_conformance, OwnershipError, ZoneClaim, ZoneOwnership}; +use distributed_game::BoxFuture; +use serde_json::{json, Value}; + +const T0: u64 = 1_700_000_000_000; + +fn clock() -> ManualClock { + ManualClock::new(SystemTime::UNIX_EPOCH + Duration::from_millis(T0)) +} + +fn lease_cell(clock: &ManualClock, zone: &str) -> AggregateCell { + let store = CellStreamStore::for_identity(StreamIdentity::new("ZoneLease", zone).unwrap()) + .with_clock(clock.clone()); + mount_zone_lease(AggregateCell::from_store(store).unwrap()) +} + +fn host_session() -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, "zone-server-a"); + session.set(ROLE_KEY, ZONE_HOST_ROLE); + session +} + +/// `ZoneOwnership` over one in-process cell, as `CellZoneOwnership` is over +/// celld (same commands, same error mapping). +struct InProcess { + cell: AggregateCell, + calls: AtomicU64, +} + +impl InProcess { + async fn call(&self, command: &str, input: Value) -> Result { + let n = self.calls.fetch_add(1, Ordering::SeqCst); + let identity = CellCommandIdentity::new( + "zone-host", + "zone-server-a", + command_id_for("t", &n.to_string()), + ) + .unwrap(); + match self + .cell + .dispatch_idempotent(command, &identity, input, host_session()) + .await + { + Ok(result) => Ok(serde_json::from_value(result.payload().clone()).unwrap()), + Err(CellDispatchError::Rejected { message, .. }) => Err(ownership_error(&message)), + Err(error) => Err(OwnershipError::Backend(format!("{error:?}"))), + } + } +} + +fn claim(payload: ClaimPayload) -> ZoneClaim { + ZoneClaim { + zone_id: payload.zone_id, + holder: payload.holder, + token: payload.token, + expires_at: SystemTime::UNIX_EPOCH + Duration::from_millis(payload.expires_at_ms), + } +} + +impl ZoneOwnership for InProcess { + fn acquire<'a>( + &'a self, + zone_id: &'a str, + holder: &'a str, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + ACQUIRE, + json!({ "zone_id": zone_id, "holder": holder, "ttl_ms": ttl.as_millis() as u64 }), + ) + .await + .map(claim) + }) + } + + fn renew<'a>( + &'a self, + current: &'a ZoneClaim, + ttl: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.call( + RENEW, + json!({ "zone_id": current.zone_id, "holder": current.holder, + "token": current.token, "ttl_ms": ttl.as_millis() as u64 }), + ) + .await + .map(claim) + }) + } + + fn release<'a>(&'a self, current: &'a ZoneClaim) -> BoxFuture<'a, ()> { + Box::pin(async move { + let _ = self + .call( + RELEASE, + json!({ "zone_id": current.zone_id, "holder": current.holder, + "token": current.token }), + ) + .await; + }) + } +} + +fn in_process(clock: &ManualClock, zone: &str) -> Arc { + Arc::new(InProcess { + cell: lease_cell(clock, zone), + calls: AtomicU64::new(0), + }) +} + +/// Outward event types in the cell's outbox, sorted (the reference host's +/// export is unordered; delivery order comes from `created_at`). +fn outward(cell: &AggregateCell) -> Vec { + let mut types = cell + .durable_outbox() + .unwrap() + .into_iter() + .map(|message| message.event_type) + .collect::>(); + types.sort(); + types +} + +#[tokio::test] +async fn the_lease_cell_passes_the_ownership_conformance_suite() { + let clock = clock(); + let own = in_process(&clock, "conformance"); + ownership_conformance(own.as_ref(), "conformance").await; + assert_eq!( + outward(&own.cell), + [ + "zone.activated", + "zone.activated", + "zone.activated", + "zone.released", + "zone.released", + "zone.released" + ] + ); +} + +#[tokio::test] +async fn renewals_produce_no_outbox_rows_and_move_the_expiry_alarm() { + let clock = clock(); + let own = in_process(&clock, "z1"); + let ttl = Duration::from_secs(10); + let mut current = own.acquire("z1", "a#1", ttl).await.unwrap(); + assert_eq!(outward(&own.cell), ["zone.activated"]); + assert_eq!( + own.cell.next_schedule_due().unwrap(), + Some(SystemTime::UNIX_EPOCH + Duration::from_millis(T0 + 10_000)) + ); + for step in 1..=20u64 { + clock.advance(Duration::from_secs(3)); + current = own.renew(¤t, ttl).await.unwrap(); + assert_eq!( + own.cell.next_schedule_due().unwrap(), + Some(clock.now() + ttl), + "step {step}" + ); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); + } + assert_eq!( + outward(&own.cell), + ["zone.activated"], + "no heartbeat events" + ); + assert_eq!( + own.cell.load().await.unwrap().unwrap().holder(), + Some("a#1") + ); +} + +#[tokio::test] +async fn a_holder_that_stops_renewing_expires_exactly_once() { + let clock = clock(); + let own = in_process(&clock, "z2"); + let ttl = Duration::from_secs(10); + let held = own.acquire("z2", "a#1", ttl).await.unwrap(); + clock.advance(Duration::from_millis(9_999)); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); + clock.advance(Duration::from_millis(1)); + let report = own.cell.fire_due_schedules().await.unwrap(); + assert_eq!(report.fired.len(), 1); + assert_eq!(report.fired[0].payload["expired"], true); + assert_eq!(outward(&own.cell), ["zone.activated", "zone.lease_expired"]); + clock.advance(Duration::from_secs(60)); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); + assert_eq!(outward(&own.cell).len(), 2, "exactly one expiry"); + assert_eq!(own.renew(&held, ttl).await, Err(OwnershipError::Lost)); + let next = own.acquire("z2", "b#1", ttl).await.unwrap(); + assert!(next.token > held.token); +} + +#[tokio::test] +async fn a_renewal_racing_the_alarm_never_produces_a_false_expiry() { + let clock = clock(); + let own = in_process(&clock, "z3"); + let ttl = Duration::from_secs(10); + let held = own.acquire("z3", "a#1", ttl).await.unwrap(); + + // The alarm comes due, but the renewal lands before the wake. + clock.advance(ttl); + let held = own.renew(&held, ttl).await.unwrap(); + assert!(own.cell.fire_due_schedules().await.unwrap().is_empty()); + + // The wake read the due alarm, then the renewal interleaved, then the + // stale expiry command runs: the handler re-reads the lease and keeps it. + clock.advance(ttl); + let stale = own.cell.armed_schedules().unwrap().remove(0); + let _held = own.renew(&held, ttl).await.unwrap(); + let result = own + .cell + .dispatch_idempotent( + EXPIRE, + &stale.command_identity().unwrap(), + stale.input.clone(), + schedule_session(&stale), + ) + .await + .unwrap(); + assert_eq!(result.payload()["expired"], false); + assert_eq!(outward(&own.cell), ["zone.activated"]); + assert_eq!( + own.cell.load().await.unwrap().unwrap().holder(), + Some("a#1") + ); + + // Callers cannot run the expiry command themselves. + let forged = own + .cell + .dispatch_idempotent( + EXPIRE, + &CellCommandIdentity::new("zone-host", "zone-server-a", command_id_for("t", "forged")) + .unwrap(), + json!({ "zone_id": "z3" }), + host_session(), + ) + .await; + assert!(forged.is_err()); +} + +#[tokio::test] +async fn only_zone_hosts_may_acquire() { + let clock = clock(); + let cell = lease_cell(&clock, "z4"); + let mut player = Session::new(); + player.set(USER_ID_KEY, "player-1"); + let refused = cell + .dispatch_idempotent( + ACQUIRE, + &CellCommandIdentity::new("zone-host", "player-1", command_id_for("t", "p")).unwrap(), + json!({ "zone_id": "z4", "holder": "p#1", "ttl_ms": 1000 }), + player, + ) + .await; + assert!(refused.is_err()); + assert!(cell.load().await.unwrap().is_none()); +} diff --git a/distributed_game/tests/zone_runtime/arena.rs b/distributed_game/tests/zone_runtime/arena.rs index d3805d52e..dbce5d8fc 100644 --- a/distributed_game/tests/zone_runtime/arena.rs +++ b/distributed_game/tests/zone_runtime/arena.rs @@ -21,6 +21,8 @@ pub struct ArenaConfig { pub reject_principal: Option, pub background: bool, pub spawn_every: Option, + /// Allocation payload handed over by the zone factory. + pub allocation: Option, } #[derive(Debug, Clone)] @@ -47,6 +49,10 @@ pub struct Arena { pub ticks: u64, pub timer_fires: u64, pub tiers: Vec<(Tier, Tier)>, + /// Extension claims seen by `on_auth`. + pub auth_extensions: Vec, + /// `lost:{player}` / `back:{player}` from the connection hooks. + pub connection_events: Vec, } impl Arena { @@ -77,6 +83,7 @@ impl Zone for Arena { type Config = ArenaConfig; type Input = (i8, i8); type Timer = ArenaTimer; + type Message = distributed_game::NoMessages; fn schema() -> EntitySchema { let pos = FieldType::QuantF32 { @@ -114,10 +121,13 @@ impl Zone for Arena { ticks: 0, timer_fires: 0, tiers: Vec::new(), + auth_extensions: Vec::new(), + connection_events: Vec::new(), } } fn on_auth(&mut self, _ctx: &mut Ctx<'_, Self>, req: &JoinRequest<'_>) -> Admission { + self.auth_extensions.push(req.claims.extensions.clone()); if self.cfg.reject_principal.as_deref() == Some(req.claims.principal.as_str()) { return Admission::Reject("banned".into()); } @@ -146,6 +156,14 @@ impl Zone for Arena { self.log.push(format!("leave:{reason:?}")); } + fn on_connection_lost(&mut self, _ctx: &mut Ctx<'_, Self>, player: PlayerId) { + self.connection_events.push(format!("lost:{}", player.0)); + } + + fn on_reconnected(&mut self, _ctx: &mut Ctx<'_, Self>, player: PlayerId) { + self.connection_events.push(format!("back:{}", player.0)); + } + fn on_observe(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId) { ctx.set_view(observer, View::new(50.0, 50.0, 1000.0)); let info = ctx diff --git a/distributed_game/tests/zone_runtime/main.rs b/distributed_game/tests/zone_runtime/main.rs index 6bd1bd5c4..2a9d45ae6 100644 --- a/distributed_game/tests/zone_runtime/main.rs +++ b/distributed_game/tests/zone_runtime/main.rs @@ -25,6 +25,7 @@ fn claims(principal: &str, zone: &str) -> TicketClaims { nonce: format!("n-{principal}"), role: Role::Player, name: None, + extensions: Default::default(), } } @@ -720,6 +721,7 @@ impl distributed_game::Zone for Paced { type Config = (); type Input = u8; type Timer = (); + type Message = distributed_game::NoMessages; fn schema() -> distributed_game::EntitySchema { distributed_game::EntitySchema::new() diff --git a/distributed_game/tests/zone_state/main.rs b/distributed_game/tests/zone_state/main.rs new file mode 100644 index 000000000..29b9665b0 --- /dev/null +++ b/distributed_game/tests/zone_state/main.rs @@ -0,0 +1,243 @@ +//! `ZoneState` cells on the in-process reference host: the zone-state +//! fencing contract (`state_store_conformance`) through the cell's +//! commands, for inline snapshots and for snapshots in object storage (an +//! in-memory blob map standing in for celld's R2 binding, split exactly as +//! the worker route splits them), plus the command's own guards. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +use base64::Engine as _; +use distributed::cell_host::{ + AggregateCell, CellCommandIdentity, CellDispatchError, CellStreamStore, +}; +use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; +use distributed::StreamIdentity; +use distributed_game::cell::command_id_for; +use distributed_game::world::zone_state::{ + blob_key, mount_zone_state, sha256_hex, stale_current, SnapshotBytes, ZoneState, DELETE, SAVE, + ZONE_HOST_ROLE, ZONE_STATE_INLINE_MAX, +}; +use distributed_game::world::{ + state_store_conformance, StateError, StoredZoneState, ZoneStateStore, +}; +use distributed_game::BoxFuture; +use serde_json::{json, Value}; + +const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; + +fn host_session() -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, "zone-server-a"); + session.set(ROLE_KEY, ZONE_HOST_ROLE); + session +} + +/// `ZoneStateStore` over one in-process cell + a blob map, as +/// `CellZoneStateStore` + the worker route are over celld. +struct InProcess { + cell: AggregateCell, + blobs: Mutex>>, + calls: AtomicU64, +} + +impl InProcess { + fn new(zone: &str) -> Self { + let store = CellStreamStore::for_identity(StreamIdentity::new("ZoneState", zone).unwrap()); + Self { + cell: mount_zone_state(AggregateCell::from_store(store).unwrap()), + blobs: Mutex::default(), + calls: AtomicU64::new(0), + } + } + + async fn dispatch( + &self, + command: &str, + input: Value, + session: Session, + ) -> Result { + let n = self.calls.fetch_add(1, Ordering::SeqCst); + let identity = CellCommandIdentity::new( + "zone-host", + "zone-server-a", + command_id_for("t", &n.to_string()), + ) + .unwrap(); + self.cell + .dispatch_idempotent(command, &identity, input, session) + .await + .map(|result| result.payload().clone()) + } + + async fn command( + &self, + command: &str, + input: Value, + uploaded: Option, + ) -> Result<(), StateError> { + let token = input["token"].as_u64().unwrap(); + match self.dispatch(command, input, host_session()).await { + Ok(payload) => { + if let Some(key) = payload["superseded_blob"].as_str() { + self.blobs.lock().unwrap().remove(key); + } + Ok(()) + } + Err(error) => { + if let Some(key) = uploaded { + self.blobs.lock().unwrap().remove(&key); + } + match error { + CellDispatchError::Rejected { message, .. } => match stale_current(&message) { + Some(current) => Err(StateError::Stale { + presented: token, + current, + }), + None => Err(StateError::Backend(message)), + }, + other => Err(StateError::Backend(format!("{other:?}"))), + } + } + } + } +} + +impl ZoneStateStore for InProcess { + fn load<'a>( + &'a self, + _zone_id: &'a str, + ) -> BoxFuture<'a, Result, StateError>> { + Box::pin(async move { + let Some(state) = self.cell.load().await.unwrap() else { + return Ok(None); + }; + let bytes = match state.bytes() { + None => return Ok(None), + Some(SnapshotBytes::Inline(bytes)) => bytes.clone(), + Some(SnapshotBytes::Blob(blob)) => { + let bytes = self + .blobs + .lock() + .unwrap() + .get(&blob.key) + .cloned() + .expect("blob"); + assert_eq!( + (bytes.len() as u64, sha256_hex(&bytes)), + (blob.len, blob.sha256.clone()) + ); + bytes + } + }; + Ok(Some(StoredZoneState { + token: state.token(), + content_version: state.content_version().to_string(), + state: bytes, + })) + }) + } + + fn save<'a>( + &'a self, + zone_id: &'a str, + state: &'a StoredZoneState, + ) -> BoxFuture<'a, Result<(), StateError>> { + Box::pin(async move { + if state.state.len() <= ZONE_STATE_INLINE_MAX { + let input = json!({ + "zone_id": zone_id, "token": state.token, "content_version": state.content_version, + "inline_b64": B64.encode(&state.state), "blob": null, + }); + self.command(SAVE, input, None).await + } else { + let sha = sha256_hex(&state.state); + let key = blob_key(zone_id, state.token, &sha); + self.blobs + .lock() + .unwrap() + .insert(key.clone(), state.state.clone()); + let input = json!({ + "zone_id": zone_id, "token": state.token, "content_version": state.content_version, + "inline_b64": null, + "blob": { "key": key, "sha256": sha, "len": state.state.len() }, + }); + self.command(SAVE, input, Some(key)).await + } + }) + } + + fn delete<'a>(&'a self, zone_id: &'a str, token: u64) -> BoxFuture<'a, Result<(), StateError>> { + Box::pin(async move { + self.command(DELETE, json!({ "zone_id": zone_id, "token": token }), None) + .await + }) + } +} + +#[tokio::test] +async fn inline_snapshots_meet_the_zone_state_contract() { + let store = InProcess::new("grid.0.0"); + state_store_conformance(&store, "grid.0.0", vec![3; 1_000]).await; + assert!( + store.blobs.lock().unwrap().is_empty(), + "small snapshots stay in the cell" + ); +} + +#[tokio::test] +async fn large_snapshots_go_to_object_storage_and_superseded_blobs_are_dropped() { + let store = InProcess::new("grid.1.0"); + let big = (0..ZONE_STATE_INLINE_MAX + 4_096) + .map(|i| (i % 251) as u8) + .collect::>(); + state_store_conformance(&store, "grid.1.0", big.clone()).await; + // Only the live snapshot's object remains (the stale upload, the + // superseded saves and the deleted one are gone). + let blobs = store.blobs.lock().unwrap(); + assert_eq!(blobs.len(), 1, "{:?}", blobs.keys().collect::>()); + assert_eq!(blobs.values().next().unwrap(), &big); + drop(blobs); + let state = store.cell.load().await.unwrap().unwrap(); + assert!( + matches!(state.bytes(), Some(SnapshotBytes::Blob(blob)) if blob.len == big.len() as u64) + ); +} + +#[tokio::test] +async fn commands_need_the_zone_host_role_and_one_well_formed_payload() { + let store = InProcess::new("grid.2.0"); + let input = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", + "inline_b64": B64.encode(b"x"), "blob": null, + }); + let mut player = Session::new(); + player.set(USER_ID_KEY, "alice"); + assert!( + store.dispatch(SAVE, input.clone(), player).await.is_err(), + "players cannot write zone state" + ); + + let both = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", "inline_b64": B64.encode(b"x"), + "blob": { "key": "k", "sha256": sha256_hex(b"x"), "len": 1 }, + }); + let neither = json!({ "zone_id": "grid.2.0", "token": 1, "content_version": "c", "inline_b64": null, "blob": null }); + let oversized = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", + "inline_b64": B64.encode(vec![0u8; ZONE_STATE_INLINE_MAX + 1]), "blob": null, + }); + let bad_digest = json!({ + "zone_id": "grid.2.0", "token": 1, "content_version": "c", "inline_b64": null, + "blob": { "key": "k", "sha256": "nope", "len": 1 }, + }); + for bad in [both, neither, oversized, bad_digest] { + assert!(matches!( + store.dispatch(SAVE, bad, host_session()).await, + Err(CellDispatchError::Rejected { .. }) + )); + } + store.dispatch(SAVE, input, host_session()).await.unwrap(); + assert_eq!(store.load("grid.2.0").await.unwrap().unwrap().state, b"x"); +} diff --git a/distributed_game/tests/zone_status/main.rs b/distributed_game/tests/zone_status/main.rs new file mode 100644 index 000000000..18e6ffa15 --- /dev/null +++ b/distributed_game/tests/zone_status/main.rs @@ -0,0 +1,232 @@ +//! The zone status read model end to end on the reference host: lease and +//! zone-state cells commit their outward events into their outboxes; the +//! messages go onto a SQLite bus; the `zone_status` projectors (a consumer +//! service like any other) upsert `zone_leases` / `zone_snapshots`; and +//! `ZoneStatus::of` tells active, hibernated and cold zones apart without +//! opening a cell. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; + +use distributed::bus::{Message, MessagePublisher, SqliteBus}; +use distributed::cell_host::{AggregateCell, CellCommandIdentity, CellStreamStore}; +use distributed::microsvc::{ + spawn_service_consumer_loop, CausalProjectorContext, HandlerError, ModeledProjection, Routes, + Service, Session, CONSUMER_IDLE_POLL, ROLE_KEY, USER_ID_KEY, +}; +use distributed::{Aggregate, BusPublisher, ManualClock, SqliteRepository, StreamIdentity}; +use distributed_game::cell::command_id_for; +use distributed_game::world::zone_lease::{mount_zone_lease, ZoneLease, ACQUIRE, RELEASE}; +use distributed_game::world::zone_state::{mount_zone_state, ZoneState, SAVE, ZONE_HOST_ROLE}; +use distributed_game::world::zone_status::{ + projectors, read_model_catalog, ZoneLeases, ZoneSnapshots, ZoneStatus, APPLICATION, + ZONE_LEASES, ZONE_SNAPSHOTS, +}; +use serde_json::{json, Value}; + +const T0: u64 = 1_700_000_000_000; +static CALLS: AtomicU64 = AtomicU64::new(0); + +fn host_session() -> Session { + let mut session = Session::new(); + session.set(USER_ID_KEY, "zone-server-a"); + session.set(ROLE_KEY, ZONE_HOST_ROLE); + session +} + +fn cell( + kind: &str, + zone: &str, + clock: &ManualClock, +) -> AggregateCell { + let store = CellStreamStore::for_identity(StreamIdentity::new(kind, zone).unwrap()) + .with_clock(clock.clone()); + AggregateCell::from_store(store).unwrap() +} + +async fn dispatch( + cell: &AggregateCell, + command: &str, + input: Value, +) -> Value { + let n = CALLS.fetch_add(1, Ordering::SeqCst); + let identity = CellCommandIdentity::new( + "zone-host", + "zone-server-a", + command_id_for("t", &n.to_string()), + ) + .unwrap(); + cell.dispatch_idempotent(command, &identity, input, host_session()) + .await + .unwrap_or_else(|error| panic!("{command}: {error:?}")) + .payload() + .clone() +} + +async fn project_leases( + context: CausalProjectorContext, + projection: ModeledProjection, +) -> Result<(), HandlerError> { + projection.apply(ZONE_LEASES, &context).await +} + +async fn project_snapshots( + context: CausalProjectorContext, + projection: ModeledProjection, +) -> Result<(), HandlerError> { + projection.apply(ZONE_SNAPSHOTS, &context).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cell_events_project_into_zone_status_rows() { + let clock = ManualClock::new(SystemTime::UNIX_EPOCH + Duration::from_millis(T0)); + // z-active: held. z-asleep: held, snapshot saved, released. + // z-cold: held and released without a snapshot. + let active = mount_zone_lease(cell::("ZoneLease", "z-active", &clock)); + let asleep = mount_zone_lease(cell::("ZoneLease", "z-asleep", &clock)); + let asleep_state = mount_zone_state(cell::("ZoneState", "z-asleep", &clock)); + let cold = mount_zone_lease(cell::("ZoneLease", "z-cold", &clock)); + let acquire = |zone: &str| json!({ "zone_id": zone, "holder": "A#1", "ttl_ms": 60_000 }); + dispatch(&active, ACQUIRE, acquire("z-active")).await; + let held = dispatch(&asleep, ACQUIRE, acquire("z-asleep")).await; + use base64::Engine as _; + dispatch( + &asleep_state, + SAVE, + json!({ "zone_id": "z-asleep", "token": held["token"], "content_version": "v1", + "inline_b64": base64::engine::general_purpose::STANDARD.encode([7u8; 42]) }), + ) + .await; + dispatch( + &asleep, + RELEASE, + json!({ "zone_id": "z-asleep", "holder": "A#1", "token": held["token"] }), + ) + .await; + let held = dispatch(&cold, ACQUIRE, acquire("z-cold")).await; + dispatch( + &cold, + RELEASE, + json!({ "zone_id": "z-cold", "holder": "A#1", "token": held["token"] }), + ) + .await; + + // Outbox → bus, in commit order per cell. + let dir = std::env::temp_dir().join(format!( + "zone-status-{}-{}", + std::process::id(), + CALLS.load(Ordering::SeqCst) + )); + std::fs::create_dir_all(&dir).unwrap(); + let bus_url = format!("sqlite:{}?mode=rwc", dir.join("bus.db").display()); + let pool = sqlx::SqlitePool::connect(&bus_url).await.unwrap(); + let bus = SqliteBus::new(pool.clone()); + bus.ensure_tables().await.unwrap(); + let publisher = BusPublisher::new(Arc::new(bus)); + let mut messages = Vec::new(); + messages.extend(active.durable_outbox().unwrap()); + messages.extend(asleep.durable_outbox().unwrap()); + messages.extend(asleep_state.durable_outbox().unwrap()); + messages.extend(cold.durable_outbox().unwrap()); + messages.sort_by_key(|m| m.created_at); + assert_eq!(messages.len(), 6, "activated ×3, released ×2, saved"); + for message in messages { + publisher.publish(Message::from(message)).await.unwrap(); + } + + // The projectors, as a consumer service. + let read_url = format!("sqlite:{}?mode=rwc", dir.join("read.db").display()); + let repo = SqliteRepository::connect_and_migrate(&read_url) + .await + .unwrap(); + repo.bootstrap_table_schema_for_dev(&read_model_catalog().table_registry().unwrap()) + .await + .unwrap(); + let read_models = repo.clone(); + spawn_service_consumer_loop(move || { + let (leases, snapshots) = projectors(); + Service::new() + .named(APPLICATION) + .routes( + Routes::new() + .with_read_model_store(read_models.clone()) + .modeled_projector(leases) + .handle(project_leases) + .modeled_projector(snapshots) + .handle(project_snapshots), + ) + .with_bus(SqliteBus::new(pool.clone()).with_idle_poll(CONSUMER_IDLE_POLL)) + }); + + let status = |zone: &'static str| { + let repo = repo.clone(); + async move { + let lease: Option = sqlx::query_as::<_, (String, String, i64, String)>( + "SELECT zone_id, holder, token, status FROM zone_leases WHERE zone_id = ?", + ) + .bind(zone) + .fetch_optional(repo.pool()) + .await + .unwrap() + .map(|(zone_id, holder, token, status)| ZoneLeases { + zone_id, + holder, + token, + status, + }); + let snapshot: Option = + sqlx::query_as::<_, (String, i64, bool, String, i64)>( + "SELECT zone_id, token, stored, content_version, bytes FROM zone_snapshots WHERE zone_id = ?", + ) + .bind(zone) + .fetch_optional(repo.pool()) + .await + .unwrap() + .map(|(zone_id, token, stored, content_version, bytes)| ZoneSnapshots { + zone_id, + token, + stored, + content_version, + bytes, + }); + (lease, snapshot) + } + }; + let start = Instant::now(); + loop { + let (lease, snapshot) = status("z-asleep").await; + let done = lease.as_ref().is_some_and(|l| l.status == "released") && snapshot.is_some(); + if done + && status("z-cold") + .await + .0 + .is_some_and(|l| l.status == "released") + { + break; + } + assert!( + start.elapsed() < Duration::from_secs(20), + "projection timed out" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let of = |(lease, snapshot): (Option, Option)| { + ZoneStatus::of(lease.as_ref(), snapshot.as_ref()) + }; + assert_eq!( + of(status("z-active").await), + ZoneStatus::Active { + holder: "A#1".into() + } + ); + assert_eq!( + of(status("z-asleep").await), + ZoneStatus::Hibernated { bytes: 42 } + ); + let (_, snapshot) = status("z-asleep").await; + assert_eq!(snapshot.unwrap().content_version, "v1"); + assert_eq!(of(status("z-cold").await), ZoneStatus::Cold); + assert_eq!(of(status("z-never").await), ZoneStatus::Cold); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/distributed_game_macros/src/claims.rs b/distributed_game_macros/src/claims.rs new file mode 100644 index 000000000..fd489190e --- /dev/null +++ b/distributed_game_macros/src/claims.rs @@ -0,0 +1,69 @@ +//! `#[derive(TicketExtension)]`: declare the extension claims a ticket +//! issuer may vouch for (match id, team, party…). + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{DeriveInput, LitStr}; + +use crate::shared::{field_name, named_fields, no_generics, runtime}; +use crate::trusted::reject_forbidden; + +pub(crate) fn expand(input: DeriveInput) -> syn::Result { + no_generics(&input, "TicketExtension")?; + let rt = runtime()?; + // Container-wide renames would change the names the schema admits + // behind the deny list's back. + for attr in input.attrs.iter().filter(|a| a.path().is_ident("serde")) { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("rename_all") || meta.path.is_ident("rename_all_fields") { + return Err(meta.error( + "#[derive(TicketExtension)] does not support `rename_all`; rename fields individually", + )); + } + if meta.input.peek(syn::Token![=]) { + let _ = meta.value()?.parse::()?; + } else if meta.input.peek(syn::token::Paren) { + meta.parse_nested_meta(|_| Ok(()))?; + } + Ok(()) + })?; + } + let fields = named_fields(&input, "TicketExtension")?; + reject_forbidden( + &fields, + "ticket claim", + "identity is only ever the verified `sub`, and rewards and economy values are decided by command handlers, never vouched for in a ticket", + )?; + let mut names = Vec::new(); + for field in &fields { + let mut name = field_name(field.ident.as_ref().expect("named")); + for attr in field.attrs.iter().filter(|a| a.path().is_ident("serde")) { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("flatten") { + return Err(meta.error( + "#[derive(TicketExtension)] does not support `flatten`; declare every claim as a field", + )); + } + if meta.path.is_ident("rename") { + if meta.input.peek(syn::Token![=]) { + name = meta.value()?.parse::()?.value(); + } else { + return Err(meta.error("use `rename = \"..\"` (one name for both directions)")); + } + } else if meta.input.peek(syn::Token![=]) { + let _ = meta.value()?.parse::()?; + } else if meta.input.peek(syn::token::Paren) { + meta.parse_nested_meta(|_| Ok(()))?; + } + Ok(()) + })?; + } + names.push(name); + } + let ident = &input.ident; + Ok(quote! { + impl #rt::TicketExtension for #ident { + const CLAIMS: &'static [&'static str] = &[#(#names),*]; + } + }) +} diff --git a/distributed_game_macros/src/fact.rs b/distributed_game_macros/src/fact.rs index 09fa5516d..3757108d3 100644 --- a/distributed_game_macros/src/fact.rs +++ b/distributed_game_macros/src/fact.rs @@ -4,43 +4,10 @@ use proc_macro2::TokenStream; use quote::quote; use syn::spanned::Spanned; -use syn::{DeriveInput, Expr, Ident, LitStr}; +use syn::{DeriveInput, Expr, Ident}; -use crate::shared::{field_name, find_attr, named_fields, no_generics, runtime}; - -/// Field names a zone-emitted fact must never carry: identity is bound by -/// the host from the verified ticket, rewards are decided by the handler. -const FORBIDDEN: &[(&str, &str)] = &[ - ("principal", "identity"), - ("user", "identity"), - ("user_id", "identity"), - ("userid", "identity"), - ("owner", "identity"), - ("owner_id", "identity"), - ("account", "identity"), - ("account_id", "identity"), - ("subject", "identity"), - ("sub", "identity"), - ("xp", "a reward"), - ("experience", "a reward"), - ("reward", "a reward"), - ("rewards", "a reward"), -]; - -fn normalized(name: &str) -> String { - name.chars() - .filter(|c| *c != '_' && *c != '-') - .flat_map(char::to_lowercase) - .collect() -} - -fn forbidden(name: &str) -> Option<&'static str> { - let n = normalized(name); - FORBIDDEN - .iter() - .find(|(bad, _)| normalized(bad) == n) - .map(|(_, why)| *why) -} +use crate::shared::{find_attr, named_fields, no_generics, runtime}; +use crate::trusted::reject_forbidden; pub(crate) fn expand(input: DeriveInput) -> syn::Result { no_generics(&input, "FactCommand")?; @@ -78,37 +45,11 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { }; let fields = named_fields(&input, "FactCommand")?; - for field in &fields { - let ident = field.ident.as_ref().expect("named"); - let mut names = vec![(field_name(ident), ident.span())]; - // Serde renames change what the handler sees. - for a in field.attrs.iter().filter(|a| a.path().is_ident("serde")) { - let _ = a.parse_nested_meta(|meta| { - if meta.path.is_ident("rename") || meta.path.is_ident("alias") { - if let Ok(value) = meta.value() { - if let Ok(lit) = value.parse::() { - names.push((lit.value(), lit.span())); - } - } - } else if meta.input.peek(syn::Token![=]) { - let _ = meta.value()?.parse::(); - } else if meta.input.peek(syn::token::Paren) { - let _ = meta.parse_nested_meta(|_| Ok(())); - } - Ok(()) - }); - } - for (name, span) in names { - if let Some(why) = forbidden(&name) { - return Err(syn::Error::new( - span, - format!( - "fact input field `{name}` looks like {why}: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards" - ), - )); - } - } - } + reject_forbidden( + &fields, + "fact input", + "a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values", + )?; let parts = match &key { None => quote!(::std::vec::Vec::new()), diff --git a/distributed_game_macros/src/lib.rs b/distributed_game_macros/src/lib.rs index b3bd97a75..9612813d2 100644 --- a/distributed_game_macros/src/lib.rs +++ b/distributed_game_macros/src/lib.rs @@ -9,6 +9,7 @@ //! | [`GameEvent`] | struct / enum | `GameEvent`: stable ids, typed payload codec | //! | [`GameRecord`] | struct | `Wire`: a nested payload record | //! | [`FactCommand`] | struct | `FactCommand`: the cell command name of a fact input | +//! | [`TicketExtension`] | struct | `TicketExtension`: the extension claims a ticket may carry | //! | [`macro@predict`] (attribute) | fn | `PredictStep` for a client-predicted step + its WebAssembly export | //! | [`macro@game_pure`] (attribute) | fn | `GamePure` for a pure function compiled to WebAssembly + its export | //! @@ -20,10 +21,12 @@ //! `FieldType` expression (e.g. a shared `const POS: FieldType`). //! - `#[long]` on a `String` payload field: varuint length instead of the //! 255-byte short string. -//! - Replicated only: `#[priority(P)]`, `#[correct(ease = MS | ease | snap | replace)]`. +//! - Replicated only: `#[priority(P)]`, `#[correct(ease = MS | ease | snap | replace)]`, +//! `#[from]` / `#[from(path.to.field)]` (with `#[replicated(.., of = Entity)]`). //! - GameInput only: `#[range(min = A, max = B)]` (clamp), `#[buttons(a, b, ..)]` //! (bit flags, unknown bits cleared; also generates `Type::A`, `Type::B` consts). +mod claims; mod event; mod fact; mod input; @@ -31,6 +34,7 @@ mod predict; mod pure; mod replicated; mod shared; +mod trusted; use proc_macro::TokenStream; use syn::{parse_macro_input, DeriveInput, ItemFn}; @@ -40,14 +44,21 @@ fn finish(result: syn::Result) -> TokenStream { } /// One replicated entity kind: `#[replicated(kind = "mob", id = 2, -/// priority = 2.0, global, x = field, y = field)]` on a struct with named -/// fields (Rust type → wire type: `bool`, `u8`, `u16`, `u32`, `i32`, `f32`, -/// `String` → short_str, `Vec` → bytes). The interest position is the -/// `x`/`y` fields unless overridden; a kind without a position must be -/// `global`. +/// priority = 2.0, global, x = field, y = field, of = Entity)]` on a struct +/// with named fields (Rust type → wire type: `bool`, `u8`, `u16`, `u32`, +/// `i32`, `f32`, `String` → short_str, `Vec` → bytes). The interest +/// position is the `x`/`y` fields unless overridden; a kind without a +/// position must be `global`. +/// +/// `of = Entity` makes the struct a view projected from a zone entity +/// (`distributed_game::entity`): fields marked `#[from]` (same name) or +/// `#[from(path.to.field)]` are copied from the entity, and the derive +/// generates `View::project(&entity, ViewRest { .. })`, `ViewRest` holding +/// the remaining hand-filled fields. When every field is `#[from]` it +/// generates `View::project(&entity)` and `impl Project for Entity`. #[proc_macro_derive( Replicated, - attributes(replicated, quant, angle, field_type, priority, correct) + attributes(replicated, quant, angle, field_type, priority, correct, from) )] pub fn derive_replicated(input: TokenStream) -> TokenStream { finish(replicated::expand(parse_macro_input!(input as DeriveInput))) @@ -81,13 +92,25 @@ pub fn derive_game_record(input: TokenStream) -> TokenStream { /// Bind a cell command input to its command: /// `#[fact(command = RECORD_KILL, key(mob, generation))]`. Requires -/// `serde::Serialize`. Identity- and reward-looking field names are -/// rejected. +/// `serde::Serialize`. Identity-, reward- and economy-looking field names +/// (`principal`, `user_id`, `xp`, `reward`, `amount`, `gold`, `price`, +/// `balance`, `payout`, …; `_`/`-`/case ignored, serde renames included) +/// are rejected. #[proc_macro_derive(FactCommand, attributes(fact))] pub fn derive_fact_command(input: TokenStream) -> TokenStream { finish(fact::expand(parse_macro_input!(input as DeriveInput))) } +/// Declare a game's ticket extension claims (match id, team, party…): +/// a struct with named fields that also derives serde `Serialize` and +/// `Deserialize`. Generates `TicketExtension::CLAIMS` (serialized names). +/// Identity-, reward- and economy-looking names are rejected with the same +/// deny list as [`FactCommand`]; `rename_all` and `flatten` are refused. +#[proc_macro_derive(TicketExtension)] +pub fn derive_ticket_extension(input: TokenStream) -> TokenStream { + finish(claims::expand(parse_macro_input!(input as DeriveInput))) +} + /// Declare a zone's client-predicted step: /// `#[predict(ArenaStep, kind = "player")]` on /// `fn(state: &State, input: &Input, context: &Context, dt: f32) -> State`. diff --git a/distributed_game_macros/src/replicated.rs b/distributed_game_macros/src/replicated.rs index 6bbc286c4..3d7d724cb 100644 --- a/distributed_game_macros/src/replicated.rs +++ b/distributed_game_macros/src/replicated.rs @@ -2,8 +2,9 @@ use proc_macro2::{Span, TokenStream}; use quote::{quote, quote_spanned}; +use syn::punctuated::Punctuated; use syn::spanned::Spanned; -use syn::{DeriveInput, Expr, Ident, LitInt, LitStr}; +use syn::{DeriveInput, Expr, Field, Ident, LitInt, LitStr, Member, Token, Type}; use crate::shared::{ check_name, field_name, find_attr, named_fields, no_generics, parse_field_attrs, reject, @@ -19,6 +20,7 @@ struct KindAttrs { global: bool, x: Option, y: Option, + of: Option, } fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { @@ -28,8 +30,8 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { "#[derive(Replicated)] needs #[replicated(kind = \"name\", id = N)]", )); }; - let (mut kind, mut id, mut priority, mut global, mut x, mut y) = - (None, None, None, false, None, None); + let (mut kind, mut id, mut priority, mut global, mut x, mut y, mut of) = + (None, None, None, false, None, None, None); attr.parse_nested_meta(|meta| { let key = meta .path @@ -46,9 +48,10 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { "global" => global = true, "x" => x = Some(meta.value()?.parse::()?), "y" => y = Some(meta.value()?.parse::()?), + "of" => of = Some(meta.value()?.parse::()?), _ => { return Err(meta.error( - "expected `kind`, `id`, `priority`, `global`, `x` or `y` in #[replicated(..)]", + "expected `kind`, `id`, `priority`, `global`, `x`, `y` or `of` in #[replicated(..)]", )) } } @@ -65,6 +68,93 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { global, x, y, + of, + }) +} + +/// `#[from]` (same field name) or `#[from(a.b)]` (a field path on the +/// entity): the field path to copy, spanned at the attribute. +fn from_path(field: &Field) -> syn::Result, Span)>> { + let Some(attr) = find_attr(&field.attrs, "from") else { + return Ok(None); + }; + let span = attr.span(); + if matches!(attr.meta, syn::Meta::Path(_)) { + let ident = field.ident.clone().expect("named"); + return Ok(Some((vec![Member::Named(ident)], span))); + } + let path = attr.parse_args_with(Punctuated::::parse_separated_nonempty)?; + Ok(Some((path.into_iter().collect(), span))) +} + +/// With `of = T`: `View::project(&T, ViewRest)` (or, when every field is +/// `#[from]`, `View::project(&T)` plus `impl Project for T`). +fn expand_of( + input: &DeriveInput, + of: &Type, + fields: &[&Field], + rt: &TokenStream, +) -> syn::Result { + let ident = &input.ident; + let vis = &input.vis; + // Zone entities are usually private to their module while views are + // public: a crate-visible constructor never leaks the entity type. + let fn_vis = quote!(pub(crate)); + let rest_ident = Ident::new(&format!("{ident}Rest"), ident.span()); + let mut inits = Vec::new(); + let mut rest_fields = Vec::new(); + for field in fields { + let name = field.ident.as_ref().expect("named"); + match from_path(field)? { + Some((path, span)) => { + // A missing path is "no field", a different type + // "mismatched types", both at the `#[from]`. + inits.push(quote_spanned! {span=> + #name: (src #(. #path)*).clone() + }); + } + None => { + let docs = field.attrs.iter().filter(|a| a.path().is_ident("doc")); + let (fvis, ty) = (&field.vis, &field.ty); + rest_fields.push(quote! { #(#docs)* #fvis #name: #ty }); + inits.push(quote! { #name: rest.#name }); + } + } + } + if rest_fields.is_empty() { + return Ok(quote! { + impl #ident { + /// Project the entity: every field is `#[from]`. + #[allow(clippy::clone_on_copy)] + #fn_vis fn project(src: &#of) -> Self { + Self { #(#inits),* } + } + } + + impl #rt::Project<#ident> for #of { + fn view(&self) -> ::core::option::Option<#ident> { + ::core::option::Option::Some(#ident::project(self)) + } + } + }); + } + let doc = + format!("The hand-filled (non-`#[from]`) fields of [`{ident}`], for [`{ident}::project`]."); + Ok(quote! { + #[doc = #doc] + #[derive(Debug, Clone, PartialEq)] + #vis struct #rest_ident { + #(#rest_fields),* + } + + impl #ident { + /// Project the entity: `#[from]` fields are copied from `src`, + /// the others taken from `rest`. + #[allow(clippy::clone_on_copy)] + #fn_vis fn project(src: &#of, rest: #rest_ident) -> Self { + Self { #(#inits),* } + } + } }) } @@ -164,6 +254,22 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { let kind = &attrs.kind; let id = attrs.id; let global = attrs.global; + let projection = match &attrs.of { + Some(of) => expand_of(&input, of, &fields, &rt)?, + None => { + if let Some(field) = fields + .iter() + .find(|f| find_attr(&f.attrs, "from").is_some()) + { + let attr = find_attr(&field.attrs, "from").expect("found"); + return Err(syn::Error::new( + attr.span(), + "#[from] needs the entity type: #[replicated(.., of = Entity)]", + )); + } + TokenStream::new() + } + }; let kind_priority = attrs .priority .as_ref() @@ -190,5 +296,7 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { #p::std::vec![#(#values),*] } } + + #projection }) } diff --git a/distributed_game_macros/src/trusted.rs b/distributed_game_macros/src/trusted.rs new file mode 100644 index 000000000..a7a23df83 --- /dev/null +++ b/distributed_game_macros/src/trusted.rs @@ -0,0 +1,142 @@ +//! The deny list for fields a game must not let untrusted data carry. +//! +//! Zone facts (`FactCommand`) and ticket extension claims share it: identity +//! is bound by the host from the verified ticket, and rewards and economy +//! amounts are decided by the deterministic command handler, never by a +//! field the zone or the browser filled in. + +use proc_macro2::Span; +use syn::{Expr, Field, LitStr}; + +use crate::shared::field_name; + +/// Denied names and what they look like. Matching ignores `_`, `-` and case, +/// so `user_id`, `userId`, `USER-ID` and `UserID` all hit `user_id`. +const FORBIDDEN: &[(&str, &str)] = &[ + ("principal", "identity"), + ("user", "identity"), + ("user_id", "identity"), + ("owner", "identity"), + ("owner_id", "identity"), + ("account", "identity"), + ("account_id", "identity"), + ("subject", "identity"), + ("sub", "identity"), + ("xp", "a reward"), + ("experience", "a reward"), + ("reward", "a reward"), + ("rewards", "a reward"), + ("amount", "an economy value"), + ("coins", "an economy value"), + ("gold", "an economy value"), + ("currency", "an economy value"), + ("price", "an economy value"), + ("cost", "an economy value"), + ("balance", "an economy value"), + ("grant", "an economy value"), + ("payout", "an economy value"), + ("credit", "an economy value"), + ("debit", "an economy value"), +]; + +fn normalized(name: &str) -> String { + name.chars() + .filter(|c| *c != '_' && *c != '-') + .flat_map(char::to_lowercase) + .collect() +} + +/// What `name` looks like, if it is denied. +pub(crate) fn forbidden(name: &str) -> Option<&'static str> { + let n = normalized(name); + FORBIDDEN + .iter() + .find(|(bad, _)| normalized(bad) == n) + .map(|(_, why)| *why) +} + +/// Every name a field is seen under by a serde consumer: the Rust name plus +/// any `#[serde(rename = ..)]` / `#[serde(alias = ..)]`. +fn seen_names(field: &Field) -> Vec<(String, Span)> { + let ident = field.ident.as_ref().expect("named"); + let mut names = vec![(field_name(ident), ident.span())]; + for a in field.attrs.iter().filter(|a| a.path().is_ident("serde")) { + let _ = a.parse_nested_meta(|meta| { + if meta.path.is_ident("rename") || meta.path.is_ident("alias") { + if let Ok(value) = meta.value() { + if let Ok(lit) = value.parse::() { + names.push((lit.value(), lit.span())); + } + } + } else if meta.input.peek(syn::Token![=]) { + let _ = meta.value()?.parse::(); + } else if meta.input.peek(syn::token::Paren) { + let _ = meta.parse_nested_meta(|_| Ok(())); + } + Ok(()) + }); + } + names +} + +/// Reject every denied field name (one spanned error each, combined). +/// `what` names the declaration ("fact input", "ticket claim") and `rule` +/// ends the message with why the value must come from elsewhere. +pub(crate) fn reject_forbidden(fields: &[&Field], what: &str, rule: &str) -> syn::Result<()> { + let mut errors: Option = None; + for field in fields { + for (name, span) in seen_names(field) { + if let Some(why) = forbidden(&name) { + let error = syn::Error::new( + span, + format!("{what} field `{name}` looks like {why}: {rule}"), + ); + match &mut errors { + Some(all) => all.combine(error), + None => errors = Some(error), + } + } + } + } + errors.map_or(Ok(()), Err) +} + +#[cfg(test)] +mod tests { + use super::{forbidden, FORBIDDEN}; + + /// The JS ticket handler enforces the same list from this fixture. + #[test] + fn deny_list_matches_the_shared_fixture() { + let fixture = + include_str!("../../distributed_game/tests/fixtures/trusted_field_denylist.json"); + let mut expected = Vec::new(); + for (group, why) in [ + ("identity", "identity"), + ("reward", "a reward"), + ("economy", "an economy value"), + ] { + let start = fixture.find(&format!("\"{group}\": [")).expect(group); + let list = &fixture[start..]; + let list = &list[list.find('[').unwrap() + 1..list.find(']').unwrap()]; + for name in list.split(',') { + expected.push((name.trim().trim_matches('"').to_string(), why)); + } + } + let actual: Vec<_> = FORBIDDEN.iter().map(|(n, w)| (n.to_string(), *w)).collect(); + assert_eq!(actual, expected); + } + + #[test] + fn matches_ignore_separators_and_case() { + for name in ["userId", "USER-ID", "user_id", "UserID", "Owner_Id"] { + assert_eq!(forbidden(name), Some("identity"), "{name}"); + } + for name in ["Gold", "GOLD", "pay_out", "Pay-Out", "c_o_i_n_s", "Balance"] { + assert_eq!(forbidden(name), Some("an economy value"), "{name}"); + } + for name in ["mob", "generation", "amounts", "coin", "costs", "kills"] { + assert_eq!(forbidden(name), None, "{name}"); + } + } +} diff --git a/distributed_game_macros/tests/compile_fail/fact_economy_fields.rs b/distributed_game_macros/tests/compile_fail/fact_economy_fields.rs new file mode 100644 index 000000000..dcec8fe77 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_economy_fields.rs @@ -0,0 +1,26 @@ +use distributed_game::FactCommand; + +// Every economy name is denied, whatever its case or separators, and under +// a serde rename or alias too. Each one gets its own error. +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "crawler.loot_found")] +struct LootFound { + chest: String, + amount: u32, + Coins: u32, + GOLD: u32, + currency_: String, + price: u32, + cost: u32, + bal_ance: u64, + grant: u32, + pay_out: u32, + CREDIT: u32, + debit: u32, + #[serde(rename = "Pay-Out")] + a: u32, + #[serde(alias = "goldAmount", alias = "gold-")] + b: u32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/fact_economy_fields.stderr b/distributed_game_macros/tests/compile_fail/fact_economy_fields.stderr new file mode 100644 index 000000000..5c1e92975 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_economy_fields.stderr @@ -0,0 +1,77 @@ +error: fact input field `amount` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:9:5 + | +9 | amount: u32, + | ^^^^^^ + +error: fact input field `Coins` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:10:5 + | +10 | Coins: u32, + | ^^^^^ + +error: fact input field `GOLD` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:11:5 + | +11 | GOLD: u32, + | ^^^^ + +error: fact input field `currency_` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:12:5 + | +12 | currency_: String, + | ^^^^^^^^^ + +error: fact input field `price` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:13:5 + | +13 | price: u32, + | ^^^^^ + +error: fact input field `cost` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:14:5 + | +14 | cost: u32, + | ^^^^ + +error: fact input field `bal_ance` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:15:5 + | +15 | bal_ance: u64, + | ^^^^^^^^ + +error: fact input field `grant` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:16:5 + | +16 | grant: u32, + | ^^^^^ + +error: fact input field `pay_out` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:17:5 + | +17 | pay_out: u32, + | ^^^^^^^ + +error: fact input field `CREDIT` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:18:5 + | +18 | CREDIT: u32, + | ^^^^^^ + +error: fact input field `debit` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:19:5 + | +19 | debit: u32, + | ^^^^^ + +error: fact input field `Pay-Out` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:20:22 + | +20 | #[serde(rename = "Pay-Out")] + | ^^^^^^^^^ + +error: fact input field `gold-` looks like an economy value: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_economy_fields.rs:22:43 + | +22 | #[serde(alias = "goldAmount", alias = "gold-")] + | ^^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr b/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr index 3707c3aac..80a24fe32 100644 --- a/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr +++ b/distributed_game_macros/tests/compile_fail/fact_identity_field.stderr @@ -1,4 +1,4 @@ -error: fact input field `principal` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards +error: fact input field `principal` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values --> tests/compile_fail/fact_identity_field.rs:7:5 | 7 | principal: String, diff --git a/distributed_game_macros/tests/compile_fail/fact_identity_variants.rs b/distributed_game_macros/tests/compile_fail/fact_identity_variants.rs new file mode 100644 index 000000000..c2c6e7f88 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_identity_variants.rs @@ -0,0 +1,15 @@ +use distributed_game::FactCommand; + +#[derive(serde::Serialize, FactCommand)] +#[fact(command = "crawler.record_kill")] +struct RecordKill { + mob: String, + UserId: String, + #[serde(rename = "owner-id")] + who: String, + ACCOUNT_ID: String, + #[serde(rename = "Experience")] + gained: u32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/fact_identity_variants.stderr b/distributed_game_macros/tests/compile_fail/fact_identity_variants.stderr new file mode 100644 index 000000000..d1c969bd5 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/fact_identity_variants.stderr @@ -0,0 +1,23 @@ +error: fact input field `UserId` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:7:5 + | +7 | UserId: String, + | ^^^^^^ + +error: fact input field `owner-id` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:8:22 + | +8 | #[serde(rename = "owner-id")] + | ^^^^^^^^^^ + +error: fact input field `ACCOUNT_ID` looks like identity: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:10:5 + | +10 | ACCOUNT_ID: String, + | ^^^^^^^^^^ + +error: fact input field `Experience` looks like a reward: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values + --> tests/compile_fail/fact_identity_variants.rs:11:22 + | +11 | #[serde(rename = "Experience")] + | ^^^^^^^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/fact_reward_field.rs b/distributed_game_macros/tests/compile_fail/fact_reward_field.rs index 20944d60b..33a404f30 100644 --- a/distributed_game_macros/tests/compile_fail/fact_reward_field.rs +++ b/distributed_game_macros/tests/compile_fail/fact_reward_field.rs @@ -5,7 +5,7 @@ use distributed_game::FactCommand; struct RecordKill { mob: String, #[serde(rename = "XP")] - amount: u32, + count: u32, } fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr b/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr index 4bf6b46a6..62e0d6d0c 100644 --- a/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr +++ b/distributed_game_macros/tests/compile_fail/fact_reward_field.stderr @@ -1,4 +1,4 @@ -error: fact input field `XP` looks like a reward: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards +error: fact input field `XP` looks like a reward: a zone fact reports what happened; the host binds the principal from the verified ticket and the command handler decides rewards and economy values --> tests/compile_fail/fact_reward_field.rs:7:22 | 7 | #[serde(rename = "XP")] diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.rs b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.rs new file mode 100644 index 000000000..3ae3ebf54 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.rs @@ -0,0 +1,21 @@ +use distributed_game::Replicated; + +struct Mob { + x: f32, + y: f32, +} + +#[derive(Replicated)] +#[replicated(kind = "mob", id = 2, of = Mob)] +struct MobView { + #[from] + x: f32, + #[from] + y: f32, + #[from] + tier: u8, +} + +fn main() { + let _ = Mob { x: 0.0, y: 0.0 }.x; +} diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.stderr b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.stderr new file mode 100644 index 000000000..ddf110407 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_missing_field.stderr @@ -0,0 +1,7 @@ +error[E0609]: no field `tier` on type `&Mob` + --> tests/compile_fail/replicated_from_missing_field.rs:16:5 + | +16 | tier: u8, + | ^^^^ unknown field + | + = note: available fields are: `x`, `y` diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.rs b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.rs new file mode 100644 index 000000000..17c670358 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.rs @@ -0,0 +1,11 @@ +use distributed_game::Replicated; + +#[derive(Replicated)] +#[replicated(kind = "mob", id = 2)] +struct MobView { + #[from] + x: f32, + y: f32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.stderr b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.stderr new file mode 100644 index 000000000..b17c524dd --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_needs_of.stderr @@ -0,0 +1,5 @@ +error: #[from] needs the entity type: #[replicated(.., of = Entity)] + --> tests/compile_fail/replicated_from_needs_of.rs:6:5 + | +6 | #[from] + | ^ diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.rs b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.rs new file mode 100644 index 000000000..9834d968d --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.rs @@ -0,0 +1,27 @@ +use distributed_game::Replicated; + +struct State { + x: f32, + y: f32, +} + +struct Player { + state: State, + hp: i32, +} + +#[derive(Replicated)] +#[replicated(kind = "player", id = 1, of = Player)] +struct PlayerView { + #[from(state.x)] + x: f32, + #[from(state.y)] + y: f32, + #[from] + hp: u8, +} + +fn main() { + let p = Player { state: State { x: 0.0, y: 0.0 }, hp: 1 }; + let _ = (p.state.x, p.state.y, p.hp); +} diff --git a/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.stderr b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.stderr new file mode 100644 index 000000000..c50c90e65 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/replicated_from_wrong_type.stderr @@ -0,0 +1,5 @@ +error[E0308]: mismatched types + --> tests/compile_fail/replicated_from_wrong_type.rs:20:5 + | +20 | #[from] + | ^ expected `u8`, found `i32` diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.rs b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.rs new file mode 100644 index 000000000..ecfdfcf84 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.rs @@ -0,0 +1,15 @@ +use distributed_game::TicketExtension; + +#[derive(serde::Serialize, serde::Deserialize)] +struct Extra { + team: u8, +} + +#[derive(serde::Serialize, serde::Deserialize, TicketExtension)] +struct MatchClaims { + match_id: String, + #[serde(flatten)] + extra: Extra, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.stderr b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.stderr new file mode 100644 index 000000000..aa6a428c9 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_flatten.stderr @@ -0,0 +1,5 @@ +error: #[derive(TicketExtension)] does not support `flatten`; declare every claim as a field + --> tests/compile_fail/ticket_extension_flatten.rs:11:13 + | +11 | #[serde(flatten)] + | ^^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_identity.rs b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.rs new file mode 100644 index 000000000..f0390da9e --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.rs @@ -0,0 +1,12 @@ +use distributed_game::TicketExtension; + +// Identity only ever comes from the verified `sub`; amounts from handlers. +#[derive(serde::Serialize, serde::Deserialize, TicketExtension)] +struct MatchClaims { + match_id: String, + userId: String, + #[serde(rename = "Gold")] + purse: u32, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_identity.stderr b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.stderr new file mode 100644 index 000000000..dfcc205e9 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_identity.stderr @@ -0,0 +1,11 @@ +error: ticket claim field `userId` looks like identity: identity is only ever the verified `sub`, and rewards and economy values are decided by command handlers, never vouched for in a ticket + --> tests/compile_fail/ticket_extension_identity.rs:7:5 + | +7 | userId: String, + | ^^^^^^ + +error: ticket claim field `Gold` looks like an economy value: identity is only ever the verified `sub`, and rewards and economy values are decided by command handlers, never vouched for in a ticket + --> tests/compile_fail/ticket_extension_identity.rs:8:22 + | +8 | #[serde(rename = "Gold")] + | ^^^^^^ diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.rs b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.rs new file mode 100644 index 000000000..65f6992ab --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.rs @@ -0,0 +1,9 @@ +use distributed_game::TicketExtension; + +#[derive(serde::Serialize, serde::Deserialize, TicketExtension)] +#[serde(rename_all = "camelCase")] +struct MatchClaims { + match_id: String, +} + +fn main() {} diff --git a/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.stderr b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.stderr new file mode 100644 index 000000000..842049fc0 --- /dev/null +++ b/distributed_game_macros/tests/compile_fail/ticket_extension_rename_all.stderr @@ -0,0 +1,5 @@ +error: #[derive(TicketExtension)] does not support `rename_all`; rename fields individually + --> tests/compile_fail/ticket_extension_rename_all.rs:4:9 + | +4 | #[serde(rename_all = "camelCase")] + | ^^^^^^^^^^ diff --git a/distributed_macros/src/aggregate_command.rs b/distributed_macros/src/aggregate_command.rs new file mode 100644 index 000000000..f9bed2edd --- /dev/null +++ b/distributed_macros/src/aggregate_command.rs @@ -0,0 +1,630 @@ +//! `#[command("aggregate.verb", ..)]` on an aggregate method inside a +//! `#[sourced]` impl: the method is the command. `#[sourced]` strips the +//! attribute and emits a portable mount (`pub struct Verb`) whose generated +//! handler binds the principal from the verified session, loads (or +//! creates) the aggregate, calls the method, maps its error to `Rejected`, +//! builds the payload and commits with publication. +//! +//! ```ignore +//! #[command("crawler.record_kill", shard = principal, load = or_create, outcome = succeeded)] +//! pub fn record_kill(&mut self, #[principal] principal: &str, input: RecordKillInput) +//! -> Result { .. } +//! ``` +//! +//! Options: `shard = ` (required unless +//! `load = create`), `load = required | or_create | create`, +//! `outcome = succeeded | eventual | Succeeded

| Eventual

`, +//! `payload = path` (`fn(&Aggregate) -> P`; otherwise the method's `Ok` +//! value is the payload), `roles = ["player", ..]`, `guard = expr`, +//! `field = "graphql_field"`, `input = Type` (map the remaining method +//! parameters to the input's fields by name instead of passing the whole +//! input), `transition = path` (override the domain transition witness), +//! `mount = Ident` (name of the generated struct). + +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, quote_spanned}; +use syn::parse::{Parse, ParseStream}; +use syn::{Expr, FnArg, Ident, ImplItemFn, LitStr, Pat, Path, ReturnType, Token, Type}; + +#[derive(Clone, Copy, PartialEq)] +enum Load { + Required, + OrCreate, + Create, +} + +enum Outcome { + /// `succeeded` / `eventual`: the payload type is the method's `Ok` type. + Keyword { eventual: bool, span: Span }, + /// `Succeeded

` / `Eventual

`. + Type { ty: Box, eventual: bool }, +} + +struct Args { + name: LitStr, + shard: Option, + load: Load, + outcome: Outcome, + payload: Option, + roles: Option>, + guard: Option, + field: Option, + input: Option, + transition: Option, + mount: Option, +} + +impl Parse for Args { + fn parse(input: ParseStream<'_>) -> syn::Result { + let name: LitStr = input.parse().map_err(|_| { + input.error("#[command] on an aggregate method starts with the command name: #[command(\"aggregate.verb\", ..)]") + })?; + validate_name(&name)?; + let mut shard = None; + let mut load = None; + let mut outcome = None; + let mut payload = None; + let mut roles = None; + let mut guard = None; + let mut field = None; + let mut input_ty = None; + let mut transition = None; + let mut mount = None; + let mut seen = std::collections::HashSet::new(); + while !input.is_empty() { + input.parse::()?; + if input.is_empty() { + break; + } + let key: Ident = input.parse()?; + if !seen.insert(key.to_string()) { + return Err(syn::Error::new( + key.span(), + format!("duplicate #[command] option `{key}`"), + )); + } + input.parse::()?; + match key.to_string().as_str() { + "shard" => shard = Some(input.parse::()?), + "load" => { + let value: Ident = input.parse()?; + load = Some(match value.to_string().as_str() { + "required" => Load::Required, + "or_create" => Load::OrCreate, + "create" => Load::Create, + other => { + return Err(syn::Error::new( + value.span(), + format!("load must be `required`, `or_create` or `create`, not `{other}`"), + )) + } + }); + } + "outcome" => { + let ty: Type = input.parse()?; + outcome = Some(parse_outcome(ty)?); + } + "payload" => payload = Some(input.parse::()?), + "roles" => { + let expr: Expr = input.parse()?; + roles = Some(role_list(&expr)?); + } + "guard" => guard = Some(input.parse::()?), + "field" => field = Some(input.parse::()?), + "input" => input_ty = Some(input.parse::()?), + "transition" => transition = Some(input.parse::()?), + "mount" => mount = Some(input.parse::()?), + other => { + return Err(syn::Error::new( + key.span(), + format!( + "unknown #[command] option `{other}`; expected `shard`, `load`, `outcome`, `payload`, `roles`, `guard`, `field`, `input`, `transition` or `mount`" + ), + )) + } + } + } + let load = load.ok_or_else(|| { + syn::Error::new( + name.span(), + "#[command] needs `load = required | or_create | create`", + ) + })?; + let outcome = outcome.ok_or_else(|| { + syn::Error::new( + name.span(), + "#[command] needs `outcome = succeeded | eventual` (or `Succeeded

` / `Eventual

`)", + ) + })?; + if load != Load::Create && shard.is_none() { + return Err(syn::Error::new( + name.span(), + "#[command] needs `shard = ` to load the aggregate", + )); + } + Ok(Self { + name, + shard, + load, + outcome, + payload, + roles, + guard, + field, + input: input_ty, + transition, + mount, + }) + } +} + +fn validate_name(name: &LitStr) -> syn::Result<()> { + let value = name.value(); + let segments: Vec<&str> = value.split('.').collect(); + if segments.len() < 2 + || segments.iter().any(|segment| { + segment.is_empty() + || !segment + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') + }) + { + return Err(syn::Error::new( + name.span(), + "command name must be 2+ dot-separated snake_case segments (e.g. \"character.equip\")", + )); + } + Ok(()) +} + +fn parse_outcome(ty: Type) -> syn::Result { + let Type::Path(path) = &ty else { + return Err(syn::Error::new_spanned( + &ty, + "outcome must be `succeeded`, `eventual`, `Succeeded

` or `Eventual

`", + )); + }; + let last = path.path.segments.last().expect("non-empty path"); + let ident = last.ident.to_string(); + let bare = path.path.segments.len() == 1 && last.arguments.is_empty(); + match ident.as_str() { + "succeeded" | "eventual" if bare => Ok(Outcome::Keyword { + eventual: ident == "eventual", + span: last.ident.span(), + }), + "Succeeded" | "Eventual" if !last.arguments.is_empty() => Ok(Outcome::Type { + eventual: ident == "Eventual", + ty: Box::new(ty.clone()), + }), + _ => Err(syn::Error::new_spanned( + &ty, + "outcome must be `succeeded`, `eventual`, `Succeeded

` or `Eventual

`", + )), + } +} + +fn role_list(expr: &Expr) -> syn::Result> { + let Expr::Array(array) = expr else { + return Err(syn::Error::new_spanned( + expr, + "roles must be an array of string literals, e.g. [\"player\"]", + )); + }; + let mut roles = Vec::new(); + for elem in &array.elems { + match elem { + Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(value), + .. + }) if !value.value().trim().is_empty() => roles.push(value.clone()), + other => { + return Err(syn::Error::new_spanned( + other, + "roles entries must be non-empty string literals", + )) + } + } + } + if roles.is_empty() { + return Err(syn::Error::new_spanned( + expr, + "roles must name at least one role (omit `roles` for an internal command)", + )); + } + Ok(roles) +} + +/// One parameter after `self`. +struct Param { + name: Ident, + ty: Type, +} + +/// A `#[command]` aggregate method, collected by `#[sourced]`. +pub(crate) struct CommandMethod { + args: Args, + method: Ident, + principal: Param, + /// Whole-input parameter (no `input = T`), or the parameters mapped to + /// input fields by name. + params: Vec, + /// Every parameter name after `self`, in declaration order. + order: Vec, + ok_type: Option, +} + +impl CommandMethod { + pub(crate) fn method(&self) -> &Ident { + &self.method + } +} + +/// Strip `#[command(..)]` (and `#[principal]`) from `method`; `None` when it +/// is not a command. +pub(crate) fn take(method: &mut ImplItemFn) -> syn::Result> { + let Some(index) = method + .attrs + .iter() + .position(|attr| attr.path().is_ident("command")) + else { + return Ok(None); + }; + let attr = method.attrs.remove(index); + let args: Args = attr.parse_args()?; + if method.sig.asyncness.is_some() { + return Err(syn::Error::new_spanned( + method.sig.asyncness, + "#[command] aggregate methods are synchronous decisions; the generated handler does the I/O", + )); + } + if !matches!( + method.sig.inputs.first(), + Some(FnArg::Receiver(receiver)) if receiver.reference.is_some() && receiver.mutability.is_some() + ) { + return Err(syn::Error::new_spanned( + &method.sig, + "#[command] methods take `&mut self`", + )); + } + let mut principal = None; + let mut params = Vec::new(); + let mut order = Vec::new(); + for arg in method.sig.inputs.iter_mut().skip(1) { + let FnArg::Typed(typed) = arg else { continue }; + let Pat::Ident(pat) = &*typed.pat else { + return Err(syn::Error::new_spanned( + &typed.pat, + "#[command] parameters must be plain identifiers", + )); + }; + let is_principal = typed + .attrs + .iter() + .any(|attr| attr.path().is_ident("principal")); + typed + .attrs + .retain(|attr| !attr.path().is_ident("principal")); + let param = Param { + name: pat.ident.clone(), + ty: (*typed.ty).clone(), + }; + order.push(param.name.clone()); + if is_principal { + if principal.is_some() { + return Err(syn::Error::new_spanned( + &typed.pat, + "only one #[principal] parameter", + )); + } + principal = Some(param); + } else { + params.push(param); + } + } + let principal = principal.ok_or_else(|| { + syn::Error::new_spanned( + &method.sig.inputs, + "a #[command] method needs a `#[principal]` parameter (`&str` or `String`): the actor comes from the verified session, never from the input", + ) + })?; + if args.input.is_none() && params.len() != 1 { + return Err(syn::Error::new_spanned( + &method.sig.inputs, + "a #[command] method takes `#[principal]` and one input parameter; with `input = Type` its other parameters are the input's fields by name", + )); + } + let ok_type = match &method.sig.output { + ReturnType::Type(_, ty) => result_ok_type(ty), + ReturnType::Default => None, + }; + if ok_type.is_none() { + return Err(syn::Error::new_spanned( + &method.sig, + "#[command] methods return `Result` (`E: Display` becomes the rejection; `T` is the payload unless `payload = ..` is set)", + )); + } + if let (Outcome::Keyword { span, .. }, Some(_)) = (&args.outcome, &args.payload) { + return Err(syn::Error::new( + *span, + "with `payload = ..` name the payload type: `outcome = Succeeded

` or `Eventual

`", + )); + } + Ok(Some(CommandMethod { + args, + method: method.sig.ident.clone(), + principal, + params, + order, + ok_type, + })) +} + +/// `T` of `Result` (any path ending in `Result` with two arguments, or +/// one for an alias like `SourcedResult`); `()` for a bare alias. +fn result_ok_type(ty: &Type) -> Option { + let Type::Path(path) = ty else { return None }; + let last = path.path.segments.last()?; + if !last.ident.to_string().ends_with("Result") { + return None; + } + match &last.arguments { + syn::PathArguments::AngleBracketed(args) => match args.args.first()? { + syn::GenericArgument::Type(ok) => Some(ok.clone()), + _ => None, + }, + syn::PathArguments::None => Some(syn::parse_quote!(())), + syn::PathArguments::Parenthesized(_) => None, + } +} + +fn is_reference(ty: &Type) -> bool { + matches!(ty, Type::Reference(_)) +} + +fn pascal(segment: &str) -> String { + segment + .split('_') + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } + }) + .collect() +} + +/// The mount, its handler and its registration. `transition` is the domain +/// transition witness `#[sourced]` discovered for the method, if any. +pub(crate) fn expand( + command: &CommandMethod, + aggregate: &Type, + transition: Option, + framework: &TokenStream, +) -> syn::Result { + let fw = framework; + let args = &command.args; + let name = &args.name; + let name_value = name.value(); + let last = name_value.rsplit('.').next().expect("validated"); + let mount = args + .mount + .clone() + .unwrap_or_else(|| format_ident!("{}", pascal(last), span = name.span())); + let method = &command.method; + let principal = &command.principal; + let principal_name = principal.name.to_string(); + + let input_ty: Type = match &args.input { + Some(ty) => ty.clone(), + None => command.params[0].ty.clone(), + }; + + // Payload type. + let payload_ty: Type = match &args.outcome { + Outcome::Type { ty, .. } => { + let Type::Path(path) = &**ty else { + unreachable!() + }; + let last = path.path.segments.last().expect("path"); + match &last.arguments { + syn::PathArguments::AngleBracketed(a) => match a.args.first() { + Some(syn::GenericArgument::Type(p)) => p.clone(), + _ => return Err(syn::Error::new_spanned(ty, "expected `Succeeded

`")), + }, + _ => return Err(syn::Error::new_spanned(ty, "expected `Succeeded

`")), + } + } + Outcome::Keyword { .. } => command.ok_type.clone().expect("checked"), + }; + let eventual = match &args.outcome { + Outcome::Keyword { eventual, .. } | Outcome::Type { eventual, .. } => *eventual, + }; + let outcome_ty = if eventual { + quote!(#fw::command::Eventual<#payload_ty>) + } else { + quote!(#fw::command::Succeeded<#payload_ty>) + }; + let terminal = if eventual { + quote!(eventual) + } else { + quote!(succeeded) + }; + + // Shard. + let shard_by_principal = args.shard.as_ref().is_some_and(|s| *s == principal.name); + let shard_fn = match &args.shard { + Some(field) if !shard_by_principal => Some(quote! { + /// Aggregate id (shard) of a command input. + pub fn shard(input: &#input_ty) -> ::std::string::String { + ::std::string::ToString::to_string(&input.#field) + } + }), + _ => None, + }; + let shard_expr = if shard_by_principal { + quote!(::std::clone::Clone::clone(&__principal)) + } else { + quote!(Self::shard(&input)) + }; + let load = match args.load { + Load::Required => quote! { + let __id = #shard_expr; + let mut __checkout = #fw::microsvc::require_loaded(ctx.get(&__id).await?, __id)?; + }, + Load::OrCreate => quote! { + let __id = #shard_expr; + let mut __checkout = match ctx.get(&__id).await? { + ::std::option::Option::Some(checkout) => checkout, + ::std::option::Option::None => ctx.create(), + }; + }, + Load::Create => quote! { + let mut __checkout = ctx.create(); + }, + }; + + // Call arguments. + let principal_arg = if is_reference(&principal.ty) { + quote!(&__principal) + } else { + quote!(::std::convert::Into::into(::std::clone::Clone::clone( + &__principal + ))) + }; + let call_args: Vec = if args.input.is_some() { + command + .params + .iter() + .map(|param| { + let field = ¶m.name; + if is_reference(¶m.ty) { + quote_spanned!(field.span()=> &input.#field) + } else { + quote_spanned!(field.span()=> input.#field) + } + }) + .collect() + } else { + vec![quote!(input)] + }; + let payload = match &args.payload { + Some(payload) => quote! { + let _ = __out; + let __payload: #payload_ty = #payload(__checkout.aggregate()); + }, + None => quote! { let __payload: #payload_ty = __out; }, + }; + + // Compile-time principal check: no input field may be named like the + // principal parameter (the session is its only source). + let message = format!( + "#[command(\"{name_value}\")]: the input has a field named `{principal_name}`, the #[principal] parameter; the principal comes from the verified session only, so it can never be an input field" + ); + let principal_check = quote_spanned! {principal.name.span()=> + const _: () = { + if #fw::command::__input_has_field( + <#input_ty as #fw::command::CommandInputType>::FIELD_NAMES, + #principal_name, + ) { + ::core::panic!(#message); + } + }; + }; + + let transition = match &args.transition { + Some(path) => Some(quote!(#path)), + None => transition, + }; + let registration = match &transition { + Some(transition) => quote! { + routes.command_transition::<#transition, #input_ty, #outcome_ty>(Self::COMMAND) + }, + None => quote! { + routes.typed_command(#fw::command::typed_command::<#input_ty, #outcome_ty>(Self::COMMAND)) + }, + }; + let field = args.field.as_ref().map(|field| quote!(.field_name(#field))); + let roles = args + .roles + .as_ref() + .map(|roles| quote!(.roles([#(#roles),*]))); + let finish = match &args.guard { + Some(guard) => quote!(.guarded(#guard, Self::handle)), + None => quote!(.handle(Self::handle)), + }; + let doc = format!( + "Mounts `{name_value}` (`{}::{method}`): principal from the verified session, then load, call, commit.", + quote!(#aggregate).to_string().replace(' ', "") + ); + + let args_in_order = order_args(command, &principal_arg, &call_args); + + Ok(quote! { + #[doc = #doc] + #[derive(Debug, Clone, Copy, Default)] + pub struct #mount; + + #principal_check + + impl #mount { + /// Command name. + pub const COMMAND: &'static str = #name; + + #shard_fn + + /// The generated handler (also usable from a hand-written mount). + pub async fn handle( + ctx: &#fw::microsvc::CausalCommandContext<'_, #aggregate>, + input: #input_ty, + ) -> ::std::result::Result< + #fw::command::PreparedCommand<#outcome_ty>, + #fw::microsvc::HandlerError, + > { + // The principal: the verified session only. + let __principal: ::std::string::String = ctx.user_id()?.to_owned(); + #load + let __out = __checkout + .aggregate_mut() + .#method(#(#args_in_order),*) + .map_err(|error| #fw::microsvc::HandlerError::Rejected(error.to_string()))?; + #payload + ctx.publish_events().commit(__checkout)?.#terminal(__payload) + } + } + + impl #fw::microsvc::PortableCommand for #mount + where + D: #fw::microsvc::CausalRouteDependencies + + ::std::marker::Send + + ::std::marker::Sync + + 'static, + { + fn install(self, routes: #fw::microsvc::Routes) -> #fw::microsvc::Routes { + #registration + #field + #roles + #finish + } + } + }) +} + +/// Arguments in the method's declaration order. +fn order_args( + command: &CommandMethod, + principal_arg: &TokenStream, + call_args: &[TokenStream], +) -> Vec { + let mut out = Vec::new(); + let mut rest = call_args.iter(); + for name in &command.order { + if *name == command.principal.name { + out.push(principal_arg.clone()); + } else if let Some(arg) = rest.next() { + out.push(arg.clone()); + } + } + out +} diff --git a/distributed_macros/src/command.rs b/distributed_macros/src/command.rs index 46fd280b0..6126d1db8 100644 --- a/distributed_macros/src/command.rs +++ b/distributed_macros/src/command.rs @@ -176,6 +176,17 @@ pub fn expand( item: proc_macro2::TokenStream, ) -> syn::Result { let framework = crate::shared::framework_path()?; + if let Ok(name) = syn::parse2::( + attr.clone() + .into_iter() + .take(1) + .collect::(), + ) { + return Err(syn::Error::new( + name.span(), + "`#[command(\"aggregate.verb\", ..)]` on an aggregate method must be inside a `#[sourced]` impl (the handler-function form is `#[command(id = \"..\", ..)]`)", + )); + } let args = syn::parse2::(attr)?; let function = syn::parse2::(item)?; let id = args.id.ok_or_else(|| { diff --git a/distributed_macros/src/command_types.rs b/distributed_macros/src/command_types.rs index 343e8a79e..39d0a83f4 100644 --- a/distributed_macros/src/command_types.rs +++ b/distributed_macros/src/command_types.rs @@ -149,6 +149,7 @@ fn expand( let mut field_tokens = Vec::new(); let mut effect_input_markers = Vec::new(); + let mut field_names: Vec = Vec::new(); for field in &fields.named { validate_serde_field_shape(&field.attrs, serde_direction)?; let field_name = field @@ -165,6 +166,11 @@ fn expand( .map(|rule| rule.apply(rust_field_name)) .unwrap_or_else(|| rust_field_name.to_string()) }); + for name in [rust_field_name.to_string(), field_name_str.clone()] { + if !field_names.contains(&name) { + field_names.push(name); + } + } let (type_name, nullable, list, item_nullable, nested) = map_type(&field.ty, field, &nested_trait, &method)?; let unsigned_integer = unsigned_integer_tokens(&field.ty, &framework); @@ -221,8 +227,13 @@ fn expand( } let type_name_str = name.to_string(); + let field_names_const = matches!(serde_direction, SerdeDirection::Deserialize).then(|| { + quote! { const FIELD_NAMES: &'static [&'static str] = &[#(#field_names),*]; } + }); Ok(quote! { impl #trait_path for #name { + #field_names_const + fn #method() -> #type_def { #type_def::new( #type_name_str, diff --git a/distributed_macros/src/lib.rs b/distributed_macros/src/lib.rs index bd24f3f4a..9652fc597 100644 --- a/distributed_macros/src/lib.rs +++ b/distributed_macros/src/lib.rs @@ -1,4 +1,5 @@ mod aggregate; +mod aggregate_command; mod application; mod command; mod command_input_defaults; @@ -20,7 +21,14 @@ use proc_macro::TokenStream; use syn::DeriveInput; /// Generate one typed command's portable contract and optional executable -/// mount from the same handler declaration. +/// mount from the same handler declaration (`#[command(id = "..", ..)]` on +/// an async handler function). +/// +/// The aggregate-method form, `#[command("aggregate.verb", shard = .., +/// load = .., outcome = ..)]` on a method inside a `#[sourced]` impl, is +/// handled by `#[sourced]` (see the `aggregate_command` module docs): it +/// generates the handler, binding the `#[principal]` parameter from the +/// verified session, and the portable mount. #[proc_macro_attribute] pub fn command(attr: TokenStream, item: TokenStream) -> TokenStream { command::expand(attr.into(), item.into()) diff --git a/distributed_macros/src/sourced.rs b/distributed_macros/src/sourced.rs index 3e16ea43e..4e1e4c10c 100644 --- a/distributed_macros/src/sourced.rs +++ b/distributed_macros/src/sourced.rs @@ -715,8 +715,13 @@ pub(crate) fn expand_sourced(attr: TokenStream2, item: TokenStream2) -> syn::Res let mut seen_variants: std::collections::HashMap = std::collections::HashMap::new(); + let mut commands = Vec::new(); for item in &mut impl_block.items { if let syn::ImplItem::Fn(method) = item { + if let Some(command) = crate::aggregate_command::take(method)? { + commands.push(command); + continue; + } match find_and_remove_event_attr(&mut method.attrs) { Ok(Some(event_attr)) => { // Event methods are replayed as `self.method(...)`, so they @@ -863,8 +868,26 @@ pub(crate) fn expand_sourced(attr: TokenStream2, item: TokenStream2) -> syn::Res } } + let command_methods: std::collections::BTreeSet = commands + .iter() + .map(|command| command.method().to_string()) + .collect(); let domain_command_transitions = - discover_domain_command_transitions(&impl_block, &event_methods); + discover_domain_command_transitions(&impl_block, &event_methods, &command_methods); + let self_ty = (*impl_block.self_ty).clone(); + let command_mounts = commands + .iter() + .map(|command| { + let witness = domain_command_transitions + .iter() + .find(|transition| transition.method_name == *command.method()) + .map(|transition| { + let ty = method_name_to_type_ident(&transition.method_name); + quote!(domain_commands::#ty) + }); + crate::aggregate_command::expand(command, &self_ty, witness, &framework) + }) + .collect::>>()?; let deletion_identity = if uses_deletion_identity { let identity = format_ident!("{struct_name}DomainIdentity"); @@ -1065,6 +1088,7 @@ pub(crate) fn expand_sourced(attr: TokenStream2, item: TokenStream2) -> syn::Res #(#generated_domain_types)* #deletion_identity #domain_commands_module + #(#command_mounts)* #upcaster_wrappers #aggregate_impl }; @@ -1166,6 +1190,7 @@ fn ungroup_expr(expression: &Expr) -> &Expr { fn discover_domain_command_transitions( impl_block: &ItemImpl, event_methods: &[EventMethodInfo], + command_methods: &std::collections::BTreeSet, ) -> Vec { let recorders: std::collections::BTreeMap = event_methods .iter() @@ -1198,7 +1223,9 @@ fn discover_domain_command_transitions( .collect::>(); let mut transitions = Vec::new(); for (name, method) in &methods { - if !matches!(method.vis, syn::Visibility::Public(_)) { + // Public methods are transitions; so is every `#[command]` method + // (its generated mount needs the witness whatever its visibility). + if !matches!(method.vis, syn::Visibility::Public(_)) && !command_methods.contains(name) { continue; } diff --git a/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.rs b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.rs new file mode 100644 index 000000000..65412335f --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.rs @@ -0,0 +1,26 @@ +use distributed::{sourced, Entity}; +use serde::Deserialize; + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct EquipInput { + pub hero_id: String, + pub owner: String, +} + +#[derive(Default)] +pub struct Hero { + entity: Entity, +} + +#[sourced(entity, aggregate_type = "hero")] +impl Hero { + #[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] + pub fn equip(&mut self, input: EquipInput) -> Result<(), String> { + let _ = input.owner; + Ok(()) + } +} + +fn main() { + let _ = Hero::default().entity; +} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.stderr b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.stderr new file mode 100644 index 000000000..fa6fbf4da --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_missing_principal.stderr @@ -0,0 +1,5 @@ +error: a #[command] method needs a `#[principal]` parameter (`&str` or `String`): the actor comes from the verified session, never from the input + --> tests/compile_fail/aggregate_command_missing_principal.rs:18:18 + | +18 | pub fn equip(&mut self, input: EquipInput) -> Result<(), String> { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.rs b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.rs new file mode 100644 index 000000000..9e6eb6354 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.rs @@ -0,0 +1,11 @@ +use distributed::command; + +pub struct Hero; + +#[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] +pub fn equip(actor: &str) -> Result<(), String> { + let _ = actor; + Ok(()) +} + +fn main() {} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.stderr b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.stderr new file mode 100644 index 000000000..f7589f842 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_outside_sourced.stderr @@ -0,0 +1,5 @@ +error: `#[command("aggregate.verb", ..)]` on an aggregate method must be inside a `#[sourced]` impl (the handler-function form is `#[command(id = "..", ..)]`) + --> tests/compile_fail/aggregate_command_outside_sourced.rs:5:11 + | +5 | #[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] + | ^^^^^^^^^^^^ diff --git a/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.rs b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.rs new file mode 100644 index 000000000..7746646a6 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.rs @@ -0,0 +1,29 @@ +use distributed::{sourced, Entity}; +use serde::Deserialize; + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct EquipInput { + pub hero_id: String, +} + +#[derive(Default)] +pub struct Hero { + entity: Entity, +} + +fn payload(_hero: &Hero) -> u32 { + 0 +} + +#[sourced(entity, aggregate_type = "hero")] +impl Hero { + #[command("hero.equip", shard = hero_id, load = required, outcome = eventual, payload = payload)] + pub fn equip(&mut self, #[principal] actor: &str, input: EquipInput) -> Result<(), String> { + let _ = (actor, input); + Ok(()) + } +} + +fn main() { + let _ = (Hero::default().entity, payload); +} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.stderr b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.stderr new file mode 100644 index 000000000..af6ae6a9e --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_payload_needs_type.stderr @@ -0,0 +1,5 @@ +error: with `payload = ..` name the payload type: `outcome = Succeeded

` or `Eventual

` + --> tests/compile_fail/aggregate_command_payload_needs_type.rs:20:73 + | +20 | #[command("hero.equip", shard = hero_id, load = required, outcome = eventual, payload = payload)] + | ^^^^^^^^ diff --git a/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.rs b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.rs new file mode 100644 index 000000000..73368d33e --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.rs @@ -0,0 +1,35 @@ +use distributed::{sourced, Entity}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct EquipInput { + pub hero_id: String, + pub actor: String, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct Equipped { + pub hero_id: String, +} + +#[derive(Default)] +pub struct Hero { + entity: Entity, +} + +#[sourced(entity, aggregate_type = "hero")] +impl Hero { + #[command("hero.equip", shard = hero_id, load = required, outcome = succeeded)] + pub fn equip(&mut self, #[principal] actor: &str, input: EquipInput) -> Result { + let _ = (actor, input.actor); + self.touched().map_err(|e| e.to_string())?; + Ok(Equipped { hero_id: input.hero_id }) + } + + #[event("hero.touched")] + fn touched(&mut self) {} +} + +fn main() { + let _ = Hero::default().entity; +} diff --git a/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.stderr b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.stderr new file mode 100644 index 000000000..87d11dc54 --- /dev/null +++ b/distributed_macros/tests/compile_fail/aggregate_command_principal_input_field.stderr @@ -0,0 +1,5 @@ +error[E0080]: evaluation panicked: #[command("hero.equip")]: the input has a field named `actor`, the #[principal] parameter; the principal comes from the verified session only, so it can never be an input field + --> tests/compile_fail/aggregate_command_principal_input_field.rs:23:42 + | +23 | pub fn equip(&mut self, #[principal] actor: &str, input: EquipInput) -> Result { + | ^^^^^ evaluation of `_` failed here diff --git a/docs/application-composition.md b/docs/application-composition.md index 884d1b5e6..28f47ce4c 100644 --- a/docs/application-composition.md +++ b/docs/application-composition.md @@ -4,7 +4,7 @@ **Goal:** Make the same packages easy to run as a monolith or as microservices, with persistence/locks/transports as a separate, boring plane. **Non-goal:** Back-compat aliases or dual public APIs for the same concept. -Related pain today: `tests/e2e-ui/crates/service/src/service.rs` (~projection catalog, surfaces, command walls) and `tests/e2e-ui/crates/runner/src/main.rs` (SQLite/Postgres × API/outbox/consumer wiring). +Motivating pain (now resolved): e2e-ui once had a single `service.rs` holding the projection catalog, surfaces and command walls, and a runner that hand-wired SQLite/Postgres × API/outbox/consumer. Today the composition root is `tests/e2e-ui/crates/service/src/application.rs` (surfaces, module inventory), infrastructure is `host.rs`, each bounded context is a module under `modules/` (listed by `modules/compose.rs`), and `crates/runner/src/main.rs` is a short entrypoint. --- @@ -187,7 +187,7 @@ Rough size: **~100–150 lines of product wiring** for logical app + a tiny depl 3. **Runtime::{sqlite, postgres, in_memory}** — pair repo + locks + bus. 4. **ProcessRole** — Full / CommandWriter / EventualProjector / QueryApi spawn policy. 5. **Atomic mount checks** — fail closed if Atomic is mounted without write path. -6. **Collapse e2e-ui `service.rs` + `runner`** onto the new APIs (no dual/legacy surface). +6. **Collapse e2e-ui `service.rs` + `runner`** onto the new APIs (no dual/legacy surface). Done: see `application.rs`, `host.rs` and `modules/` in `tests/e2e-ui/crates/service/src/`. No second application-owned optimism map: projection mutation IR is the source of cache effects. diff --git a/docs/coordination-primitives.md b/docs/coordination-primitives.md index 608a31b5c..fcb4ce0f9 100644 --- a/docs/coordination-primitives.md +++ b/docs/coordination-primitives.md @@ -10,6 +10,7 @@ twice. |------|-----------|--------| | Change state in two cells with rollback | Saga / process manager | `distributed::saga` | | Run something at a time in the future, across restarts | `OutboxMessage::deliver_at` | `distributed::outbox` | +| A cell schedules its own future or recurring work | Re-armable cell alarm | `distributed::cell_host` | | One owner per key, with stale writers rejected | Lease with fencing token | `distributed::lease` | | Deterministic time in tests | `Clock`, `ManualClock` | `distributed::clock` | @@ -45,6 +46,114 @@ retry) resets availability to now, so a timer that failed to publish is retried promptly instead of being re-delayed. `OutboxMessage.deliver_at` is `#[serde(default)]`. +## Schedules: an aggregate schedules its own work + +`deliver_at` leaves the aggregate and needs a consumer to send a command +back. When an aggregate must act on itself later (a matchmaking tick, an +invite expiry, a daily reset, a lease expiry), a causal command handler +**schedules** it with its events (`distributed::schedule`): + +```rust +// on the route: what the command needs from its host +routes.typed_command(start).requires(Capability::Scheduling).requires(Capability::Clock) + +// in the handler +let now = ctx.now()?; // the host clock +ctx.schedule(Schedule::after("tick", Duration::from_secs(5), "pool.tick", &TickInput { pool })?)?; +ctx.cancel_schedule("invite-expiry")?; + +// in the schedule's own handler: re-schedule from the fired due time (no drift) +let due = ctx.scheduled_due().expect("fired by a schedule") + period; +ctx.schedule(Schedule::at("tick", due, "pool.tick", &input)?)?; +``` + +- Schedule rows are part of `CommitBatch` (`batch.schedules`) and commit in + the **same transaction** as the command's events, ledger receipt and + outbox rows, on every backend: a cell's own `cell_alarms` table (migration + 11), the in-memory repository's per-stream table, SQL's + `aggregate_schedules` (migration 12, keyed by the owning stream). A failed + command schedules nothing. The batch must commit exactly one aggregate, + which owns the schedule. `Schedule::after` resolves against the backend's + clock at commit (cell clock, injected `Clock`, database clock). +- One row per (owner, name). Scheduling a name again replaces it and bumps + its `generation`; `cancel_schedule` disarms it. +- When due, the host dispatches the command **into the owner** through its + command ledger, with a command id derived from `(name, generation)` and + the trusted session from `schedule_session` (role `schedule`, plus claims + only the host sets). Guard schedule-only commands with `schedule_guard` + so callers cannot invoke them. A crash after the handler committed but + before the schedule settled replays the receipt; the handler never runs + twice for one arm. +- Rejections disarm the schedule (reported as `failed`). Retryable failures + back off from 1 s, doubling, up to 5 min (`retried`). At most 32 schedules + fire per wake, 16 schedule writes per command and 1,024 armed schedules + per aggregate. + +**Capabilities.** Backends declare what they provide (`Capabilities`): + +| Backend | Stores schedules | Dispatches them (`Scheduling`) | Clock (`Clock`) | +|---|---|---|---| +| Aggregate cell (celld; reference host) | `cell_alarms` | yes: the Durable Object alarm; `fire_due_schedules` on the reference host | DO clock / store clock | +| In-memory repository | per-stream table | with a schedule worker attached (`with_schedule_worker`) | injected `Clock` (`with_clock`) | +| SQL (Postgres, SQLite) | `aggregate_schedules` | with a schedule worker attached (`with_schedule_worker`) | database clock | + +A route declares what it needs with `.requires(..)`; registering it on a +backend without it panics with the missing capabilities while the service is +built, never at the first commit. `ctx.now()` answers only on routes that +declared `Clock` (the dispatcher stamps the host clock into the session and +removes any caller value); `ctx.schedule` refuses unless the route declared +`Scheduling`. + +**Shared stores: the schedule worker** (`distributed::schedule::worker`, +`graphql` feature). SQL and in-memory repositories store schedules for +every aggregate in one table, so a worker process fires them: + +```rust +let (repo, schedules) = PostgresRepository::connect_and_migrate(url) + .await? + .with_schedule_worker(ScheduleWorkerConfig::default()); // repo now declares Scheduling +let service = Arc::new(build_service(repo.clone())); +let host: Arc = Arc::new(LocalCommandHost::new(Arc::clone(&service))); +let worker = schedules.spawn(host); // worker.stop().await on shutdown +``` + +- Each pass claims up to 32 due, unclaimed rows in one statement + (`UPDATE … WHERE … IN (SELECT … FOR UPDATE SKIP LOCKED) RETURNING`; + SQLite serializes writers instead), marking them `claimed_until_ms` + (default 30 s ahead). Concurrent workers in several processes get + disjoint rows. +- Each claimed arm is dispatched through the `CommandHost` with + `schedule_session` and a command id derived from the owning stream, the + name and the generation (one ledger serves every aggregate here, so the + owner is part of the id). A worker that crashes after the command + committed leaves the claim to lapse; the next worker re-dispatches the + same id and the ledger replays it, so the arm fires once. +- Settlement touches only the generation that fired, like on celld. +- The scheduled command finds its aggregate through its input; the handler + that called `ctx.schedule` supplies that input. + +**Latency.** On celld the Durable Object alarm wakes the cell at the due +time (a few milliseconds late) without any extra process. The schedule +worker fires a due row within one poll interval (`poll_interval`, default +250 ms) plus the dispatch, and only while a worker runs; in the test suite +with a 20 ms poll the observed delay past due was about 20–45 ms on SQLite. +Lower the interval for tighter timing at the cost of one claim query per +interval per worker. + +**celld.** A Durable Object has one alarm. The outbox watchdog +(`CelldOutbox`) and due schedules share it: every `setAlarm` targets the +earliest of the next schedule and the next drain, so a watchdog rearm never +postpones a schedule and a far schedule never postpones a drain. The +Worker's `alarm()` handler calls `AggregateCell::alarm(&env)`, which fires +due schedules, then drains the outbox and re-plans the wake. Commands that +schedule move the wake earlier right after they commit. + +**Reference host.** The in-process host (unit tests; durable cells run on +celld) has no timer. Give the store a `ManualClock` +(`CellStreamStore::with_clock`), advance it and call +`AggregateCell::fire_due_schedules()`: the same ledger and settlement path +runs deterministically. + ## Leases with fencing tokens ```rust @@ -123,6 +232,56 @@ Guarantees and obligations: the saga id and rejects a late action. - **`on_step_succeeded` is replayed,** so keep it deterministic. +### Fan-out steps + +A step can send commands to N cells chosen at run time, for example one +ticket reservation per party when a match starts. `action` returns +`SagaAction::FanOut(commands)` instead of a single command +(`SagaCommand` converts with `.into()`); `compensation` and +`on_step_succeeded` receive the member index (`0` for single steps). + +```rust +fn action(step: &str, saga_id: &str, d: &MatchStart) -> Result { + match step { + "reserve" => Ok(SagaAction::fan_out( + d.parties.iter().map(|p| SagaCommand::json(format!("party:{p}"), "ticket.reserve", &saga_id)) + .collect::, _>>()?, + )), + _ => Ok(SagaCommand::json("match", "match.confirm", &saga_id)?.into()), + } +} +fn compensation(step: &str, member: usize, saga_id: &str, d: &MatchStart) + -> Result, SagaError> { + match step { + "reserve" => { + SagaCommand::json(format!("party:{}", d.parties[member]), "ticket.release", &saga_id).map(Some) + } + _ => Ok(None), + } +} +``` + +- 1 to `MAX_SAGA_FAN_OUT` (64) members; an empty or larger fan-out is a + `Definition` error. Membership must be a deterministic function of the data. +- Every member gets its own command and timeout timer, both carrying + `saga-member` metadata. Command ids are + `type:saga:step:phase:attempt:member`, so retries dedupe per member. +- The step **succeeds** when every member replies `succeeded`. It + **settles** only after every member has replied or timed out, so the saga + knows which members applied. On any rejection or timeout the step fails and + only the members that applied, or timed out (outcome unknown), are + compensated, one at a time, highest index first. Members that rejected are + not compensated. +- A reply arriving after its member timed out is `Ignored`; the timeout + already settled that member and it will be compensated. +- Events: `saga.member_succeeded`, `saga.member_failed`, + `saga.fan_out_settled`. Replaying the stream reproduces the per-member + state (`Saga::fan_out`, `Saga::applied_members`). + +The tests are in `src/saga/fan_out_tests.rs` (N = 1..16, partial failure, +per-member timeouts and late replies, per-member compensation retries, +bounds, and replay from a repository). + ### Worked example `tests/saga_transfer/main.rs` moves value between wallets in two separate @@ -147,6 +306,13 @@ A test network delivers outbox rows whose `deliver_at` is due on a ```sh cargo test --features sqlite,postgres --test saga_transfer --test coordination_lease --test scheduled_outbox cargo test --features sqlite --lib -- outbox lease clock +cargo test --features workers-rs --lib cell_host +(cd tests/celld/alarm-worker && worker-build --release) && node tests/celld/alarms.mjs ``` +`tests/celld/alarms.mjs` owns its own `celld dev` process (no Docker; set +`CELLD_BIN` to choose the binary) and proves the alarm row commits atomically, +periodic re-arm at the alarm cadence, that the watchdog and domain alarms do +not starve each other, and exactly-once firing across a celld restart. + The Postgres cases skip when `DATABASE_URL` is unset (see `compose.yaml`). diff --git a/docs/gateway/metrics.md b/docs/gateway/metrics.md index 6fac87025..e7e5796a8 100644 --- a/docs/gateway/metrics.md +++ b/docs/gateway/metrics.md @@ -18,11 +18,14 @@ worker-build as described in [worker.md](worker.md): cargo test --no-default-features --features gateway-graphql-native,gateway-delivery,sqlite --test graphql_query_protocol separate_origin_and_client_savings -- --ignored --nocapture ``` -`tests/gateway-worker/artifacts/load-report.json` records the source revision and +`tests/gateway-worker/load-runtime.mjs` writes its report to +`tests/gateway-worker/artifacts/load-report.json` (uploaded by CI, not +checked in); [`load-report.json`](load-report.json) beside this page is a +recorded run. The report records the source revision and whether it was dirty, runtime/platform, repetitions, payload size and resource limits. Native and workerd each run disabled, coalescing-only, snapshot-only and -live-only modes. A barrier holds actual SQL until100 coalesced consumers join. -Snapshots report warmup separately before100 hits. Every mode holds100 live +live-only modes. A barrier holds actual SQL until 100 coalesced consumers join. +Snapshots report warmup separately before 100 hits. Every mode holds 100 live consumers during a real projection commit; result SQL, origin validation, steady producers and all ongoing/temporary origin WebSocket connections are measured. Different-subject controls prove private isolation. External writes, explicit diff --git a/docs/live-query-delivery.md b/docs/live-query-delivery.md index aa373488d..e4133f26b 100644 --- a/docs/live-query-delivery.md +++ b/docs/live-query-delivery.md @@ -102,3 +102,47 @@ projection stopped or provide a server ordering signal. This is a breaking v5 protocol change: `mode` replaces the ambiguous `supported` boolean. Upgrade server and generated-client runtime together. Old or unknown wire forms fail closed; applications do not need a polling or reload workaround. + +## Leaderboard roots + +Sorted-set leaderboards (`distributed::leaderboard`, Redis with the `redis` +feature) are exposed through the same surface by +`GraphqlEngineBuilder::leaderboard(LeaderboardSurface, store)`. For a surface +named `xp`, a role's grant (`LeaderboardGrant`) selects the roots: +`xpLeaderboard(first)`, `xpLeaderboardAround(radius)`, `xpMyRank` and, only +with `rank_any_member`, `xpRank(member)`. A board family +(`LeaderboardSurface::family`) adds `board: String!` to each. The viewer +roots take the member from the verified session claim (default `x-user-id`, +`LeaderboardSurface::member_claim`), never from an argument, and fail with +`FORBIDDEN` when the claim is absent. The same declaration feeds the static +SDL, the client manifest (`leaderboards`, omitted when empty so other +manifests and fingerprints are unchanged) and the generated +`leaderboards.ts` (typed documents and `@live` subscription documents). + +`xpLeaderboard`, `xpLeaderboardAround` and `xpMyRank` are also subscription +roots. They re-execute when the board's change key +(`leaderboard:{board}`) is published, which +`GraphqlEngine::leaderboard_change_listener` does when registered on the +`LeaderboardProjector` that writes the board (applied changes and rebuild +promotions; duplicates are silent). Pushes are hash-gated like SQL live +queries, and a failed execution is terminal. With the causal protocol on, +a leaderboard result carries an incomplete snapshot +(`recordsComplete: false`, `indexesComparable: false`) and live frames use +`snapshot` mode with `reset: true`: leaderboard rows are never normalized +into the replica cache or resumed from cursors. + +The listener is in-process. When the projector runs in another process +(a zone server projecting kills into Redis while a GraphQL service serves +the same boards), the change goes over the application's bus: + +- the projector's process registers `LeaderboardChangePublisher::listener()` + (feature `leaderboard-background`), which publishes a + `distributed.leaderboard.changed` event (`{ "board": "" }`) per + changed board, coalescing bursts and retrying until the bus accepts it; +- the GraphQL process runs `engine.leaderboard_change_handlers(group)` with + `bus.subscribe(..)` under a consumer group of its own, so every GraphQL + process receives every notice and republishes `leaderboard:{board}` on + its `ChangeHub`. + +A notice only names the board; the subscriber re-reads its store, so +duplicate or replayed notices push nothing new. diff --git a/examples/game/.gitignore b/examples/game/.gitignore index fdd3cf487..4688cb85f 100644 --- a/examples/game/.gitignore +++ b/examples/game/.gitignore @@ -1,2 +1,6 @@ -# Mob Pit / Leaderboard durable cells (run.sh default MOBPIT_DATA_DIR). +# Mob Pit / Leaderboard local files and celld dev state (run.sh default MOBPIT_DATA_DIR). /.data/ +# The Mob Pit cells worker build (worker-build) and celld state. +/cells/build/ +/cells/target/ +/cells/.celld/ diff --git a/examples/game/README.md b/examples/game/README.md index 8454a8223..bc3f1a98a 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -6,16 +6,16 @@ Small playable demos of the authoritative game runtime on one mechanic. **Dev only**: the demo server mints tickets for anonymous guests and lets any page change the network simulation. -| Demo | Run | Mechanic it demonstrates | -|---|---|---| -| **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | -| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player native SQLite cells (`cell-sqlite`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery | -| **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | -| **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | -| **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | -| **Endless Grid** | `./run.sh grid` | Seeded procedural world (terrain/biomes/props generated bit-identically in Rust and TS, nothing static replicated), 32 m cells as zones, observer connections to the 8 neighbours, handoff across borders **and hosts** (2 processes, hash ring, Redirect + transfer tickets), hibernation to SQLite and restore | -| **Mob Pit · SvelteKit** | `./run.sh sveltekit` | The same Mob Pit server behind a SvelteKit app: Auth.js session sign-in, admission tickets minted from the session (`createGameTicketHandler`), the game client and WASM step generated by the Vite plugin (`games`), a runes HUD (`useGame`) next to a plain canvas loop, and the settled kills as a business-side read model: Crawler domain events projected by `mobpit-service` into SQL and read through generated `@load @live` GraphQL (SSR, then live). See [SvelteKit](#sveltekit) | -| **Boss Ripple** | `./run.sh ripple` | Zone hierarchy realm › city › borough › hood (`DottedTopology`), per-tier tick rates (2/5/10/30 Hz), boss defeats propagating up as tier-up events (each hop on another host), parent broadcasts reaching players through observer connections | +| Demo | Run | Mechanic it demonstrates | Durable state | +|---|---|---|---| +| **Arena** | `./run.sh arena` | Fixed 30 Hz tick / 20 Hz patches, binary deltas, interest management (view ring, drones pop in/out), client prediction + reconciliation (the Rust movement step compiled to WebAssembly, bit-exact), interpolation of remote entities, live latency/jitter/loss | none (stateless: positions only live in the zone) | +| **Mob Pit** | `./run.sh mobpit` | Authoritative melee combat vs. 12 telegraphing mobs; kills are **non-blocking durable facts** committed exactly once into per-player celld cells (`cells/`, `celld dev` started by `run.sh`); "slow storage" toggle (tick stays flat, pending facts climb); `kill -9` crash recovery; presence cells fed by the runtime's player lifecycle facts (`/dev/mobpit/presence`) | celld: `crawler/{player}`, `presence/{player}`; crawler list = `player_stats` read model projected from the cells' events | +| **Leaderboard** | `./run.sh leaderboard` | Same server as Mob Pit: kills/XP sorted-set boards (in-memory or Redis) fed fire-and-forget with idempotent apply keys, top 10 pushed live, your rank + neighbours, rebuild from the cells' event history | celld (the Mob Pit cells); boards are a projection (memory or Redis) | +| **Shooting Gallery** | `./run.sh gallery` | Lag compensation: hitscan shots resolved with `ctx.rewind().hit_test_shot` at the shooter's view time (`client_tick` + interpolation delay in the payload), `max_rewind` clamp, per-shot compensation on/off, predicted vs confirmed hits | none (stateless: score is per session) | +| **The Freeze** | `./run.sh freeze` | World-version barrier (`WorldCoordinator::rollout`) across **two server processes** as an in-world "admin update": freeze → apply → resume with change notes, inputs dropped while frozen, Hard reload vs safe-room gating (`StalePolicy::SafeRoom`, `Zone::is_gate`) of old clients, aborted rollout (`freeze_ready` → `NotAcked`), facts/events stamped with the content version | celld: `gem-pouch/{player}`, `content-log/{zone}` | +| **Endless Grid** | `./run.sh grid` | Seeded procedural world (terrain/biomes/props generated bit-identically in Rust and TS, nothing static replicated), 32 m cells as zones, observer connections to the 8 neighbours, handoff across borders **and hosts** (2 processes, hash ring, Redirect + transfer tickets), hibernation to celld and restore | celld: `zone-lease/{zone}`, `zone-state/{zone}` (hibernation) | +| **Mob Pit · SvelteKit** | `REDIS_URL=… ./run.sh sveltekit` | The same Mob Pit server behind a SvelteKit app: Auth.js session sign-in, admission tickets minted from the session (`createGameTicketHandler`), the game client and WASM step generated by the Vite plugin (`games`), a runes HUD (`useGame`) next to a plain canvas loop, the settled kills as a business-side read model (Crawler domain events projected by `mobpit-service` into SQL, read through generated `@load @live` GraphQL), and the zone server's Redis XP board through GraphQL board roots, live across processes over the bus. See [SvelteKit](#sveltekit) | celld (the Mob Pit cells) + `mobpit-service`'s SQL read model + the Redis boards | +| **Boss Ripple** | `./run.sh ripple` | Zone hierarchy realm › city › borough › hood (`DottedTopology`), per-tier tick rates (2/5/10/30 Hz), boss defeats propagating up as tier-up events (each hop on another host), parent broadcasts reaching players through observer connections | celld: `zone-lease/{zone}`, `zone-state/{zone}` (hibernation) | `./run.sh smoke` runs every demo's smoke tests (and checks the generated clients are current, and drives the SvelteKit demo headlessly); `./run.sh gen` regenerates the clients; `./run.sh @@ -131,8 +131,12 @@ is about a client receiving content it was not built with. Each zone switch emits a typed `ContentApplied` fact (`freeze.content_applied`) and a stamped event; gems emit `GemCollected` facts (`freeze.gem_collected`, naming the gem spot — the gem's value is a -content-pack rule for the handler, not a zone-reported amount) — the admin -panel lists them with their stamp. Display names live on the primary host: a player joining Ember +content-pack rule for the handler, not a zone-reported amount). Both hosts +settle them in celld (`run.sh` starts `celld dev`; both processes share it +through `CELLD_URL` / `DISTRIBUTED_INTERNAL_SECRET`): gems in the player's +`gem-pouch/{principal}` cell, content switches in the zone's +`content-log/{zone}` cell. The admin panel lists the last settled facts with +their stamp. Display names live on the primary host: a player joining Ember Fields directly (`&zone=freeze-ember`) shows as a guest there, names carry over portals. @@ -146,8 +150,28 @@ examples/game/run.sh grid Two `game-demos grid` processes: host **A** on `GAME_DEMOS_PORT` (Vite proxies to it) and **B** on the next port, each listing the other with `--peer`, sharing one ticket secret (the admin secret for host-to-host calls -is derived from it) and keeping hibernation snapshots under one -`--state-dir` (`grid-A.sqlite3`, `grid-B.sqlite3`). +is derived from it) and one celld (`run.sh` starts `celld dev` with +`cells/`; `--celld-url` / `CELLD_URL` and `DISTRIBUTED_INTERNAL_SECRET` are +required, there is no local fallback): + +- **Ownership**: every zone claim is a celld `ZoneLease:{zone}` cell + (`CellZoneOwnership`); a host that stops renewing gets a + `zone.lease_expired` from the cell's alarm. +- **Hibernation**: every snapshot is a celld `ZoneState:{zone}` cell + (`CellZoneStateStore`), fenced by the lease token; a sleeping zone is a + sleeping Durable Object. Snapshots above 64 KiB go to celld's object + storage (R2 binding `ZONE_SNAPSHOTS`), the cell keeps their digest. +- **Zone status**: both cells publish their transitions (`zone.activated` + / `zone.released` / `zone.lease_expired`, `zone_state.saved` / + `zone_state.deleted`, each carrying the cell's small status, never the + snapshot bytes). celld's outbox Queue relays them to host A's + `/internal/celld-queue/relay`, which publishes onto the world's shared + projection bus (`--data-dir`, `run.sh` uses `.data/`); every host + projects the bus into its own zone status read model + (`distributed_game::world::zone_status`, `common/zone_status.rs`). + +The cluster smoke tests (`tests/grid_smoke.rs`, `tests/ripple_smoke.rs`) +and the headless tests start their own `celld dev` the same way. - **World.** Cell `(x, y)` is zone `grid..` (canonical integers, |x|,|y| ≤ 10⁶; the server uses `FlatTopology`, the dots are not a @@ -173,10 +197,13 @@ is derived from it) and keeping hibernation snapshots under one `cell × 32 + local` around a floating origin. - **Hibernation.** Nobody inside or observing → Idle → Dormant after 2 s → hibernated after 4 s more (`--hibernate-after-ms`): critters and markers - (E) go to SQLite and come back on the next activation; its hello says - `restored` and the page toasts. The minimap (`GET /dev/grid/map`, which - asks the peer for its cells) shows active / hibernated / cold cells per - host; borders are tinted by owning host. + (E) go to the zone's celld `ZoneState` cell and come back on the next + activation (on either host); its hello says + `restored` and the page toasts. The minimap (`GET /dev/grid/map`) shows + active / hibernated / cold cells per host from the zone status read model + (eventually consistent; it never opens a cell, since reading a never-used + celld cell creates it), with live tier and player counts asked from the + host running each active cell; borders are tinted by owning host. Controls: WASD / stick, Space/Shift sprint, E drops a marker. @@ -211,14 +238,20 @@ hood to fight there. ## SvelteKit ```sh -examples/game/run.sh sveltekit # → http://localhost:5573/ — sign in with any username +docker compose up -d redis # at the repo root: the leaderboard is the Redis board +REDIS_URL=redis://127.0.0.1:6379 examples/game/run.sh sveltekit +# → http://localhost:5573/ — sign in with any username ``` `sveltekit/` is Mob Pit as a SvelteKit app over two backends: the Mob Pit zone server (gameplay) and `mobpit-service` (the business-side read model). -The script starts `game-demos mobpit` (no `--dev`) on `GAME_DEMOS_PORT` -(default 2967 here; data in `.data/mobpit-sveltekit`) with a fresh ticket secret and -`--projection-bus sqlite:…/bus.db`, then runs `distributed dev server/`: the +The script starts `celld dev` with the Mob Pit cells (`cells/`; state in +`.data/mobpit-sveltekit/celld`), then `game-demos mobpit` (no `--dev`) on +`GAME_DEMOS_PORT` (default 2967 here; data in `.data/mobpit-sveltekit`) with +a fresh ticket secret, `--redis-url "$REDIS_URL"` (its kill boards) and +`--projection-bus sqlite:…/bus.db` (celld's outbox Queue relays the Crawler +domain events to it; the board change notices go there too), then runs +`distributed dev server/`: the application lifecycle generates the `$distributed` GraphQL client, starts `mobpit-service` on `MOBPIT_SERVICE_PORT` (default `GAME_DEMOS_PORT + 2`) and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: @@ -260,10 +293,26 @@ and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: `src/lib/pit/MyStats.graphql` (my `player_stats` row; `$playerId` is bound by `MyStats.graphql.bindings.js` to `sessionClaim('user', 'id')`, so the value is the verified session's id for SSR and in the browser, never a URL - or form value) and `src/lib/pit/Leaderboard.graphql` (top 5 by XP). The - root layout's `createDistributedSvelteKitServer` runs them during SSR, the - page hydrates without refetching, and the `graphql-transport-ws` - subscription pushes every later settlement — to every signed-in player. + or form value) and `src/lib/pit/Leaderboard.graphql` (display names from + `player_stats`). The root layout's `createDistributedSvelteKitServer` runs + them during SSR, the page hydrates without refetching, and the + `graphql-transport-ws` subscription pushes every later settlement — to + every signed-in player. +- **Leaderboard (Redis, through GraphQL).** The ranking is the zone + server's Redis XP board, served by `mobpit-service` as GraphQL board roots + (`xpLeaderboard`, `xpLeaderboardAround`, `xpMyRank` and the `kills…` + twins; `leaderboard_surfaces()`), and generated into `$distributed` as + `XpLeaderboardLeaderboard` & co. (`leaderboards.ts`). The page `load` + (`routes/pit/+page.server.ts`) runs it once with + `loadDistributedLeaderboard` for SSR; `Leaderboard.svelte` follows it with + `useDistributedLeaderboard` (`@hops-ops/distributed/sveltekit`), which + subscribes to the `@live` document over the client's WebSocket. Pushes + cross processes on the bus: the zone server's `LeaderboardProjector` + publishes `distributed.leaderboard.changed` for each changed board + (`LeaderboardChangePublisher`), and `mobpit-service` subscribes with + `GraphqlEngine::leaderboard_change_handlers` (`spawn_board_notices`). + Both processes use `REDIS_URL` and the same key prefix + (`MOBPIT_REDIS_PREFIX`, default `mobpit`). The rows come from the cells (see *Mob Pit*, step 5): the zone's `profile`/`board` pushes and the `--dev` leaderboard route are not used by this app (they remain for the vanilla client pages). @@ -280,7 +329,8 @@ and `vite dev` on `SVELTEKIT_PORT` (default 5573). What each piece replaces: `./run.sh smoke` runs `check:games`, then `distributed build server/` (the GraphQL client, `svelte-check`, `vite build` with the game codegen and WASM) and `tests/pit.test.mjs`: the production build (`node build`) against the -real zone server and `mobpit-service` in headless Chromium — sign-in, ticket +real zone server and `mobpit-service` (on the Redis in `REDIS_URL`, with a +per-run key prefix) in headless Chromium — sign-in, ticket rejection paths, an unauthenticated GraphQL request refused, join, keyboard movement, auto-hunt until a kill settles, the killer's stats and a second signed-in player's leaderboard updating live without a reload, the @@ -339,24 +389,34 @@ What the TypeScript still restates, and why it stays: ``` examples/game/ run.sh one-command dev + smoke runner + cells/ crate `game-demo-cells` (own workspace, wasm32, worker-build): the + demos' celld cells — Crawler + presence (Mob Pit), gem pouch + + content log (The Freeze), zone lease + zone state (multi-host + demos), and the `game-demo-outbox` Queue consumer (relay) server/ Rust crate `game_demos`, its own cargo workspace (with sim/, crawler/, service/); also the `distributed build|dev` project crawler/ crate `mobpit_crawler`: Crawler aggregate, RecordKill (XP rules), `CrawlerState` domain event (shared by the zone and the service) service/ crate `mobpit_service` (bin `mobpit-service`): player_stats read - model, modeled projector from the bus, GraphQL @live (Bearer only), - tests/projection.rs (cells → bus → table → live push, no mutations, 401) + model, modeled projector from the bus, the Redis boards as GraphQL + board roots (@live from bus notices), GraphQL @live (Bearer only), + tests/projection.rs (cells → bus → table → live push, board push + over bus notices, no mutations, 401) sim/ crate `game_sim`: the `#[predict]` steps (arena.rs movement + floor, mobpit.rs, ripple.rs inputs + steps, grid.rs world + walker step + `#[game_pure]` grid_cell / grid_sample); compiled natively for the server and to wasm32 for the browser src/main.rs `game-demos ` subcommands (+ hidden `predict-trace`) src/predict_trace.rs native traces of every step (recorded inputs) and pure function (arguments) - src/common/ shared: Demo trait, run/spawn, dev routes, guests + src/common/ shared: Demo trait, run/spawn, dev routes, guests; + celld.rs (CelldOptions: celld URL + internal secret), + relay.rs (authenticated /internal/celld-queue/relay → SqliteBus), + zone_status.rs (zone lease/snapshot status read model) + src/devcelld.rs `celld dev` harness for tests (own project, port, process tree) src/cluster.rs multi-host demos: --host-id/--peer, directory, ownership, snapshots src/arena/ Arena zone (`movement` = `game_sim::arena`) - src/mobpit/ Mob Pit zone, Crawler cells, fact sink + projection-bus drain, - leaderboards, dev routes + src/mobpit/ Mob Pit zone, fact sink to the Crawler/presence cells, crawler + directory projection, leaderboards, dev routes src/gallery/ Shooting Gallery zone (targets, rewind hit tests) src/freeze/ The Freeze zone, content packs, two-host world + admin routes src/grid/ Endless Grid zone (`world` = `game_sim::grid`), dev map routes @@ -395,7 +455,8 @@ examples/game/ arena-headless, mobpit-headless, leaderboard-headless, gallery-headless, freeze-headless, grid-headless, ripple-headless (+ helpers/demo-server.mjs, helpers/mobpit.mjs, - helpers/cluster.mjs for multi-host demos) + helpers/cluster.mjs for multi-host demos, helpers/celld.mjs for + a per-test `celld dev`) ``` The server crate is excluded from the root workspace (`Cargo.toml` @@ -427,17 +488,23 @@ after dying. Level `n` starts at `40·(n−1)²` XP. `emit_fact_for` only enqueues: the host binds the killer's **verified ticket principal**; the zone never awaits storage. Backpressure (256 unsettled facts) parks facts in a backlog retried next tick. -2. The zone's fact worker (`GameServer::with_fact_sink`) runs - `store::CrawlerFactSink`: optional artificial latency, then - `CellFactSink::commit` into the killer's own cell - (`$MOBPIT_DATA_DIR/Crawler%3A{principal}.sqlite3`, `AggregateCell::from_store` - over `CellStreamStore::open_sqlite`). The command id is +2. The zone's fact worker (`GameServer::with_fact_sink`, retrying until + settled) runs `store::MobPitFactSink`: optional artificial latency, then + `distributed_game::celld::CelldFactSink` posts the fact over the + internal path (`DISTRIBUTED_INTERNAL_SECRET`, principal and `FactStamp` + as trusted headers) to the killer's own celld cell, `crawler/{principal}` + (`cells/`, a workers-rs Durable Object per player). The command id is `sha256(service ‖ fact key)`, so the cell's command ledger turns any repeat - of the key into a replay of the stored receipt. -3. `crawler.rs`: `crawler.record_kill` takes the principal from the trusted - session (`x-user-id`) and **decides the XP from the tier**; the input has - no identity or XP field and unknown fields are rejected. Event, ledger - receipt and snapshot commit in one SQLite transaction (`synchronous=FULL`). + of the key into a replay of the stored receipt. Nothing else is written + per kill: there is no index cell every player passes through (celld + cannot enumerate cells, and such a cell would be a global hot spot); the + crawler list is a read model (step 5). celld down: the fact stays queued + in the zone (backpressure at capacity), never in a local store. +3. `crawler/src/lib.rs`: `crawler.record_kill` (a `#[command]` method) takes + the principal from the trusted session (`x-user-id`) and **decides the XP + from the tier**; the input has no identity or XP field and unknown fields + are rejected. Event, ledger receipt, snapshot and outbox row commit in one + Durable Object SQLite transaction. 4. Tick N+k: `Zone::on_fact_settled` sends the killer `saved` (with `waitedMs`) and `profile`; the popup turns from "saving…" to "+XP saved ✓". The sink also hands the kill to the leaderboard @@ -445,14 +512,22 @@ after dying. Level `n` starts at `40·(n−1)²` XP. 5. The read model (`crawler/`, `service/`): `crawler.mob_killed` is a **domain event** — the same transaction writes the post-kill `CrawlerState` (principal, name, XP, kills, level) to the cell's outbox - (`repo.publish_events()`). With `--projection-bus sqlite:` the sink - drains that outbox onto a `SqliteBus` after the commit (and a sweep drains - every cell at startup and every 2 s, so a crash between commit and drain - or a bus outage only delays it). `mobpit-service` consumes the bus with a - modeled projector (`projection!` → `upsert_player_stats`), checkpointed - and deduplicated in its own SQLite database, and serves `player_stats` - over GraphQL with `@live`. The service has no commands: nothing on that - side can name a player or an amount. + (`repo.publish_events()`). celld drains the outbox to the `game-demo-outbox` + Queue; the cells worker's Queue consumer relays each message to the zone + server's `/internal/celld-queue/relay` (secret-authenticated), which + publishes it onto the `SqliteBus` at `--projection-bus` (default + `sqlite:{data-dir}/bus.db`). A crash between commit and drain, or a relay + outage, only delays it (the cell's alarm re-drains; the Queue retries). + `mobpit-service` consumes the bus with a + modeled projector (`projection!` → `upsert_player_stats`, defined in + `crawler/src/stats.rs`), checkpointed and deduplicated in its own SQLite + database, and serves `player_stats` over GraphQL with `@live`. The service + has no commands: nothing on that side can name a player or an amount. + The zone server projects the same events into its own copy + (`mobpit/directory.rs`, bus consumer `mobpit-zone`, + `{data-dir}/read-models.db`): `/dev/mobpit/crawlers` and the leaderboard + rebuild list crawlers from it (eventually consistent; the rebuild then + reads each listed crawler's kill history from its cell). The read path: on join the zone asks a background task to load the player's cell and sends `profile` (`source: "cell"`) a few ticks later. @@ -475,13 +550,16 @@ examples/game/run.sh mobpit # restart (same .data/mobpit), The ticket secret is persisted in the data dir (`dev-ticket-secret`), so the page's guest token still maps to the same principal, and the profile comes -back from the cell. What survives: +back from the cell. `run.sh` keeps celld's state in `.data/mobpit/celld` +(`.celld/dev`), so celld itself can be killed and restarted too. What +survives: -- **Settled facts** (you saw "saved ✓"): durable. The commit was acknowledged - only after the SQLite transaction (event + ledger receipt) was fsynced. +- **Settled facts** (you saw "saved ✓"): durable. celld acknowledged the + commit only after the cell's SQLite transaction (event + ledger receipt) + was durable. - **Facts in flight** at the crash: each is either fully committed (the - transaction finished; you just never saw the settlement) or not at all - (rolled back with the WAL) — never half-applied, never twice. The zone's + transaction finished; you just never saw the settlement) or not at all — + never half-applied, never twice. The zone's in-memory state (mob HP, positions, the fact queue) dies with the process, so an uncommitted kill is lost like any other unsaved world state; there is no zone-side outbox (see `distributed_game` README, *Facts on dispose*). @@ -492,12 +570,24 @@ back from the cell. What survives: epoch (`Ctx::boot_epoch`, grows across restarts) because zone ticks and mob generations restart at 0. -`tests/mobpit_smoke.rs` proves it with the real binary: bots fight, storage is -made slow, the server gets `SIGKILL` with facts in flight, restarts on the -same directory; then `settled ≤ durable kills ≤ settled + in flight`, the -board (rebuilt at boot) equals the cells, and re-committing a pre-crash fact -(same key and input) against the files returns the original receipt with -`replayed: true` and unchanged kills. +`tests/mobpit_smoke.rs` proves it with the real binary on its own +`celld dev`: bots fight, storage is made slow, the server gets `SIGKILL` with +facts in flight, then celld is killed (`SIGKILL` of its process tree) and +restarted on its state, and the server restarts; then `settled ≤ durable +kills ≤ settled + in flight`, the board (rebuilt at boot) equals the cells, +and re-committing a pre-crash fact (same key and input) returns the original +receipt with `replayed: true` and unchanged kills. + +**celld.** Every demo with durable state runs its cells on celld: +`run.sh mobpit|leaderboard|freeze|grid|ripple|sveltekit|smoke` need the celld +version pinned in the repository's `.celld-version` (`CELLD_BIN`, default +`celld` on PATH; `scripts/celld.sh check` verifies it and prints the install +command) and `worker-build` (`cargo install worker-build --locked`) for +`cells/`. Only `arena` and `gallery` run without celld. The tests +(`tests/mobpit_smoke.rs`, `tests/freeze_smoke.rs`, `tests/grid_smoke.rs`, +`tests/ripple_smoke.rs`, `service/tests/projection.rs`, the headless and +Playwright tests) each start their own `celld dev` on a temp project +(`game_demos::devcelld`, `client/tests/helpers/celld.mjs`). ## Leaderboard @@ -528,7 +618,9 @@ board (rebuilt at boot) equals the cells, and re-committing a pre-crash fact Dev routes (`--dev` only, none takes an identity or a reward): `GET|POST /dev/mobpit/storage`, `GET /dev/mobpit/stats` (sink counters, -pending, recent commits), `GET /dev/mobpit/crawlers` (every cell on disk), +pending, recent commits), `GET /dev/mobpit/crawlers` (every crawler in the +directory: the `player_stats` projection of the cells' events, eventually +consistent), `GET /dev/mobpit/presence` (the presence cells' view), `GET /dev/mobpit/leaderboard[?member=]`, `POST /dev/mobpit/leaderboard/rebuild`. ## Adding a demo diff --git a/examples/game/cells/Cargo.toml b/examples/game/cells/Cargo.toml new file mode 100644 index 000000000..65545b171 --- /dev/null +++ b/examples/game/cells/Cargo.toml @@ -0,0 +1,32 @@ +# The game demos' durable cells on celld (workers-rs, wasm32): Mob Pit's +# per-player Crawler aggregate and per-player presence +# (with the outbox Queue consumer that relays Crawler domain events to the +# zone server), and the zone lease and hibernation-state cells every +# multi-host demo uses. +# Own workspace (cdylib for wasm32-unknown-unknown); built with +# `worker-build --release` by `examples/game/run.sh`. +[workspace] +members = ["."] +resolver = "2" + +[package] +name = "game-demo-cells" +version = "0.1.0" +edition = "2021" +publish = false +description = "Game demo cells for celld: Mob Pit, zone leases and zone state (dev demo)" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +console_error_panic_hook = "0.1" +distributed = { path = "../../..", default-features = false, features = ["workers-rs"] } +distributed_game = { path = "../../../distributed_game", default-features = false, features = ["cell", "workers-rs"] } +mobpit_crawler = { path = "../server/crawler" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +worker = { version = "0.8", features = ["queue"] } + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs new file mode 100644 index 000000000..b7cfc59e7 --- /dev/null +++ b/examples/game/cells/src/lib.rs @@ -0,0 +1,849 @@ +//! The game demos' durable cells on celld. Mob Pit's are fed by the zone +//! server's `distributed_game::celld::CelldFactSink` over the internal HTTP +//! path; every multi-host demo keeps its zone ownership and hibernation +//! snapshots here. +//! +//! - `crawler/{principal}` (`CrawlerCell`): the `mobpit_crawler::Crawler` +//! aggregate with its `#[command]` `crawler.record_kill` (XP decided here +//! from the mob tier; the principal comes from the trusted fact session). +//! Each commit writes the `crawler.mob_killed` domain event to the cell +//! outbox, which drains to the `game-demo-outbox` Queue. +//! - `presence/{principal}` (`PresenceCell`): the runtime's player lifecycle +//! facts (`zone.*`). A fact emitted before the one that set the current +//! entry (`FactStamp::emission_order`) is ignored, so a late "left pit-1" +//! cannot mark a player who already joined pit-2 offline. +//! - `gem-pouch/{principal}` (`GemPouchCell`): The Freeze's +//! `freeze.gem_collected` facts (which gem spot, at which content +//! version; the zone reports no amount). +//! - `content-log/{zone}` (`ContentLogCell`): The Freeze's +//! `freeze.content_applied` zone facts (every content switch of a zone). +//! - `zone-lease/{zone}` (`ZoneLeaseCell`): zone ownership +//! (`distributed_game::world::zone_lease`, `CellZoneOwnership`); its +//! alarm declares `zone.lease_expired`. +//! - `zone-state/{zone}` (`ZoneStateCell`): hibernation snapshots +//! (`distributed_game::world::zone_state`, `CellZoneStateStore`); large +//! ones in the `ZONE_SNAPSHOTS` R2 bucket. +//! +//! Every route requires `DISTRIBUTED_INTERNAL_SECRET`. Mob Pit commands also +//! require the trusted fact session (principal + `FactStamp` headers, +//! rebuilt with `fact_session_from_headers`), so nothing but a zone server's +//! fact worker can run them; zone lease and state commands require the +//! `zone-host` role. +//! +//! The Queue consumer relays each outbox message to `CELLD_QUEUE_RELAY_URL` +//! (the zone server's authenticated `/internal/celld-queue/relay`), which +//! publishes it onto the projection bus read by `mobpit-service`. Delivery +//! is at-least-once with stable message ids; the projector deduplicates. + +use distributed::bus::{CelldQueueEnvelope, CelldQueueHttpPublisher, CelldQueueRelay}; +use distributed::cell_host::{ + AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, + InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, +}; +use distributed::command::{typed_command, PreparedCommand, Succeeded}; +use distributed::microsvc::{ + CausalCommandContext, CausalRouteDependencies, HandlerError, PortableCommand, Routes, Session, +}; +use distributed::{sourced, Aggregate, Entity}; +use distributed_game::cell::{fact_session_from_headers, session_stamp}; +use distributed_game::lifecycle::{ + Lifecycle, CONNECTION_LOST, PLAYER_DISCONNECTED, PLAYER_JOINED, PLAYER_LEFT, PLAYER_RECONNECTED, +}; +use distributed_game::world::zone_lease::{mount_zone_lease, ZoneLease}; +use distributed_game::world::zone_state::{celld_worker, mount_zone_state, ZoneState}; +use distributed_game::FactStamp; +use mobpit_crawler::{ + Crawler, MobKilledArgs, RecordKill, CRAWLER_KIND, MOB_KILLED_EVENT, PRESENCE_KIND, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use worker::*; + +fn rejected(error: impl std::fmt::Display) -> HandlerError { + HandlerError::Rejected(error.to_string()) +} + +// ---------------------------------------------------------------- Presence + +/// The ordering key of the fact that set a presence entry. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StampKey { + pub zone_id: String, + pub activation: u64, + pub tick: u64, + pub emitted_at_ms: u64, + pub seq: u64, +} + +impl StampKey { + fn of(stamp: &FactStamp) -> Self { + Self { + zone_id: stamp.zone_id.clone(), + activation: stamp.activation, + tick: stamp.tick, + emitted_at_ms: stamp.emitted_at_ms, + seq: stamp.seq, + } + } + + fn stamp(&self) -> FactStamp { + FactStamp { + zone_id: self.zone_id.clone(), + activation: self.activation, + tick: self.tick, + emitted_at_ms: self.emitted_at_ms, + seq: self.seq, + ..FactStamp::default() + } + } +} + +/// One presence transition. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PresenceChange { + pub zone: String, + /// `online`, `away` or `offline`. + pub status: String, + pub since_ms: u64, + pub stamp: StampKey, +} + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct PlayerPresence { + entity: Entity, + current: Option, +} + +#[sourced( + entity, + aggregate_type = "PlayerPresence", + events = "PlayerPresenceEvent" +)] +impl PlayerPresence { + #[event("presence.changed", version = 1)] + fn changed(&mut self, principal: String, change: PresenceChange) { + if self.entity.id().is_empty() { + self.entity.set_id(principal); + } + self.current = Some(change); + } +} + +/// Any lifecycle fact input (the union of `distributed_game::lifecycle`'s +/// inputs; decoded per command with `Lifecycle::decode`). +#[derive(Debug, Serialize, Deserialize, distributed::CommandInput)] +pub struct LifecycleInput { + pub zone: String, + pub via: Option, + pub from_zone: Option, + pub reason: Option, + pub grace_ms: Option, + pub cause: Option, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct PresenceApplied { + /// False when a later-emitted fact already set the entry. + pub applied: bool, +} + +/// One mount per lifecycle command (handlers are plain `async fn`s). +macro_rules! lifecycle_command { + ($mount:ident, $handler:ident, $command:expr) => { + struct $mount; + impl PortableCommand for $mount + where + D: CausalRouteDependencies + Send + Sync + 'static, + { + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + $command, + )) + .handle($handler) + } + } + + async fn $handler( + ctx: &CausalCommandContext<'_, PlayerPresence>, + input: LifecycleInput, + ) -> std::result::Result>, HandlerError> { + handle_lifecycle(ctx, $command, input).await + } + }; +} + +lifecycle_command!(Joined, handle_joined, PLAYER_JOINED); +lifecycle_command!(Lost, handle_lost, CONNECTION_LOST); +lifecycle_command!(Reconnected, handle_reconnected, PLAYER_RECONNECTED); +lifecycle_command!(Disconnected, handle_disconnected, PLAYER_DISCONNECTED); +lifecycle_command!(Left, handle_left, PLAYER_LEFT); + +async fn handle_lifecycle( + ctx: &CausalCommandContext<'_, PlayerPresence>, + command: &'static str, + input: LifecycleInput, +) -> std::result::Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + let stamp = session_stamp(ctx.session()) + .ok_or_else(|| rejected("lifecycle facts need a fact stamp"))?; + let event = Lifecycle::decode(command, &serde_json::to_value(&input).map_err(rejected)?) + .ok_or_else(|| rejected(format!("malformed `{command}` input")))?; + let status = match event { + Lifecycle::Joined(_) | Lifecycle::Reconnected(_) => "online", + Lifecycle::ConnectionLost(_) => "away", + Lifecycle::Left(_) | Lifecycle::Disconnected(_) => "offline", + }; + let repo = ctx.repo(); + let mut presence = match repo.get(&principal).await? { + Some(presence) => presence, + None => repo.create(), + }; + let newer = presence.current.as_ref().is_none_or(|current| { + current.stamp.stamp().emission_order(&stamp) == std::cmp::Ordering::Less + }); + if newer { + presence + .changed( + principal, + PresenceChange { + zone: event.zone().to_string(), + status: status.to_string(), + since_ms: stamp.emitted_at_ms, + stamp: StampKey::of(&stamp), + }, + ) + .map_err(rejected)?; + } + repo.commit(presence)? + .succeeded(PresenceApplied { applied: newer }) +} + +// ---------------------------------------------------------------- The Freeze + +/// `freeze.gem_collected`. +pub const GEM_COLLECTED: &str = "freeze.gem_collected"; +/// `freeze.content_applied`. +pub const CONTENT_APPLIED: &str = "freeze.content_applied"; + +/// One collected gem. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Gem { + pub zone: String, + pub spot: u32, + pub content_version: String, +} + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct GemPouch { + entity: Entity, + gems: u32, + last: Option, +} + +#[sourced(entity, aggregate_type = "GemPouch", events = "GemPouchEvent")] +impl GemPouch { + #[event("freeze.gem_recorded", version = 1)] + fn recorded(&mut self, principal: String, gem: Gem) { + if self.entity.id().is_empty() { + self.entity.set_id(principal); + } + self.gems += 1; + self.last = Some(gem); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct GemInput { + pub spot: u32, +} + +#[derive(Debug, Serialize, distributed::CommandOutput)] +pub struct Counted { + pub count: u32, +} + +struct CollectGem; +impl PortableCommand for CollectGem +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>(GEM_COLLECTED)) + .handle(handle_gem) + } +} + +async fn handle_gem( + ctx: &CausalCommandContext<'_, GemPouch>, + input: GemInput, +) -> std::result::Result>, HandlerError> { + let principal = ctx.user_id()?.to_string(); + let stamp = + session_stamp(ctx.session()).ok_or_else(|| rejected("gem facts need a fact stamp"))?; + let repo = ctx.repo(); + let mut pouch = match repo.get(&principal).await? { + Some(pouch) => pouch, + None => repo.create(), + }; + pouch + .recorded( + principal, + Gem { + zone: stamp.zone_id, + spot: input.spot, + content_version: stamp.content_version, + }, + ) + .map_err(rejected)?; + let count = pouch.gems; + repo.commit(pouch)?.succeeded(Counted { count }) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ContentSwitch { + pub from: String, + pub to: String, + pub tick: u64, +} + +#[derive(Clone, Default, Serialize, Deserialize, distributed::Snapshot)] +pub struct ContentLog { + entity: Entity, + switches: Vec, +} + +#[sourced(entity, aggregate_type = "ContentLog", events = "ContentLogEvent")] +impl ContentLog { + #[event("freeze.content_recorded", version = 1)] + fn recorded(&mut self, zone: String, switch: ContentSwitch) { + if self.entity.id().is_empty() { + self.entity.set_id(zone); + } + self.switches.push(switch); + } +} + +#[derive(Debug, Deserialize, distributed::CommandInput)] +pub struct ContentInput { + pub from: String, + pub to: String, +} + +struct ApplyContent; +impl PortableCommand for ApplyContent +where + D: CausalRouteDependencies + Send + Sync + 'static, +{ + fn install(self, routes: Routes) -> Routes { + routes + .typed_command(typed_command::>( + CONTENT_APPLIED, + )) + .handle(handle_content) + } +} + +async fn handle_content( + ctx: &CausalCommandContext<'_, ContentLog>, + input: ContentInput, +) -> std::result::Result>, HandlerError> { + let stamp = + session_stamp(ctx.session()).ok_or_else(|| rejected("content facts need a fact stamp"))?; + let repo = ctx.repo(); + // The zone's own log: the shard is the stamped zone. + let mut log = match repo.get(&stamp.zone_id).await? { + Some(log) => log, + None => repo.create(), + }; + log.recorded( + stamp.zone_id.clone(), + ContentSwitch { + from: input.from, + to: input.to, + tick: stamp.tick, + }, + ) + .map_err(rejected)?; + let count = log.switches.len() as u32; + repo.commit(log)?.succeeded(Counted { count }) +} + +async fn read_pouch(cell: &AggregateCell) -> std::result::Result { + let pouch = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ + "gems": pouch.as_ref().map_or(0, |p| p.gems), + "last": pouch.and_then(|p| p.last), + })) +} + +async fn read_content_log(cell: &AggregateCell) -> std::result::Result { + let log = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ "switches": log.map(|l| l.switches).unwrap_or_default() })) +} + +#[durable_object] +pub struct GemPouchCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for GemPouchCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 50) + .expect("gem pouch cell identity") + .mount(CollectGem) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_pouch, fact_session).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +#[durable_object] +pub struct ContentLogCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ContentLogCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 50) + .expect("content log cell identity") + .mount(ApplyContent) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_content_log, fact_session).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +// ---------------------------------------------------------------- Hosting + +/// Mob Pit cells run only facts: the principal and the stamp, from the +/// trusted headers of the authenticated internal path. +fn fact_session(header: &dyn Fn(&str) -> Option) -> Option { + fact_session_from_headers(header) +} + +/// Zone lease cells: the zone server's identity and role (the handlers +/// require `zone-host`). +fn zone_host_session(header: &dyn Fn(&str) -> Option) -> Option { + let mut session = Session::new(); + for key in [distributed::USER_ID_KEY, distributed::microsvc::ROLE_KEY] { + if let Some(value) = header(key) { + session.set(key, value); + } + } + Some(session) +} + +type Sessions = fn(&dyn Fn(&str) -> Option) -> Option; + +async fn dispatch( + env: &Env, + cell: &AggregateCell, + command: &str, + req: &mut Request, + sessions: Sessions, +) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => return error_response(CellDispatchError::BadRequest("invalid JSON".into())), + }; + let parsed = match CellWaitPathRequest::parse(body) { + Ok(parsed) => parsed, + Err(error) => return error_response(CellDispatchError::BadRequest(error)), + }; + let header = |name: &str| req.headers().get(name).ok().flatten(); + let identity = match ( + header(CELL_SERVICE_ID_HEADER), + header(CELL_PRINCIPAL_PARTITION_HEADER), + ) { + (Some(service), Some(partition)) => { + match CellCommandIdentity::new(service, partition, &parsed.command_id) { + Ok(identity) => identity, + Err(error) => return error_response(error), + } + } + _ => return error_response(CellDispatchError::Unauthorized), + }; + let Some(session) = sessions(&header) else { + return error_response(CellDispatchError::Forbidden); + }; + let result = cell + .dispatch_idempotent(command, &identity, parsed.input, session) + .await; + if let Err(error) = cell.drain_outbox(env).await { + worker::console_error!("post-commit drain deferred: {error}"); + } + match result { + Ok(dispatch) => json_status(dispatch.wait_path_wire(), 200), + Err(error) => error_response(error), + } +} + +/// `GET /{kind}/{shard}` → `read(cell)`; `POST /{kind}/{shard}/{command}`. +async fn route( + env: &Env, + cell: &AggregateCell, + mut req: Request, + read: F, + sessions: Sessions, +) -> Result +where + A: Aggregate + Send + Sync + 'static, + F: AsyncFnOnce(&AggregateCell) -> std::result::Result, +{ + if let Err(error) = authenticate(&req, env) { + return error_response(error); + } + let url = req.url()?; + let parts: Vec = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .map(str::to_string) + .collect(); + match (req.method(), parts.get(2)) { + (Method::Get, None) => match read(cell).await { + Ok(body) => json_status(body, 200), + Err(error) => error_response(CellDispatchError::Internal(error)), + }, + (Method::Post, Some(command)) => { + let command = command.clone(); + dispatch(env, cell, &command, &mut req, sessions).await + } + _ => json_status(json!({ "error": "not found" }), 404), + } +} + +async fn read_crawler(cell: &AggregateCell) -> std::result::Result { + let state = cell.load().await.map_err(|error| error.to_string())?; + let mut kills = Vec::new(); + for record in cell.history().await.map_err(|error| error.to_string())? { + if record.event_name != MOB_KILLED_EVENT { + continue; + } + let (crawler, name, mob, tier, xp): MobKilledArgs = + record.decode().map_err(|error| error.to_string())?; + let at_ms = record + .timestamp + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis() as u64); + kills.push(json!({ + "sequence": record.sequence, "atMs": at_ms, "crawler": crawler, + "name": name, "mob": mob, "tier": tier, "xp": xp, + })); + } + Ok(json!({ + "state": state.map(|c| json!({ + "name": c.name, "xp": c.xp, "kills": c.kills, "level": c.level, + "version": c.version(), + })), + "kills": kills, + })) +} + +async fn read_presence(cell: &AggregateCell) -> std::result::Result { + let presence = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ "current": presence.and_then(|p| p.current) })) +} + +fn outbox(env: &Env) -> CelldOutbox { + CelldOutbox::from_env(env, "OUTBOX").expect("OUTBOX Queue binding") +} + +#[durable_object] +pub struct CrawlerCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for CrawlerCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 100) + .expect("crawler cell identity") + .mount(RecordKill) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_crawler, fact_session).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +#[durable_object] +pub struct PresenceCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for PresenceCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + let cell = AggregateCell::::from_state_with_snapshots(state, 50) + .expect("presence cell identity") + .mount(Joined) + .mount(Lost) + .mount(Reconnected) + .mount(Disconnected) + .mount(Left) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_presence, fact_session).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +async fn alarm(env: &Env, cell: &AggregateCell) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let wake = cell + .alarm(env) + .await + .map_err(|error| Error::RustError(error.to_string()))?; + for error in &wake.drain.deferred { + worker::console_error!("drain deferred: {error}"); + } + Response::ok("ok") +} + +#[durable_object] +pub struct ZoneLeaseCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ZoneLeaseCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + // Renewals are internal events: snapshots keep loads short. + let cell = mount_zone_lease( + AggregateCell::::from_state_with_snapshots(state, 50) + .expect("zone lease cell identity"), + ) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + route(&self.env, &self.cell, req, read_lease, zone_host_session).await + } + + async fn alarm(&self) -> Result { + let wake = self + .cell + .alarm(&self.env) + .await + .map_err(|error| Error::RustError(error.to_string()))?; + for failed in wake.schedules.failed.iter().chain(&wake.schedules.retried) { + worker::console_error!("lease alarm failed: {} {}", failed.code, failed.message); + } + Response::ok("ok") + } +} + +async fn read_lease(cell: &AggregateCell) -> std::result::Result { + let lease = cell.load().await.map_err(|error| error.to_string())?; + Ok(json!({ + "holder": lease.as_ref().and_then(|l| l.holder().map(str::to_string)), + "token": lease.as_ref().map(|l| l.token()), + })) +} + +#[durable_object] +pub struct ZoneStateCell { + cell: AggregateCell, + env: Env, +} + +impl DurableObject for ZoneStateCell { + fn new(state: State, env: Env) -> Self { + console_error_panic_hook::set_once(); + // Every save replaces the value: snapshot each commit. + let cell = mount_zone_state( + AggregateCell::::from_state_with_snapshots(state, 1) + .expect("zone state cell identity"), + ) + .with_celld_outbox(outbox(&env)); + Self { cell, env } + } + + async fn fetch(&self, req: Request) -> Result { + if let Err(error) = authenticate(&req, &self.env) { + return error_response(error); + } + let bucket = celld_worker::bucket(&self.env, "ZONE_SNAPSHOTS")?; + celld_worker::fetch(&self.cell, &bucket, req).await + } + + async fn alarm(&self) -> Result { + alarm(&self.env, &self.cell).await + } +} + +#[event(fetch)] +async fn main(req: Request, env: Env, _ctx: Context) -> Result { + console_error_panic_hook::set_once(); + let url = req.url()?; + let path = url.path(); + if path == "/" || path == "/health" { + return Response::ok("mobpit cells\n"); + } + if let Err(error) = authenticate(&req, &env) { + return error_response(error); + } + let parts: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + let binding = match parts.first().copied() { + Some(CRAWLER_KIND) => "CRAWLER", + Some(PRESENCE_KIND) => "PRESENCE", + Some("gem-pouch") => "GEM_POUCH", + Some("content-log") => "CONTENT_LOG", + Some("zone-lease") => "ZONE_LEASE", + Some("zone-state") => "ZONE_STATE", + _ => return json_status(json!({ "error": "unknown cell kind" }), 404), + }; + let Some(shard) = parts.get(1) else { + return json_status(json!({ "error": "missing shard" }), 400); + }; + let shard = percent_decode(shard).ok_or_else(|| Error::RustError("bad shard".into()))?; + let stub = env + .durable_object(binding)? + .id_from_name(&shard)? + .get_stub()?; + stub.fetch_with_request(req).await +} + +/// Outbox Queue consumer: relay each canonical message to the zone server. +#[event(queue)] +pub async fn queue(batch: MessageBatch, env: Env, _ctx: Context) -> Result<()> { + console_error_panic_hook::set_once(); + let relay = match relay(&env) { + Ok(relay) => relay, + Err(error) => { + // Keep the messages: they are retried until a relay is configured. + worker::console_error!("mobpit outbox relay unavailable: {error}"); + batch.retry_all(); + return Ok(()); + } + }; + for delivery in batch.messages()? { + match relay.relay(delivery.body().clone()).await { + Ok(()) => delivery.ack(), + Err(error) => { + worker::console_error!( + "mobpit outbox relay failed for {}: {}", + delivery.id(), + error + ); + delivery.retry(); + } + } + } + Ok(()) +} + +fn relay(env: &Env) -> std::result::Result, String> { + let var = |name: &str| { + env.var(name) + .map(|value| value.to_string()) + .or_else(|_| env.secret(name).map(|value| value.to_string())) + .map_err(|_| format!("{name} is required")) + }; + let endpoint = var("CELLD_QUEUE_RELAY_URL")?; + let secret = var(CELL_INTERNAL_SECRET_ENV)?; + // `http://127.0.0.1` relays are dev-only; anything else must be HTTPS. + let local = + endpoint.starts_with("http://127.0.0.1") || endpoint.starts_with("http://localhost"); + let publisher = if local { + CelldQueueHttpPublisher::new_local_test(endpoint, &secret) + } else { + CelldQueueHttpPublisher::new(endpoint) + } + .map_err(|error| error.to_string())? + .with_header(CELL_INTERNAL_SECRET_HEADER, secret); + Ok(CelldQueueRelay::new(publisher)) +} + +fn percent_decode(segment: &str) -> Option { + let bytes = segment.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + out.push(u8::from_str_radix(segment.get(i + 1..i + 3)?, 16).ok()?); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).ok() +} + +fn authenticate(req: &Request, env: &Env) -> std::result::Result<(), CellDispatchError> { + let configured = env + .secret(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + .or_else(|_| { + env.var(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + }) + .map_err(|_| CellDispatchError::Internal("internal secret is required".into()))?; + let secret = InternalHttpSecret::new(configured).map_err(CellDispatchError::Internal)?; + let candidate = req + .headers() + .get(CELL_INTERNAL_SECRET_HEADER) + .ok() + .flatten() + .ok_or(CellDispatchError::Unauthorized)?; + if secret.matches(&candidate) { + Ok(()) + } else { + Err(CellDispatchError::Unauthorized) + } +} + +fn error_response(error: CellDispatchError) -> Result { + json_status( + json!({ "error": error.client_message(), "code": error.code() }), + error.status_code(), + ) +} + +fn json_status(body: Value, status: u16) -> Result { + Ok(Response::from_json(&body)?.with_status(status)) +} diff --git a/examples/game/cells/wrangler.jsonc b/examples/game/cells/wrangler.jsonc new file mode 100644 index 000000000..c78fe9da8 --- /dev/null +++ b/examples/game/cells/wrangler.jsonc @@ -0,0 +1,46 @@ +// The game demos' cells for `celld dev` (and `celld deploy`). The launcher +// (`examples/game/run.sh`, the test harnesses) writes `.dev.vars` beside a +// copy of this file with DISTRIBUTED_INTERNAL_SECRET and +// CELLD_QUEUE_RELAY_URL (the zone server's `/internal/celld-queue/relay`). +{ + "name": "game-demo-cells", + "main": "build/worker/shim.mjs", + "compatibility_date": "2026-01-01", + "durable_objects": { + "bindings": [ + { "name": "CRAWLER", "class_name": "CrawlerCell" }, + { "name": "PRESENCE", "class_name": "PresenceCell" }, + { "name": "ZONE_LEASE", "class_name": "ZoneLeaseCell" }, + { "name": "ZONE_STATE", "class_name": "ZoneStateCell" }, + { "name": "GEM_POUCH", "class_name": "GemPouchCell" }, + { "name": "CONTENT_LOG", "class_name": "ContentLogCell" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["CrawlerCell", "PresenceCell"] }, + { "tag": "v2", "new_sqlite_classes": ["ZoneLeaseCell", "ZoneStateCell"] }, + { "tag": "v3", "new_sqlite_classes": ["GemPouchCell", "ContentLogCell"] } + ], + // Hibernation snapshots above the inline limit (celld's object storage). + "r2_buckets": [ + { "binding": "ZONE_SNAPSHOTS", "bucket_name": "zone-snapshots" } + ], + "queues": { + "producers": [ + { "binding": "OUTBOX", "queue": "game-demo-outbox" } + ], + "consumers": [ + { + "queue": "game-demo-outbox", + "max_batch_size": 10, + "max_batch_timeout": 1, + "max_retries": 100, + "dead_letter_queue": "game-demo-outbox-dlq", + "retry_delay": 2 + } + ] + }, + "vars": { + "OUTBOX_DRAIN_INTERVAL_MS": "5000" + } +} diff --git a/examples/game/client/index.html b/examples/game/client/index.html index dda69fcad..4615fc991 100644 --- a/examples/game/client/index.html +++ b/examples/game/client/index.html @@ -12,7 +12,7 @@

distributed_game demos

Each demo runs its own server: examples/game/run.sh <demo>. The page for the running demo works; the others need their server.