From a9e2f58c17d95274b630f7c77f654993924dbe37 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Sat, 31 Oct 2026 13:21:43 +0600 Subject: [PATCH 1/5] Refactor concurrency and token revocation handling to use LeaseManager and InMemoryCacheStore - Replaced MemoryConcurrencyLimiter with LeaseManager in stream thread naming tests. - Updated test_thread_lock.py to utilize LeaseManager for concurrency limits. - Refactored token revocation tests to use InMemoryCacheStore instead of InMemoryTokenRevocation and RedisTokenRevocation. - Adjusted websocket guard tests to implement LeaseManager for connection limits. - Enhanced websocket limits tests to share the bound cache store for rate limiting and connection management. --- 10xgraph.json | 1 - CHANGELOG.md | 63 ++ CLAUDE.md | 44 +- README.md | 10 +- SECURITY.md | 4 +- config-editor-ui/src/icons.js | 2 +- config-editor-ui/src/pages.jsx | 6 +- config-editor-ui/src/store.js | 2 +- pyproject.toml | 2 +- tenxgraph_api/__init__.py | 10 + tenxgraph_api/cli/commands/init.py | 41 +- tenxgraph_api/cli/commands/revoke.py | 66 +- tenxgraph_api/cli/config_editor/schema.py | 173 +++--- tenxgraph_api/cli/config_editor/static/app.js | 6 +- tenxgraph_api/cli/config_editor/validation.py | 82 ++- tenxgraph_api/cli/main.py | 14 +- tenxgraph_api/cli/templates/prod/.env.example | 3 +- .../cli/templates/prod/10xgraph.json | 8 +- .../10xgraph/references/api-configuration.md | 17 +- .../references/auth-and-authorization.md | 24 +- .../10xgraph/references/rate-limiting.md | 114 +--- .../src/app/core/auth/authorization.py | 30 +- .../src/app/core/auth/ownership_resolver.py | 109 ++-- .../src/app/core/auth/permissions.py | 6 +- tenxgraph_api/src/app/core/auth/revocation.py | 369 ++++-------- .../src/app/core/cache_store/__init__.py | 13 + .../src/app/core/cache_store/base.py | 122 ++++ .../src/app/core/cache_store/loader.py | 110 ++++ .../src/app/core/cache_store/memory_store.py | 326 ++++++++++ .../src/app/core/cache_store/redis_store.py | 327 ++++++++++ tenxgraph_api/src/app/core/concurrency.py | 264 -------- .../src/app/core/config/graph_config.py | 193 +++--- tenxgraph_api/src/app/core/config/settings.py | 25 +- .../src/app/core/config/setup_middleware.py | 14 +- tenxgraph_api/src/app/core/leases.py | 331 ++++++++++ .../src/app/core/middleware/__init__.py | 15 +- .../core/middleware/rate_limit/__init__.py | 10 +- .../app/core/middleware/rate_limit/base.py | 46 -- .../app/core/middleware/rate_limit/factory.py | 101 ---- .../app/core/middleware/rate_limit/limiter.py | 134 +++++ .../app/core/middleware/rate_limit/memory.py | 132 ---- .../core/middleware/rate_limit/middleware.py | 47 +- .../app/core/middleware/rate_limit/redis.py | 146 ----- tenxgraph_api/src/app/loader.py | 176 +++--- tenxgraph_api/src/app/main.py | 65 +- .../src/app/routers/graph/idempotency.py | 71 +-- .../src/app/routers/graph/realtime_guard.py | 129 +--- tenxgraph_api/src/app/routers/graph/router.py | 2 +- .../src/app/routers/graph/run_guard.py | 2 +- .../src/app/routers/graph/run_limits.py | 9 +- .../routers/graph/services/graph_service.py | 2 +- .../src/app/routers/graph/thread_lock.py | 20 +- .../src/app/routers/ping/readiness.py | 190 ++---- tenxgraph_api/src/app/routers/ping/router.py | 12 +- .../src/app/routers/ping/startup_checks.py | 190 +++--- tenxgraph_api/testing/__init__.py | 1 + tenxgraph_api/testing/cache_store_contract.py | 304 ++++++++++ tests/cli/test_group6_cli_templates.py | 109 +++- tests/cli/test_init_non_interactive.py | 61 +- tests/cli/test_init_prod.py | 4 +- tests/cli/test_production_readiness.py | 37 +- .../integration_tests/test_idempotency_key.py | 61 +- .../test_token_revocation_api.py | 38 +- tests/unit_tests/auth/test_token_claims.py | 24 +- tests/unit_tests/cache_store/__init__.py | 0 .../cache_store/test_cache_store_contract.py | 23 + .../cache_store/test_memory_cache_store.py | 219 +++++++ .../cache_store/test_redis_cache_store.py | 273 +++++++++ tests/unit_tests/test_closing_and_cancel.py | 11 +- tests/unit_tests/test_concurrency_limiter.py | 569 ++++++++++++------ tests/unit_tests/test_config_editor.py | 73 ++- tests/unit_tests/test_graph_config.py | 14 + tests/unit_tests/test_group3a_storage.py | 38 +- tests/unit_tests/test_health_check_probes.py | 23 +- tests/unit_tests/test_identity_cache.py | 10 +- tests/unit_tests/test_loader.py | 26 +- tests/unit_tests/test_lock_lease_integrity.py | 128 ++-- tests/unit_tests/test_middleware_order.py | 2 +- .../test_ownership_authorization.py | 1 + tests/unit_tests/test_ownership_resolver.py | 69 ++- tests/unit_tests/test_pure_asgi_middleware.py | 16 +- tests/unit_tests/test_rate_limit.py | 204 +++---- tests/unit_tests/test_rate_limit_keying.py | 8 +- .../unit_tests/test_rate_limit_memory_lru.py | 143 ++--- .../unit_tests/test_rate_limit_unavailable.py | 99 +-- tests/unit_tests/test_rate_limit_user_key.py | 10 +- tests/unit_tests/test_readiness.py | 171 ++---- tests/unit_tests/test_redis_hardening.py | 190 ++---- tests/unit_tests/test_reserved_user_id.py | 11 +- tests/unit_tests/test_run_cleanup_order.py | 2 +- tests/unit_tests/test_run_entry_guards.py | 11 +- tests/unit_tests/test_run_limits.py | 12 +- tests/unit_tests/test_setup_middleware.py | 13 +- tests/unit_tests/test_shutdown_readiness.py | 37 +- tests/unit_tests/test_shutdown_wiring.py | 34 ++ tests/unit_tests/test_startup_checks.py | 350 ++++++++--- tests/unit_tests/test_stream_thread_naming.py | 9 +- tests/unit_tests/test_thread_lock.py | 40 +- tests/unit_tests/test_token_revocation.py | 314 +++++----- tests/unit_tests/test_websocket_guard.py | 49 +- tests/unit_tests/test_websocket_limits.py | 78 ++- 101 files changed, 5132 insertions(+), 3217 deletions(-) create mode 100644 tenxgraph_api/src/app/core/cache_store/__init__.py create mode 100644 tenxgraph_api/src/app/core/cache_store/base.py create mode 100644 tenxgraph_api/src/app/core/cache_store/loader.py create mode 100644 tenxgraph_api/src/app/core/cache_store/memory_store.py create mode 100644 tenxgraph_api/src/app/core/cache_store/redis_store.py delete mode 100644 tenxgraph_api/src/app/core/concurrency.py create mode 100644 tenxgraph_api/src/app/core/leases.py delete mode 100644 tenxgraph_api/src/app/core/middleware/rate_limit/base.py delete mode 100644 tenxgraph_api/src/app/core/middleware/rate_limit/factory.py create mode 100644 tenxgraph_api/src/app/core/middleware/rate_limit/limiter.py delete mode 100644 tenxgraph_api/src/app/core/middleware/rate_limit/memory.py delete mode 100644 tenxgraph_api/src/app/core/middleware/rate_limit/redis.py create mode 100644 tenxgraph_api/testing/__init__.py create mode 100644 tenxgraph_api/testing/cache_store_contract.py create mode 100644 tests/unit_tests/cache_store/__init__.py create mode 100644 tests/unit_tests/cache_store/test_cache_store_contract.py create mode 100644 tests/unit_tests/cache_store/test_memory_cache_store.py create mode 100644 tests/unit_tests/cache_store/test_redis_cache_store.py diff --git a/10xgraph.json b/10xgraph.json index bad6e381..c692db8b 100644 --- a/10xgraph.json +++ b/10xgraph.json @@ -6,7 +6,6 @@ "auth": null, "rate_limit": { "enabled": true, - "backend": "memory", "requests": 100, "window": 60, "by": "ip", diff --git a/CHANGELOG.md b/CHANGELOG.md index e775ffe2..27e333fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,69 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- +## [1.0.0rc4] - Unreleased + +The fourth release candidate of `10xgraph-api` 1.0. It requires `10xgraph` 1.0.0rc4 +(checkpointers have `aping()`). Everything the API shares between workers now lives in one +`BaseCacheStore`, and the startup check no longer refuses to boot over a Redis that nothing +configured needs. `10xgraph-client` stays at 1.0.0-rc.2. + +### Breaking + +- Removed, with no deprecation aliases (release candidates): `ConcurrencyLimiter` and + `core/concurrency.py`, `TokenRevocationBackend` (with `InMemoryTokenRevocation`, + `RedisTokenRevocation`, `build_token_revocation`), `BaseRateLimitBackend` (with + `MemoryRateLimitBackend`, `RedisRateLimitBackend` and the rate-limit factory). Run locks and + caps, idempotency, the ownership cache, token revocation and rate limiting all use the bound + `BaseCacheStore`; implement one to replace Redis. +- `rate_limit.backend` and `rate_limit.redis` are removed. Either key is a config error naming + the replacement: the counters live in the cache store (`cache_store`, else `redis`, else + `REDIS_URL`, else in memory). Delete both keys; move a dedicated rate-limit Redis URL to + `redis` (one Redis now serves every feature). +- `token_revocation` is `true`, `false`, `null` or `{"enabled": true, "token_lifetime": ..., + "user_ttl": ...}`. The string values (`"redis"`, `"memory"`, `"module:attr"`) are config + errors naming `cache_store`. Replace `"redis"` with `true` and set `redis` (or `REDIS_URL`). +- `GET /health/ready`: the `redis` key in `checks` is renamed `cache` and reports the bound + cache store (`{"checkpointer": "ok", "store": "ok", "cache": "ok"}`). Readiness and the + startup check only call `aping()` on the bound checkpointer, store and cache store; a + component's `readiness_check()` hook and the probing of private checkpointer attributes are + gone. +- `10xgraph init --rate-limit redis` writes `"redis": "${REDIS_URL}"` and, with auth, + `"token_revocation": true` (was `rate_limit.backend: "redis"`, `rate_limit.redis.url` and + `"token_revocation": "redis"`). `--rate-limit memory` turns rate limiting on without a + `redis` key. + +### Added + +- `BaseCacheStore` with `RedisCacheStore` (also Valkey, Dragonfly, KeyDB) and + `InMemoryCacheStore`, exported from `tenxgraph_api`. A store has an abstract `aping()` and an + optional `describe()` that names its endpoint (never credentials) in logs and startup errors. +- `"cache_store": "module:attr"` in `10xgraph.json`: a `BaseCacheStore` instance or zero-argument + factory; wins over `redis` and `REDIS_URL`. One INFO line at boot names the store and its + source, for example `Cache store: RedisCacheStore (from REDIS_URL, localhost:6379)`. +- A contract suite for custom cache stores in `tenxgraph_api.testing.cache_store_contract`. +- The `redis` key expands `$VAR` and `${VAR}`; an unset variable is a config error. +- `10xgraph config` edits `cache_store`, and token revocation as a section with `enabled`, + `token_lifetime` and `user_ttl`. + +### Changed + +- The startup check is fatal for the cache store only when a fail-closed feature uses it: + `token_revocation` on, or `rate_limit` with `fail_open: false`. Otherwise an unreachable + cache store is one WARNING (`... unreachable: locks and caps fall back to per process ...`) + and the server starts; `/health/ready` reports `cache: fail` and stays 200. This fixes rc3 + refusing to boot an in-memory server with a leftover `REDIS_URL`. Startup errors name each + component's class, where its setting came from (`REDIS_URL`, `redis in 10xgraph.json`, + `cache_store`) and its `describe()` endpoint. The hint now reads "Start the backend, fix its + URL, or set STARTUP_CHECKS=false to skip this check." +- `10xgraph revoke` writes to the cache store the server would pick (`cache_store`, `redis`, + `REDIS_URL`; `--redis-url` overrides) and refuses with a clear error when that is the + in-memory store. +- Shutdown closes the cache store once, after the graph and after pending lease releases. The + authorization backend no longer closes a Redis client. + +--- + ## [1.0.0rc3] - 2026-10-30 The third release candidate of `10xgraph-api` 1.0. It requires `10xgraph` 1.0.0rc3 (stores have diff --git a/CLAUDE.md b/CLAUDE.md index be7abf4a..373d6d49 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ core framework see `agentflow/CLAUDE.md`; for the TS client, docs, or playground for the monorepo overview see the workspace-root `CLAUDE.md`. - Package name (PyPI): `10xgraph-api` (formerly `10xscale-agentflow-cli`, last release 0.6.x) -- Version: `1.0.0rc3` (`pyproject.toml`; 0.7.0 was the first release under the 10xGraph name). `CLI_VERSION` +- Version: `1.0.0rc4` (`pyproject.toml`; 0.7.0 was the first release under the 10xGraph name). `CLI_VERSION` and `tenxgraph_api.__version__` are single-sourced from the installed distribution metadata (falling back to `pyproject.toml` only for a non-installed source checkout). - Requires: Python >= 3.12 · Status: `5 - Production/Stable` @@ -52,7 +52,7 @@ Public exports from the package root (`from tenxgraph_api import ...`): `BaseAut | Command | Purpose | Notable options | |---|---|---| | `10xgraph dev` | The only server command: local Uvicorn dev server (reload on) via `commands/api.py::APICommand`, opens the hosted playground when ready. `api` and `play` were removed (use `dev --no-open` / `dev`); production runs Gunicorn in the `10xgraph build` image | `--config/-c` (default `10xgraph.json`), `--host/-H`, `--port/-p` (8000), `--reload/--no-reload`, `--open/--no-open`, `-v/-q` | -| `10xgraph init` | Interactively scaffold a project (guided prompts pick dev vs production, auth, rate limit) | `--path/-p`, `--force/-f`, `--name`, `--template` (quick-start\|production), `--auth` (none\|jwt\|custom), `--rate-limit` (none\|memory\|redis), `--provider` (google\|openai\|anthropic), `--websocket/--no-websocket` (default off; writes `websocket.enabled`). Prod `pyproject.toml` depends on `10xgraph[]` and `10xgraph-api[jwt,redis,websocket]` as chosen, `--yes/-y`, `--non-interactive`, `--dry-run`. There is **no `--prod` flag** | +| `10xgraph init` | Interactively scaffold a project (guided prompts pick dev vs production, auth, rate limit) | `--path/-p`, `--force/-f`, `--name`, `--template` (quick-start\|production), `--auth` (none\|jwt\|custom), `--rate-limit` (none\|memory\|redis: `memory` = rate limit on, no `redis` key; `redis` = rate limit on plus `"redis": "${REDIS_URL}"` and, with auth, `"token_revocation": true`), `--provider` (google\|openai\|anthropic), `--websocket/--no-websocket` (default off; writes `websocket.enabled`). Prod `pyproject.toml` depends on `10xgraph[]` and `10xgraph-api[jwt,redis,websocket]` as chosen, `--yes/-y`, `--non-interactive`, `--dry-run`. There is **no `--prod` flag** | | `10xgraph build` | Generate a `Dockerfile` (and optionally `docker-compose.yml` / `k8s.yaml`). The builder stage strips CLI assets, runs `tenxgraph_api/_slim_image.py` (uninstalls typer/rich/questionary and what only they need, unless another package needs them) and removes pip, so the image carries no CLI code paths | `--output/-o`, `--force/-f`, `--python-version` (3.13), `--port`, `--docker-compose/--no-docker-compose`, `--k8s/--no-k8s`, `--service-name` | | `10xgraph eval` | Run agent evaluations; discovers `*_eval.py`/`eval_*.py`, runs cases (optionally `--parallel`), writes JSON and HTML reports to `eval_reports/` and opens the HTML report in the browser (only at an interactive terminal: skipped with `--no-open`, `--no-report`, `-q`, json/jsonl, `--non-interactive`, `CI`, non-TTY; `evaluation.open_report` in `10xgraph.json` sets the default). Simulations print per-turn step lines via `ProgressRun.step` | `--output/-o`, `--no-report`, `--threshold/-t`, `--parallel/-p`, `--max-concurrency/-c`, `--open/--no-open` | | `10xgraph test` | Run project tests via pytest (args after `--` forwarded verbatim) | `--coverage/-C`, `--html`, `-k`, path arg | @@ -92,22 +92,24 @@ Parsed by `tenxgraph_api/src/app/core/config/graph_config.py`. Supported keys: | `checkpointer` | `"module:attr"` -> a `BaseCheckpointer` | | `injectq` | `"module:attr"` -> an InjectQ container | | `store` | `"module:attr"` -> a `BaseStore` | -| `redis` | Redis URL string | +| `redis` | Redis URL (`$VAR` / `${VAR}` expanded; unset variable is a config error) of the API's cache store, a `RedisCacheStore` (also Valkey, Dragonfly, KeyDB). Unset falls back to `REDIS_URL`, then `InMemoryCacheStore` (per process) | +| `cache_store` | `"module:attr"` -> a `BaseCacheStore` instance or zero-arg factory (`core/cache_store/`); wins over `redis` / `REDIS_URL`. Selected and bound at boot by `core/cache_store/loader.py::load_and_bind_cache_store` (one INFO line `Cache store: (, )`), closed once in the lifespan. Holds the run locks and caps, idempotency, ownership cache L2, token revocation and rate-limit counters | | `rate_limit` | Object (see below) | | `ag_ui` | `{"enabled": bool}`, default off. Mounts `POST /v1/ag-ui` (AG-UI protocol, for CopilotKit and other AG-UI clients). Needs the `ag-ui` extra | | `websocket` | `enabled` (`false` leaves out `/v1/graph/ws` and `/v1/graph/live`; `true` without `websockets`/`wsproto` refuses to boot; unset keeps them and warns if no library; `routers/setup_router.py::_graph_router`), `max_connections` (1000), `max_connections_per_user` (10), `realtime_models` (models a `/v1/graph/live` client may pick), `realtime_client_overrides` (init options a client may set; default `voice`, `modalities`, `vad`; add `system_prompt` only for trusted callers) | | `runs` | `RunsConfig`: `thread_ids` (`server` default = only existing ids may be named on REST invoke/stream, `/v1/graph/ws` and checkpointer writes, a new thread is started by omitting the id; enforced by the ownership/rbac backends via `OwnershipAuthorizationBackend(server_thread_ids=...)`. AG-UI and `/v1/graph/live` are exempt: they pass `client_thread_ids=True` to `authorize` because their frontends make their own ids, and the caller owns a new one. `client` = any request may name a new thread, explicit opt-in), `config_keys` / `state_keys` (allow-lists for request config and `initial_state`/PUT state; 422 on other keys; `thread_id`, `run_id` always allowed), `max_concurrent_per_user` (HTTP invoke/stream/AG-UI runs in flight, default 10, 429 over it; `routers/graph/run_limits.py`), `timeout` (seconds; invoke 504, streams end with an error chunk), `max_parallel_tool_calls` (positive int, default unset = core's default 10 or env `TENXGRAPH_MAX_PARALLEL_TOOL_CALLS`; set into every run's config in `_prepare_input`, reserved from clients) | -| `token_revocation` | `"redis"` / `"memory"` / `"module:attr"` -> `core/auth/revocation.py::TokenRevocationBackend`, checked by `RequirePermission` (401), before every `/v1/graph/ws` run, and before and every 30 s during a `/v1/graph/live` session (`REALTIME_RECHECK_INTERVAL`, close 1008); backend errors fail closed (503) | +| `token_revocation` | `true`, or `{"enabled", "token_lifetime", "user_ttl"}` (`graph_config.TokenRevocationConfig`); the old strings (`"redis"`, `"memory"`, `module:attr`) are config errors naming `cache_store`. Revocations live in the cache store (`core/auth/revocation.py`: `revoke_token`, `revoke_user`, `is_revoked`), checked by `RequirePermission` (401), before every `/v1/graph/ws` run, and before and every 30 s during a `/v1/graph/live` session (`REALTIME_RECHECK_INTERVAL`, close 1008); cache store errors fail closed (503), and an unreachable cache store refuses the boot | | `media` | `delete_with_thread` (default true): delete an uploaded file when the last message or thread that references it is deleted. Links are recorded in the file's `threads` metadata (`MediaService.link_threads` / `unlink_threads`); files without a `threads` key (uploaded before link tracking) and stores without `update_metadata` (core older than the release that added it) are never auto-deleted. `false` still records links. Parsed by `graph_config.MediaConfig` | | `graph_introspection` | `"open"` or `"scoped"` (default scoped in production): `GET /v1/graph`, `/v1/graph/tools`, `/v1/graph:StateSchema` need an explicit `graph:read` scope (`RequirePermission(..., introspection=True)`) | `rate_limit` object: `enabled`, `requests` (default 100), `window` secs (60), `by` (`ip` | -`user` | `global`), `backend` (`memory` | `redis` | `custom`), `trusted_proxy_headers` (honour +`user` | `global`), `trusted_proxy_headers` (honour `X-Forwarded-For` only when true), `trusted_proxies` (networks the header is honoured from; empty means loopback and private networks only, `DEFAULT_TRUSTED_PROXIES`), `trusted_proxy_hops`, -`exclude_paths`, `fail_open` (on backend error: allow vs deny), -and for redis backend a `redis` sub-object `{ "url", "prefix" }` (or shorthand URL string). For -`custom`, bind a `BaseRateLimitBackend` in InjectQ. +`exclude_paths`, `fail_open` (on a cache store error: allow vs deny with 503 +`RATE_LIMIT_UNAVAILABLE`; default true). Counters live in the cache store +(`core/middleware/rate_limit/limiter.py::check`). `backend` and `redis` were removed in rc4; +either key is a config error naming `cache_store`. ## Auth @@ -142,13 +144,13 @@ and for redis backend a `redis` sub-object `{ "url", "prefix" }` (or shorthand U stop/fix + all checkpointer read/write/delete): a thread is accessible only to its owner; a foreign `invoke`/`stream` is rejected up front (403), never reaching the graph. - **Scalable, not a DB call per request.** Ownership is immutable, so it is cached by - `core/auth/ownership_resolver.py::ThreadOwnershipResolver`: in-process LRU (L1) + optional - shared Redis (L2, reuses `config.redis`/`settings.REDIS_URL`). The backing lookup is + `core/auth/ownership_resolver.py::ThreadOwnershipResolver`: in-process LRU (L1) + the + cache store as L2 (skipped when it is in-memory). The backing lookup is `BaseCheckpointer.aget_thread_owner` (implemented in pg/in-memory/sqlite; base raises `NotImplementedError`). Cache is evicted on thread delete (`CheckpointerService.delete_thread`), - which also leaves a tombstone for one L1 TTL (L1 + Redis) so the deleted id cannot be + which also leaves a tombstone for one L1 TTL (L1 + cache store) so the deleted id cannot be recreated by someone else while another worker still caches the old owner - (`ThreadOwnershipResolver.recently_deleted`); the L2 client is closed in the lifespan shutdown. + (`ThreadOwnershipResolver.recently_deleted`); the resolver never closes the cache store. - **Identity reuse:** with `rate_limit.by == "user"` the limiter's verified identity is reused by `RequirePermission` for backends with `BaseAuth.reusable_within_request = True` (`JwtAuth`), see `core/auth/identity_cache.py`. Look up the app container inside a request with @@ -186,15 +188,15 @@ and for redis backend a `redis` sub-object `{ "url", "prefix" }` (or shorthand U else a default `MediaFetchPolicy`, else skips (core without it). The policy lives in core; the API has no config for it. - **Per-thread run lock** (`routers/graph/thread_lock.py`): invoke, stream, each WS run and - AG-UI take a one-slot lease `runs:thread:` from `core/concurrency.py` (Redis when - configured, else per process; 60 s lease renewed while alive) before the graph starts, and + AG-UI take a one-slot lease `runs:thread:` from `core/leases.py::LeaseManager` over the cache store (shared + with a Redis cache store, else per process; 60 s lease renewed while alive) before the graph starts, and release it in `finally` (error, cancel, disconnect via `release_when_done`). Routers lock when the request names a thread (streaming routes before the response headers, so the 409 is real); `GraphService` locks a thread it creates (`_lock_new_thread`). Busy: wait up to `THREAD_BUSY_WAIT_SECONDS` (30), then 409 "thread is busy" in the standard error envelope (WS: error chunk `data.reason = "thread is busy"`, socket stays open). Request option `on_busy` (`"wait"` | `"cancel"`, top level next to `recursion_limit`; not on AG-UI): `cancel` calls - `GraphService.stop_graph` while waiting. `/v1/graph/live` takes the lock for the session (and a run slot) too, refused with a fatal `thread_busy` / `too_many_runs` error event and close 1013. Every entry point takes lock + run slot through `routers/graph/run_guard.py::acquire_run_guard`. A Redis error falls back to a per-process count; a refresh that finds its lease gone sets `Lease.lost` and takes it again. + `GraphService.stop_graph` while waiting. `/v1/graph/live` takes the lock for the session (and a run slot) too, refused with a fatal `thread_busy` / `too_many_runs` error event and close 1013. Every entry point takes lock + run slot through `routers/graph/run_guard.py::acquire_run_guard`. A cache store error falls back to a per-process count; a refresh that finds its lease gone sets `Lease.lost` and takes it again. - **Production error chunks:** `graph_service.sanitize_error_chunk` (called in `stream_chunks`, so stream, WS and AG-UI share it) replaces `data.reason` / `data.error` / `data.message` and `state.execution_meta.internal_data["error"]` of `ERROR` chunks with the @@ -216,8 +218,13 @@ and for redis backend a `redis` sub-object `{ "url", "prefix" }` (or shorthand U from `RunAgentInput.tools` go to the graph as the server-owned `remote_tools` run-config key (via `GraphService.stream_chunks(..., server_config=...)`); `RunAgentInput.resume` becomes the graph `resume` input. `GraphInputSchema.resume` also resumes over `/v1/graph/invoke|stream`. -- **Ping**: `GET /ping` (liveness). **Readiness**: `GET /health/ready` (checkpointer, Redis, - store; 200 or 503 with only ok/fail/skipped per component; `routers/ping/readiness.py`). +- **Ping**: `GET /ping` (liveness). **Readiness**: `GET /health/ready` (`aping()` on the + bound checkpointer, store and cache store; keys `checkpointer`, `store`, `cache`; 200 or 503 + with only ok/fail per component; a failed `cache` counts only when `token_revocation` is on + or `rate_limit.fail_open` is false; `routers/ping/readiness.py`). The startup check + (`routers/ping/startup_checks.py`, `STARTUP_CHECKS`) applies the same rule at boot: required + components retry for `STARTUP_CHECK_TIMEOUT_SECONDS` then refuse the boot; an optional + unreachable cache store is one WARNING and the server starts. Routers are wired in `routers/setup_router.py` (`init_routes`). The `a2a.py` / `a2ui.py` stubs were removed in the 0.5.0 readiness pass; they were fully commented out and never mounted. @@ -239,7 +246,8 @@ production. In production: set `MODE=production`, a non-`*` `ORIGINS`, and a str Middleware is pure ASGI. Order, outermost first: CORS, request id, security headers, trusted host, rate limit, request size, gzip (`setup_middleware.py`). Run and WebSocket connection caps -use `core/concurrency.py`: Redis leases when Redis is configured, per process otherwise. +are leases in the cache store (`core/leases.py`): shared with a Redis cache store, per process +with the in-memory one. Production error responses hide only 5xx details; raise `request_config.ClientInputError` for 422 messages that must survive production. diff --git a/README.md b/README.md index 2aaa9676..dd798c5f 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,10 @@ Other frameworks give you the graph. 10xGraph gives you the graph and the server - **Authentication.** `"auth": "jwt"`, or your own `BaseAuth` subclass. - **Scoped authorization on every endpoint.** Routes require `resource:action` scopes such as `graph:invoke`, `graph:stream` or `checkpointer:read`, checked by a pluggable `AuthorizationBackend`. The server refuses to start if any non-public route has no guard. - **Owner-only threads.** In production mode a thread is readable and writable only by the user who created it, on every step that touches it. Ownership checks are cached in-process and optionally in Redis, so they do not hit the database on each request. -- **Rate limiting.** Sliding-window limits with a memory, Redis or custom backend. +- **Rate limiting.** Sliding-window limits kept in the API's cache store: in memory, on Redis, or in your own `BaseCacheStore`. - **Production hardening.** Request size limits (10 MB default), security headers, error and log sanitization, startup warnings for insecure settings, and Snowflake IDs for multi-node deployments. - **Deployment files.** `10xgraph build --docker-compose --k8s` writes a `Dockerfile`, `.dockerignore`, `docker-compose.yml` and `k8s.yaml`. -- **A secure-by-default scaffold.** `10xgraph init --yes --template production --auth jwt --rate-limit redis` writes a project with JWT auth, owner-only thread access, a Redis rate limit, a prompt-injection validator, evals, tests, `.env.example` and a pre-commit config. +- **A secure-by-default scaffold.** `10xgraph init --yes --template production --auth jwt --rate-limit redis` writes a project with JWT auth, owner-only thread access, a Redis cache store for rate limits and token revocation, a prompt-injection validator, evals, tests, `.env.example` and a pre-commit config. Not built in: automatic per-tool permissions (user A may call `refund`, user B may not). Tools receive the caller's verified scopes and can check them with `tenxgraph.core.authz.has_scope`. @@ -86,7 +86,7 @@ pip install 10xgraph-api Optional extras; install only what you configure: ```bash -pip install "10xgraph-api[redis]" # Redis rate-limit / cache backend +pip install "10xgraph-api[redis]" # Redis cache store (locks, rate limits, revocation) pip install "10xgraph-api[jwt]" # JWT authentication pip install "10xgraph-api[media]" # Document text extraction (multimodal) pip install "10xgraph-api[images]" # Image resize / EXIF / format conversion (Pillow) @@ -323,7 +323,9 @@ The configuration file (`10xgraph.json`) defines your agent, authentication, and | `injectq` | string \| null | Path to an InjectQ container | | `store` | string \| null | Path to a data store | | `remote_tools` | array | Trusted schemas for tools executed by clients; attached at startup | -| `redis` | string \| null | Redis connection URL | +| `redis` | string \| null | Redis URL of the API's cache store (`${VAR}` expanded); falls back to `REDIS_URL` | +| `cache_store` | string \| null | Path to your own `BaseCacheStore`; wins over `redis` | +| `token_revocation` | bool \| object \| null | `true` rejects revoked tokens (`10xgraph revoke`); kept in the cache store | | `rate_limit` | object \| null | Sliding-window rate limiting configuration | | `thread_name_generator` | string \| null | Path to a custom thread name generator | | `ag_ui` | object | `{"enabled": true}` mounts `POST /v1/ag-ui` (needs the `ag-ui` extra) | diff --git a/SECURITY.md b/SECURITY.md index 8b1b8da9..98d2a1c8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -102,7 +102,7 @@ Some protections keep state per process. With more than one worker or instance: expiring leases, so the cap holds across workers and a crashed worker's slots free up on their own within a minute. Without Redis each process counts its own, so the real cap is the limit times the number of workers. -- **Rate limiting.** Use `"backend": "redis"` in `rate_limit`; the memory backend counts per - process. +- **Rate limiting.** The counters live in the cache store: set `redis` (or `REDIS_URL`, or a + `cache_store`) to share them; the in-memory store counts per process. - **Media extraction cache.** Extracted document text is cached per process. This only costs repeated extraction work, not correctness. diff --git a/config-editor-ui/src/icons.js b/config-editor-ui/src/icons.js index de0e1007..c3efb38e 100644 --- a/config-editor-ui/src/icons.js +++ b/config-editor-ui/src/icons.js @@ -40,7 +40,7 @@ export const ICONS = { export const SECTION_ICONS = { "get-running": "flag", environment: "terminal", - core: "flag", auth: "lock", authorization: "shield", token_revocation: "key", rate_limit: "gauge", + core: "flag", auth: "lock", authorization: "shield", token_revocation: "key", graph_introspection: "code", rate_limit: "gauge", observability: "activity", runs: "play", websocket: "bolt", ag_ui: "external", remote_tools: "wrench", media: "file", plugins: "layers", test: "flask", evaluation: "target", }; diff --git a/config-editor-ui/src/pages.jsx b/config-editor-ui/src/pages.jsx index 4b8d9afc..b8e4be74 100644 --- a/config-editor-ui/src/pages.jsx +++ b/config-editor-ui/src/pages.jsx @@ -2,7 +2,7 @@ import { copyCommand, go, setSection } from "./actions.js"; import { Field, fieldGroups } from "./fields.jsx"; import { SECTION_ICONS } from "./icons.js"; import { - PAGES, WIDGET_FIELD_IDS, getPath, isAgentReady, isOn, issueAt, issuesFor, navItem, section, store, + PAGES, WIDGET_FIELD_IDS, getPath, isAgentReady, isOn, issuesFor, navItem, section, store, valueOrDefault, } from "./store.js"; import { Callout, Icon, IssueCallout, Segmented } from "./ui.jsx"; @@ -58,12 +58,10 @@ function rateSummary() { const v = (k) => valueOrDefault(["rate_limit", k]); if (v("enabled") === false) return { text: "Settings kept, not enforced." }; const by = { ip: "per IP", user: "per user", global: "across all clients" }[v("by")] || v("by"); - const warn = issueAt("rate_limit.backend")?.level === "warning"; const custom = ["requests", "window", "by"].some((k) => getPath(store.config, ["rate_limit", k]) !== undefined && getPath(store.config, ["rate_limit", k]) !== store.fields.get(`rate_limit.${k}`)?.default); return { - text: `${v("requests")} requests per ${v("window")} s, ${by}${custom ? "" : " (default)"}.${warn ? " The memory backend counts per process." : ""}`, - warn, + text: `${v("requests")} requests per ${v("window")} s, ${by}${custom ? "" : " (default)"}.`, }; } diff --git a/config-editor-ui/src/store.js b/config-editor-ui/src/store.js index 54232982..27b2ff30 100644 --- a/config-editor-ui/src/store.js +++ b/config-editor-ui/src/store.js @@ -7,7 +7,7 @@ // are pages of their own rather than schema sections. export const NAV = [ { label: null, items: ["get-running"] }, - { label: "Before production", items: ["auth", "authorization", "rate_limit", "observability", "token_revocation"] }, + { label: "Before production", items: ["auth", "authorization", "rate_limit", "observability", "token_revocation", "graph_introspection"] }, { label: "Runs and streaming", items: ["runs", "websocket", "ag_ui", "remote_tools", "media"] }, { label: "Project wiring", items: ["plugins"] }, { label: "Developer tools", items: ["test", "evaluation"] }, diff --git a/pyproject.toml b/pyproject.toml index eb9d53d0..5a869779 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "10xgraph-api" -version = "1.0.0rc3" +version = "1.0.0rc4" description = "Production server for 10xGraph agents (formerly 10xscale-agentflow-cli). Serves a compiled graph over REST, SSE, WebSocket, realtime audio and AG-UI with JWT or custom auth, scoped authorization on every endpoint, owner-only threads and rate limiting, and generates Docker Compose and Kubernetes files. Includes the 10xgraph CLI. MIT licensed, self-hosted." readme = "README.md" license = "MIT" diff --git a/tenxgraph_api/__init__.py b/tenxgraph_api/__init__.py index 80a196f1..2a4401a3 100644 --- a/tenxgraph_api/__init__.py +++ b/tenxgraph_api/__init__.py @@ -9,6 +9,12 @@ # Lets expose few things the user suppose to use from .src.app.core.auth.base_auth import BaseAuth +from .src.app.core.cache_store import ( + BaseCacheStore, + InMemoryCacheStore, + RedisCacheStore, + SlotResult, +) from .src.app.utils.snowflake_id_generator import SnowFlakeIdGenerator from .src.app.utils.thread_name_generator import ( ThreadNameGenerator, @@ -17,6 +23,10 @@ __all__ = [ "BaseAuth", + "BaseCacheStore", + "InMemoryCacheStore", + "RedisCacheStore", + "SlotResult", "SnowFlakeIdGenerator", "ThreadNameGenerator", "__version__", diff --git a/tenxgraph_api/cli/commands/init.py b/tenxgraph_api/cli/commands/init.py index d148c7d2..a2973651 100644 --- a/tenxgraph_api/cli/commands/init.py +++ b/tenxgraph_api/cli/commands/init.py @@ -107,8 +107,8 @@ def _template_vars(context: dict[str, Any]) -> dict[str, Any]: The production pyproject.toml lists only what the answers need: the provider SDK and the Postgres + Redis checkpointer (``pg_checkpoint``) through the core extras, - PyJWT for JWT auth, redis (always, the checkpointer and the default rate limiter use - it), and websockets for WebSocket streaming / realtime. The generated Dockerfile + PyJWT for JWT auth, redis (always, the checkpointer and the API's cache store use it), + and websockets for WebSocket streaming / realtime. The generated Dockerfile installs exactly this list. Quick-start has no Postgres or Redis, so its core requirement is the provider extra alone. """ @@ -327,8 +327,7 @@ def _non_interactive_context( ) if resolved_rate_limit not in {"none", "memory", "redis"}: raise ValidationError( - f"Invalid rate-limit mode '{resolved_rate_limit}'. " - "Choose none, memory, or redis.", + f"Invalid rate-limit mode '{resolved_rate_limit}'. Choose none, memory, or redis.", field="rate_limit", ) @@ -453,11 +452,19 @@ def _prompt_user(self) -> dict | None: # noqa: PLR0911 context["auth"] = auth rate_limit = prompts.select( - "Rate limiting backend?", + "Rate limiting?", [ Choice("none", "None", "No request throttling"), - Choice("memory", "Memory", "Per-process counters — single instance only"), - Choice("redis", "Redis", "Shared counters — requires REDIS_URL"), + Choice( + "memory", + "On, no shared cache", + "Counters per process unless REDIS_URL is set; single instance only", + ), + Choice( + "redis", + "On, Redis cache store", + "Shared counters, locks and token revocation; requires REDIS_URL", + ), ], default="redis", ) @@ -544,7 +551,8 @@ def _print_summary(self, context: dict) -> None: rows.append( ( "Rate limit", - f"{rl.capitalize()} · {context.get('rl_requests', 100)} req / " + f"{'Redis cache store' if rl == 'redis' else 'Per process'} · " + f"{context.get('rl_requests', 100)} req / " f"{context.get('rl_window', 60)}s · {rl_by}{proxy}", ) ) @@ -593,24 +601,23 @@ def _build_config(self, context: dict, is_prod: bool) -> dict: config["injectq"] = "graph.agent:container" rate_limit = context["rate_limit"] + if rate_limit == "redis": + # The API's cache store (rate limit counters, run locks and caps, idempotency, + # ownership cache, token revocation), resolved from REDIS_URL in .env. + config["redis"] = "${REDIS_URL}" if rate_limit != "none": config["rate_limit"] = { "enabled": True, - "backend": rate_limit, "requests": context.get("rl_requests", 100), "window": context.get("rl_window", 60), "by": context.get("rl_by", "user" if auth in ("jwt", "custom") else "ip"), "trusted_proxy_headers": context.get("rl_trusted_proxy", False), "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"], } - if rate_limit == "redis": - # Resolved from REDIS_URL in .env; the redis backend refuses to start - # without a URL or a Redis client bound in InjectQ. - config["rate_limit"]["redis"] = {"url": "${REDIS_URL}"} - if auth in ("jwt", "custom"): - # Redis is configured, so a leaked or logged-out token can be revoked - # for every worker. See `10xgraph revoke` and the token_revocation docs. - config["token_revocation"] = "redis" # noqa: S105 - a backend name, not a secret # nosec B105 + if rate_limit == "redis" and auth in ("jwt", "custom"): + # The cache store is shared, so a leaked or logged-out token can be revoked for + # every worker. See `10xgraph revoke` and the token_revocation docs. + config["token_revocation"] = True return config diff --git a/tenxgraph_api/cli/commands/revoke.py b/tenxgraph_api/cli/commands/revoke.py index 75de1906..1101b005 100644 --- a/tenxgraph_api/cli/commands/revoke.py +++ b/tenxgraph_api/cli/commands/revoke.py @@ -1,13 +1,17 @@ """Revoke a token or every token of a user: ``10xgraph revoke``. -Writes to the same Redis the server reads (``token_revocation: "redis"`` in 10xgraph.json), -so the revocation applies to every worker at once. The server needs no restart. +Writes to the cache store the server reads (``"token_revocation": true`` in 10xgraph.json), +picked the same way the server picks it: ``cache_store``, else ``redis``, else ``REDIS_URL``. +With a shared store (Redis) the revocation applies to every worker at once; the server needs +no restart. ``--redis-url`` overrides the selection. """ from __future__ import annotations import asyncio import os +import sys +from types import SimpleNamespace from typing import Any from tenxgraph_api.cli.commands import BaseCommand @@ -19,23 +23,42 @@ class RevokeCommand(BaseCommand): """Revoke one token (``--jti``) or all of a user's older tokens (``--user``).""" - def _redis_url(self, redis_url: str | None, config: str) -> str: - if redis_url: - return redis_url - env_url = os.environ.get("REDIS_URL") - if env_url: - return env_url + def _cache_store(self, redis_url: str | None, config: str) -> tuple[Any, float | None]: + """The cache store to write to and the default ``user_ttl`` from the project config.""" + from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, RedisCacheStore + from tenxgraph_api.src.app.core.cache_store.loader import build_cache_store + from tenxgraph_api.src.app.core.config.graph_config import ( + DEFAULT_USER_REVOCATION_TTL, + GraphConfig, + ) + path = resolve_default_config(config) - if path.exists(): - from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + graph_config = GraphConfig(str(path)) if path.exists() else None + user_ttl: float | None = DEFAULT_USER_REVOCATION_TTL + if graph_config is not None: + settings = graph_config.token_revocation # raises for a removed value + if settings is not None: + user_ttl = settings.user_ttl + if redis_url: + return RedisCacheStore.from_url(redis_url), user_ttl - configured = GraphConfig(str(path)).redis_url - if isinstance(configured, str) and configured and "${" not in configured: - return configured - raise ValidationError( - "No Redis URL. Pass --redis-url, set REDIS_URL, or set `redis` in 10xgraph.json.", - field="redis_url", + if graph_config is not None: + # A ``cache_store: "module:attr"`` imports from the project, like the server does. + root = str(path.resolve().parent) + if root not in sys.path: + sys.path.insert(0, root) + # GraphConfig has loaded the project's env file, so REDIS_URL is read after it. + store, source = build_cache_store( + graph_config, SimpleNamespace(REDIS_URL=os.environ.get("REDIS_URL")) ) + if isinstance(store, InMemoryCacheStore): + raise ValidationError( + f"The cache store is in-memory ({source}): a CLI process cannot reach the " + "server's memory. Set `redis` in 10xgraph.json, REDIS_URL or a `cache_store` " + "(the server must use the same one), or pass --redis-url.", + field="redis_url", + ) + return store, user_ttl def execute( self, @@ -50,19 +73,18 @@ def execute( try: if bool(user) == bool(jti): raise ValidationError("Pass exactly one of --user or --jti.", field="user") - url = self._redis_url(redis_url, config) + store, user_ttl = self._cache_store(redis_url, config) - from tenxgraph_api.src.app.core.auth.revocation import RedisTokenRevocation + from tenxgraph_api.src.app.core.auth import revocation async def run() -> None: - backend = RedisTokenRevocation(url) try: if user: - await backend.revoke_user(user, before=before) + await revocation.revoke_user(store, user, before=before, ttl=user_ttl) else: - await backend.revoke_token(str(jti), expires_at=expires_at) + await revocation.revoke_token(store, str(jti), expires_at=expires_at) finally: - await backend.aclose() + await store.aclose() asyncio.run(run()) if user: diff --git a/tenxgraph_api/cli/config_editor/schema.py b/tenxgraph_api/cli/config_editor/schema.py index 093eee7e..d29d1ee5 100644 --- a/tenxgraph_api/cli/config_editor/schema.py +++ b/tenxgraph_api/cli/config_editor/schema.py @@ -22,8 +22,8 @@ Field shape:: { - "path": ["rate_limit", "redis", "url"], - "label": "Redis URL", + "path": ["rate_limit", "requests"], + "label": "Requests per window", "type": "text" | "number" | "bool" | "select" | "list", "help": "...", "placeholder": "...", @@ -37,7 +37,7 @@ "unlimited_value": 0, # number only: an "Unlimited" checkbox that writes this value "null_label": "No limit", # number only: a choice that writes JSON null "exclusive_min": 0, # number only: the value must be greater than this - "group": "Backend", # fields sharing a group render together + "group": "Limits", # fields sharing a group render together "show_if": {"path": [...], "equals": value}, } @@ -87,6 +87,7 @@ "authorization", "token_revocation", "graph_introspection", + "cache_store", "rate_limit", "websocket", "runs", @@ -182,8 +183,64 @@ }, { "id": "token_revocation", - "title": "Access extras", - "description": "Who may read graph internals, and rejecting revoked tokens early.", + "title": "Token revocation", + "description": ( + "Reject revoked tokens (10xgraph revoke) before they expire. Revocations live in " + "the API's cache store." + ), + "key": "token_revocation", + "optional": True, + "off_summary": "Tokens are accepted until they expire.", + "default": True, + "fields": [ + { + "path": ["token_revocation", "enabled"], + "default": True, + "label": "Enabled", + "type": "bool", + "help": "Turn off to keep the settings without checking revocations.", + }, + { + "path": ["token_revocation", "token_lifetime"], + "label": "Token lifetime (seconds)", + "type": "number", + "exclusive_min": 0, + "step": 1, + "placeholder": "3600", + "help": ( + "The longest lifetime of the tokens you issue. Dates a token that has no " + "iat claim from its exp, for user revocations." + ), + }, + { + "path": ["token_revocation", "user_ttl"], + "default": 2592000, + "label": "User revocation kept (seconds)", + "type": "number", + "exclusive_min": 0, + "step": 1, + "null_label": "Until deleted", + "help": "How long a revoke --user entry is kept by default.", + }, + ], + "notes": [ + "Fails closed: while the cache store is unreachable, authenticated requests get " + "503, and the server refuses to start.", + "Share revocations between workers with a shared cache store (redis, REDIS_URL or " + "cache_store under Plugins). With the in-memory store a revocation reaches " + "one process only, and 10xgraph revoke cannot reach it.", + ], + "env": [ + { + "name": "REDIS_URL", + "purpose": "Redis for the cache store when neither redis nor cache_store is set.", + }, + ], + }, + { + "id": "graph_introspection", + "title": "Graph introspection", + "description": "Who may read the graph structure, tool schemas and state schema.", "key": None, "optional": False, "fields": [ @@ -198,22 +255,6 @@ "requires the graph:read scope." ), }, - { - "path": ["token_revocation"], - "label": "Revocation list", - "type": "text", - "placeholder": "redis", - "help": ( - "redis (shared, uses the redis key or REDIS_URL), memory (one process), " - "or module:attr for your own TokenRevocationBackend. Empty means off." - ), - }, - ], - "env": [ - { - "name": "REDIS_URL", - "purpose": "Redis for the revocation list when no redis key is set.", - }, ], }, { @@ -225,7 +266,6 @@ "off_summary": "Requests are not rate limited.", "default": { "enabled": True, - "backend": "memory", "requests": 100, "window": 60, "by": "ip", @@ -266,44 +306,15 @@ "options": ["ip", "user", "global"], "help": "Count requests per client IP, per authenticated user, or all together.", }, - { - "path": ["rate_limit", "backend"], - "group": "Backend", - "default": "memory", - "label": "Backend", - "type": "select", - "options": ["memory", "redis", "custom"], - "help": ( - "memory counts per process. Use redis for multiple workers. custom " - "expects a BaseRateLimitBackend bound in InjectQ." - ), - }, - { - "path": ["rate_limit", "redis", "url"], - "group": "Backend", - "label": "Redis URL", - "type": "text", - "placeholder": "${REDIS_URL}", - "help": "Supports $VAR and ${VAR}. Not needed if Redis is bound in InjectQ.", - "show_if": {"path": ["rate_limit", "backend"], "equals": "redis"}, - }, - { - "path": ["rate_limit", "redis", "prefix"], - "group": "Backend", - "label": "Redis key prefix", - "type": "text", - "placeholder": "10xgraph:rate-limit", - "show_if": {"path": ["rate_limit", "backend"], "equals": "redis"}, - }, { "path": ["rate_limit", "fail_open"], - "group": "Backend", + "group": "Limits", "default": True, "label": "Fail open", "type": "bool", "help": ( - "Allow requests when the backend errors. Off means deny them with 503 " - "RATE_LIMIT_UNAVAILABLE." + "Allow requests while the cache store errors. Off means deny them with 503 " + "RATE_LIMIT_UNAVAILABLE, and the server refuses to start without it." ), }, { @@ -350,13 +361,14 @@ ), }, ], + "notes": [ + "Counters live in the API's cache store: shared between workers with redis, " + "REDIS_URL or cache_store (see Plugins), per process otherwise.", + ], "env": [ { "name": "REDIS_URL", - "purpose": ( - "Not read by rate limiting itself. Reference it as ${REDIS_URL} in " - "rate_limit.redis.url; required then when the backend is redis." - ), + "purpose": "Redis for the cache store when neither redis nor cache_store is set.", }, ], }, @@ -364,14 +376,14 @@ "id": "websocket", "title": "WebSocket", "description": ( - "WebSocket streaming and realtime audio, and per-process caps on concurrent " - "connections." + "WebSocket streaming and realtime audio, and caps on concurrent connections " + "(shared between workers with a shared cache store, per process otherwise)." ), "key": "websocket", "optional": True, "off_summary": ( "/v1/graph/ws and /v1/graph/live stay mounted with the default caps " - "(1000 per process, 10 per user)." + "(1000 connections, 10 per user)." ), "default": {"max_connections": 1000, "max_connections_per_user": 10}, "fields": [ @@ -397,7 +409,10 @@ "step": 1, "placeholder": "1000", "unlimited_value": 0, - "help": "Per worker process. Defaults to 1000; 0 or null means unlimited.", + "help": ( + "Per worker process unless the cache store is shared. Defaults to 1000; 0 " + "or null means unlimited." + ), }, { "path": ["websocket", "max_connections_per_user"], @@ -487,7 +502,10 @@ "placeholder": "10", "default": 10, "unlimited_value": 0, - "help": "HTTP runs in flight per user, per process. 0 or null means unlimited.", + "help": ( + "HTTP runs in flight per user (per process unless the cache store is " + "shared). 0 or null means unlimited." + ), }, { "path": ["runs", "timeout"], @@ -832,20 +850,33 @@ }, { "path": ["redis"], - "group": "Connections", + "group": "Cache store", "label": "Redis URL", "type": "text", - "placeholder": "redis://localhost:6379/0", + "placeholder": "${REDIS_URL}", + "help": ( + "The API's cache store (run locks and caps, idempotency, rate limits, " + "ownership cache, token revocation) on Redis, Valkey, Dragonfly or KeyDB. " + "Supports $VAR and ${VAR}. Empty falls back to REDIS_URL, then to an " + "in-memory store per process." + ), + }, + { + "path": ["cache_store"], + "group": "Cache store", + "label": "Custom cache store", + "type": "text", + "placeholder": "graph.dependencies:cache_store", "help": ( - "Shared cache for ownership checks. Used as-is: $VAR is not expanded " - "here, so leave it empty to fall back to REDIS_URL." + "module:attr of a BaseCacheStore instance or zero-argument factory. Wins " + "over the Redis URL." ), }, ], "env": [ { "name": "REDIS_URL", - "purpose": "Redis for the ownership cache when the redis key above is empty.", + "purpose": "Redis for the cache store when neither redis nor cache_store is set.", }, ], }, @@ -955,8 +986,10 @@ { "name": "STARTUP_CHECKS", "purpose": ( - "Refuse to start while a configured Postgres, Redis or store is unreachable, " - "in development and production. Defaults to true; false skips the check." + "Refuse to start while the checkpointer, the store or a required cache store " + "is unreachable (the cache store only for token_revocation or rate_limit " + "with fail_open false), in development and production. Defaults to true; " + "false skips the check." ), }, { diff --git a/tenxgraph_api/cli/config_editor/static/app.js b/tenxgraph_api/cli/config_editor/static/app.js index 7b4e32c9..f05d3362 100644 --- a/tenxgraph_api/cli/config_editor/static/app.js +++ b/tenxgraph_api/cli/config_editor/static/app.js @@ -1,3 +1,3 @@ -var be,x,mt,bn,H,ut,ht,_t,De,fe,X,gt,Fe,He,Le,vn,_e={},ge=[],yn=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,ve=Array.isArray;function P(e,t){for(var r in t)e[r]=t[r];return e}function qe(e){e&&e.parentNode&&e.parentNode.removeChild(e)}function kn(e,t,r){var o,s,a,i={};for(a in t)a=="key"?o=t[a]:a=="ref"?s=t[a]:i[a]=t[a];if(arguments.length>2&&(i.children=arguments.length>3?be.call(arguments,2):r),typeof e=="function"&&e.defaultProps!=null)for(a in e.defaultProps)i[a]===void 0&&(i[a]=e.defaultProps[a]);return me(e,i,o,s,null)}function me(e,t,r,o,s){var a={type:e,props:t,key:r,ref:o,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:s??++mt,__i:-1,__u:0};return s==null&&x.vnode!=null&&x.vnode(a),a}function w(e){return e.children}function he(e,t){this.props=e,this.context=t}function F(e,t){if(t==null)return e.__?F(e.__,e.__i+1):null;for(var r;tt&&H.sort(_t),e=H.shift(),t=H.length,wn(e)}finally{H.length=xe.__r=0}}function bt(e,t,r,o,s,a,i,d,c,p,m){var g,u,f,v,$,S,C=o&&o.__k||ge,b=t.length;for(c=Cn(r,t,C,c,b),g=0;g0?i=e.__k[a]=me(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):e.__k[a]=i,c=a+u,i.__=e,i.__b=e.__b+1,d=null,(p=i.__i=Sn(i,r,c,g))!=-1&&(g--,(d=r[p])&&(d.__u|=2)),d==null||d.__v==null?(p==-1&&(s>m?u--:sc?u--:u++,i.__u|=4))):e.__k[a]=null;if(g)for(a=0;a(m?1:0)){for(s=r-1,a=r+1;s>=0||a=0?s--:a++])!=null&&(2&p.__u)==0&&d==p.key&&c==p.type)return i}return-1}function pt(e,t,r){t[0]=="-"?e.setProperty(t,r??""):e[t]=r==null?"":typeof r!="number"||yn.test(t)?r:r+"px"}function pe(e,t,r,o,s){var a,i;e:if(t=="style")if(typeof r=="string")e.style.cssText=r;else{if(typeof o=="string"&&(e.style.cssText=o=""),o)for(t in o)r&&t in r||pt(e.style,t,"");if(r)for(t in r)o&&r[t]==o[t]||pt(e.style,t,r[t])}else if(t[0]=="o"&&t[1]=="n")a=t!=(t=t.replace(gt,"$1")),i=t.toLowerCase(),t=i in e||t=="onFocusOut"||t=="onFocusIn"?i.slice(2):t.slice(2),e.l||(e.l={}),e.l[t+a]=r,r?o?r[X]=o[X]:(r[X]=Fe,e.addEventListener(t,a?Le:He,a)):e.removeEventListener(t,a?Le:He,a);else{if(s=="http://www.w3.org/2000/svg")t=t.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if(t!="width"&&t!="height"&&t!="href"&&t!="list"&&t!="form"&&t!="tabIndex"&&t!="download"&&t!="rowSpan"&&t!="colSpan"&&t!="role"&&t!="popover"&&t in e)try{e[t]=r??"";break e}catch{}typeof r=="function"||(r==null||r===!1&&t[4]!="-"?e.removeAttribute(t):e.setAttribute(t,t=="popover"&&r==1?"":r))}}function ft(e){return function(t){if(this.l){var r=this.l[t.type+e];if(t[fe]==null)t[fe]=Fe++;else if(t[fe]0?e:ve(e)?e.map(wt):e.constructor!==void 0?null:P({},e)}function $n(e,t,r,o,s,a,i,d,c){var p,m,g,u,f,v,$,S=r.props||_e,C=t.props,b=t.type;if(b=="svg"?s="http://www.w3.org/2000/svg":b=="math"?s="http://www.w3.org/1998/Math/MathML":s||(s="http://www.w3.org/1999/xhtml"),a!=null){for(p=0;p=r.__.length&&r.__.push({}),r.__[e]}function we(e){return ke=1,Je(Rt,e)}function Je(e,t,r){var o=Ge(ee++,2);if(o.t=e,!o.__c&&(o.__=[r?r(t):Rt(void 0,t),function(d){var c=o.__N?o.__N[0]:o.__[0],p=o.t(c,d);c!==p&&(o.__N=[p,o.__[1]],o.__c.setState({}))}],o.__c=y,!y.__f)){var s=function(d,c,p){if(!o.__c.__H)return!0;var m=!1,g=o.__c.props!==d;if(o.__c.__H.__.some(function(f){if(f.__N){m=!0;var v=f.__[0];f.__=f.__N,f.__N=void 0,v!==f.__[0]&&(g=!0)}}),a){var u=a.call(this,d,c,p);return m?u||g:u}return!m||g};y.__f=!0;var a=y.shouldComponentUpdate,i=y.componentWillUpdate;y.componentWillUpdate=function(d,c,p){if(this.__e){var m=a;a=void 0,s(d,c,p),a=m}i&&i.call(this,d,c,p)},y.shouldComponentUpdate=s}return o.__N||o.__}function M(e,t){var r=Ge(ee++,3);!k.__s&&Nt(r.__H,t)&&(r.__=e,r.u=t,y.__H.__h.push(r))}function Ce(e){return ke=5,An(function(){return{current:e}},[])}function An(e,t){var r=Ge(ee++,7);return Nt(r.__H,t)&&(r.__=e(),r.__H=t,r.__h=e),r.__}function En(){for(var e;e=zt.shift();){var t=e.__H;if(e.__P&&t)try{t.__h.some(ye),t.__h.some(Ve),t.__h=[]}catch(r){t.__h=[],k.__e(r,e.__v)}}}k.__b=function(e){y=null,Tt&&Tt(e)},k.__=function(e,t){e&&t.__k&&t.__k.__m&&(e.__m=t.__k.__m),Pt&&Pt(e,t)},k.__r=function(e){At&&At(e),ee=0;var t=(y=e.__c).__H;t&&(Be===y?(t.__h=[],y.__h=[],t.__.some(function(r){r.__N&&(r.__=r.__N),r.u=r.__N=void 0})):(t.__h.some(ye),t.__h.some(Ve),t.__h=[],ee=0)),Be=y},k.diffed=function(e){Et&&Et(e);var t=e.__c;t&&t.__H&&(t.__H.__h.length&&(zt.push(t)!==1&&$t===k.requestAnimationFrame||(($t=k.requestAnimationFrame)||In)(En)),t.__H.__.some(function(r){r.u&&(r.__H=r.u,r.u=void 0)})),Be=y=null},k.__c=function(e,t){t.some(function(r){try{r.__h.some(ye),r.__h=r.__h.filter(function(o){return!o.__||Ve(o)})}catch(o){t.some(function(s){s.__h&&(s.__h=[])}),t=[],k.__e(o,r.__v)}}),It&&It(e,t)},k.unmount=function(e){Ot&&Ot(e);var t,r=e.__c;r&&r.__H&&(r.__H.__.some(function(o){try{ye(o)}catch(s){t=s}}),r.__H=void 0,t&&k.__e(t,r.__v))};var Mt=typeof requestAnimationFrame=="function";function In(e){var t,r=function(){clearTimeout(o),Mt&&cancelAnimationFrame(t),setTimeout(e)},o=setTimeout(r,35);Mt&&(t=requestAnimationFrame(r))}function ye(e){var t=y,r=e.__c;typeof r=="function"&&(e.__c=void 0,r()),y=t}function Ve(e){var t=y;e.__c=e.__(),y=t}function Nt(e,t){return!e||e.length!==t.length||t.some(function(r,o){return r!==e[o]})}function Rt(e,t){return typeof t=="function"?t(e):t}var Se=[{label:null,items:["get-running"]},{label:"Before production",items:["auth","authorization","rate_limit","observability","token_revocation"]},{label:"Runs and streaming",items:["runs","websocket","ag_ui","remote_tools","media"]},{label:"Project wiring",items:["plugins"]},{label:"Developer tools",items:["test","evaluation"]},{label:"Reference",items:["environment"]}],N={"get-running":{title:"Get running",icon:"flag",description:"Only the agent graph is required. Everything else is optional."},environment:{title:"Server environment (.env)",icon:"terminal",description:"Variables the server reads from .env. The editor never reads that file, so values are not shown."}},Ke={"auth.path":"auth-path",authorization:"authz-path"},jt="tg-config-token",Dt=!1;function On(){let e=new URLSearchParams(location.hash.slice(1)).get("token")||"";try{e&&sessionStorage.setItem(jt,e);let t=sessionStorage.getItem(jt)||"";return Dt=t!==""&&t===(e||t),e||t}catch{return e}}var te=On(),Ye=e=>Dt||!te?`#${e}`:`#token=${encodeURIComponent(te)}&page=${e}`;function Ht(){let e=location.hash.slice(1);if(e.includes("=")){let t=new URLSearchParams(e);return t.get("page")||t.get("section")||""}try{return decodeURIComponent(e)}catch{return e}}var l={schema:null,sections:[],nav:[],fields:new Map,config:{},version:null,exists:!1,path:"",parseError:null,saved:"",validated:null,issues:[],page:"get-running",query:"",sideOpen:!0,sideRender:!0,sideShown:!0,drawerOpen:!1,tab:"json",copied:!1,busy:null,toolParams:{},toasts:[],fatal:null},Lt=()=>{},Ft=e=>{Lt=e};function _(e){e&&e(l),Lt()}var qt=e=>JSON.parse(JSON.stringify(e)),I=()=>JSON.stringify(l.config),z=e=>e!==null&&typeof e=="object"&&!Array.isArray(e),q=(e,t)=>t.reduce((r,o)=>z(r)?r[o]:void 0,e),ne=(e,t)=>`${e} ${t}${e===1?"":"s"}`;function Ut(e,t,r){let o=e;for(let s of t.slice(0,-1))z(o[s])||(o[s]={}),o=o[s];o[t[t.length-1]]=r}function Ze(e,t){let r=[],o=e;for(let s of t.slice(0,-1)){if(!z(o))return;r.push([o,s]),o=o[s]}z(o)&&delete o[t[t.length-1]];for(let s=r.length-1;s>=1;s--){let[a,i]=r[s];if(z(a[i])&&Object.keys(a[i]).length===0)delete a[i];else break}}function G(e){let t=q(l.config,e);return t!==void 0?t:l.fields.get(e.join("."))?.default}var Pn=(e,t)=>JSON.stringify(e)===JSON.stringify(t),T=e=>l.sections.find(t=>t.id===e),R=()=>I()!==l.saved,$e=e=>!e.optional||e.key in l.config&&l.config[e.key]!==null,J=e=>l.issues.filter(t=>t.section===e),U=e=>l.issues.find(t=>t.path===e);function re(e=l.issues){let t=e.filter(r=>r.level==="error").length;return{errors:t,warnings:e.length-t}}var Te=e=>Ke[e]||`f-${e.replaceAll(".","-")}`,Qe=e=>{let t=q(l.config,e.path);return t!==void 0&&!Pn(t,e.default)},Wt=e=>e.id==="core"?!1:e.optional?$e(e):e.fields.some(Qe);function Bt(e){let t=J(e);return t.find(r=>r.level==="error")||t[0]||null}var Vt=e=>!!e.show_if&&G(e.show_if.path)!==e.show_if.equals;function Gt(e){let r=l.fields.get(e.show_if.path.join("."))?.label||e.show_if.path.join("."),o=e.show_if.equals;return`Used when ${r} is ${o===!0?"on":o===!1?"off":String(o)}.`}var oe=e=>{let t=T(e);return N[e]?{id:e,...N[e]}:t?{id:e,title:t.title,description:t.description,icon:null}:null},Jt=()=>/^[\w.]+:[\w.]+$/.test(String(l.config.agent||""));async function et(e,t,r){let o={"X-10xGraph-Token":te};r!==void 0&&(o["Content-Type"]="application/json");let s=await fetch(t,{method:e,headers:o,body:r===void 0?void 0:JSON.stringify(r)}),a={};try{a=await s.json()}catch{}return{ok:s.ok,status:s.status,data:a}}var Xe=(e,t)=>_(r=>{r.fatal={title:e,text:t}});async function Zt(){if(!te)return Xe("Open the link printed by 10xgraph config","This page needs the session token included in that link.");let e;try{e=await et("GET","/api/state")}catch{return Xe("Cannot reach the config server","Check that 10xgraph config is still running in your terminal, then reload.")}if(!e.ok)return Xe("This link has expired",e.data.error||"Restart 10xgraph config and open the new link it prints.");let t=e.data;_(r=>{r.schema=t.schema,r.sections=t.schema.sections;let o=new Set(Se.flatMap(i=>i.items)),s=r.sections.map(i=>i.id).filter(i=>i!=="core"&&!o.has(i));r.nav=Se.map((i,d)=>({label:i.label,items:dN[c]||T(c)):[...s,...i.items]}));for(let i of r.sections)for(let d of i.fields)r.fields.set(d.path.join("."),d);r.config=t.config,r.version=t.version,r.exists=t.exists,r.path=t.path,r.parseError=t.parse_error,r.issues=t.issues;let a=t.exists&&!t.parse_error;r.saved=a?I():"",r.validated=a?I():null}),Ee({focus:!1}),history.replaceState(null,"",Ye(l.page))}var Kt=null;async function Qt(){let e=I();if(!(!l.schema||l.validated===e||Kt===e)&&!Object.values(l.toolParams).some(t=>t.error)){Kt=e;try{let t=await et("POST","/api/validate",{config:JSON.parse(e)});if(!t.ok)return;_(r=>{r.issues=t.data.issues,r.validated=e})}catch{}}}async function Ae(){if(!(l.busy||!l.schema)){if(l.exists&&!R()&&!l.parseError)return j("success","No changes to save.");if(Mn()){_(e=>{e.busy="save"});try{let e=await et("POST","/api/save",{config:l.config,version:l.version});if(e.data.issues&&(l.issues=e.data.issues,l.validated=I()),e.ok){l.version=e.data.version,l.config=e.data.config,l.toolParams={},l.validated=I(),l.saved=I(),l.exists=!0,l.parseError=null;let t=e.data.path.split(/[\\/]/).pop();j("success",e.data.backup?`Saved ${t}. The previous version is in ${t}.bak.`:`Created ${t}.`)}else if(e.status===409)j("error",e.data.error,{label:"Reload",run:()=>location.reload()});else if(e.status===422){let{errors:t}=re();j("error",`Not saved: ${ne(t,"error")} to fix first.`,{label:"Show",run:zn})}else j("error",e.data.error||"Save failed.")}catch{j("error","Cannot reach the config server.")}finally{_(e=>{e.busy=null})}}}}function Mn(){let e=Object.values(l.toolParams).some(t=>t.error);return e&&(O("remote_tools"),j("error","A remote tool has invalid parameters JSON.")),!e}function Xt(e,t){_(r=>{t?r.config[e.key]=qt(e.default):delete r.config[e.key],e.key==="remote_tools"&&(r.toolParams={})})}function zn(){let e=l.issues.find(t=>t.level==="error")||l.issues[0];e&&O(e.section,Te(e.path))}var Nn=()=>window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth";function Rn(e){return e==="core"?"get-running":N[e]||T(e)?e:"get-running"}function Ee({focus:e=!0}={}){_(t=>{t.page=Rn(Ht()),t.query="",t.drawerOpen=!1}),e&&requestAnimationFrame(()=>{window.scrollTo(0,0),document.getElementById("page-title")?.focus({preventScroll:!0})})}function O(e,t){let r=Ye(e);location.hash!==r&&history.pushState(null,"",r),Ee(),t&&requestAnimationFrame(()=>requestAnimationFrame(()=>{let o=document.getElementById(t);o&&(o.scrollIntoView({block:"center",behavior:Nn()}),o.focus({preventScroll:!0}))}))}function Ie(e){_(t=>{t.query=e})}var jn=()=>window.matchMedia("(prefers-reduced-motion: reduce)").matches,Yt;function K(e,{instant:t=!1}={}){try{localStorage.setItem("tg-sidebar",e?"shown":"hidden")}catch{}clearTimeout(Yt);let r=t||jn();_(o=>{o.sideOpen=e,e?(o.sideRender=!0,o.sideShown=r):o.sideShown=!1,!e&&r&&(o.sideRender=!1)}),e&&!r&&requestAnimationFrame(()=>requestAnimationFrame(()=>_(o=>{o.sideShown=!0}))),!e&&!r&&(Yt=setTimeout(()=>_(o=>{o.sideOpen||(o.sideRender=!1)}),200))}function tt(){l.sideOpen||K(!0,{instant:!0}),requestAnimationFrame(()=>{let e=document.getElementById("q");e?.focus(),e?.select()})}function en(e){_(t=>{e&&(t.tab=e),t.drawerOpen=!0})}var Oe=()=>_(e=>{e.drawerOpen=!1});async function tn(){let e="10xgraph dev";try{await navigator.clipboard.writeText(e)}catch{try{let t=document.createElement("textarea");t.value=e,document.body.append(t),t.select(),document.execCommand("copy"),t.remove()}catch{}}_(t=>{t.copied=!0}),setTimeout(()=>_(t=>{t.copied=!1}),2e3)}async function nn(){try{await navigator.clipboard.writeText(JSON.stringify(l.config,null,2)+` -`),j("success","Copied 10xgraph.json to the clipboard.")}catch{j("error","The clipboard is not available here.")}}var Dn=0;function j(e,t,r){let o=++Dn;_(s=>{s.toasts=[...s.toasts,{id:o,kind:e,message:t,action:r}]}),setTimeout(()=>Pe(o),r?8e3:4e3)}function Pe(e){_(t=>{t.toasts=t.toasts.filter(r=>r.id!==e)})}var rn={box:'',key:'',shield:'',gauge:'',arrows:'',activity:'',layers:'',wrench:'',flask:'',target:'',checkCircle:'',xCircle:'',alert:'',info:'',file:'',copy:'',x:'',plus:'',trash:'',panel:'',chevronDown:'',expand:'',collapse:'',power:'',spinner:'',book:'',flag:'',lock:'',play:'',bolt:'',terminal:'',search:'',chevronRight:'',code:'',circle:'',external:''},W={"get-running":"flag",environment:"terminal",core:"flag",auth:"lock",authorization:"shield",token_revocation:"key",rate_limit:"gauge",observability:"activity",runs:"play",websocket:"bolt",ag_ui:"external",remote_tools:"wrench",media:"file",plugins:"layers",test:"flask",evaluation:"target"};var Hn=0;function n(e,t,r,o,s,a){t||(t={});var i,d,c=t;if("ref"in c)for(d in c={},t)d=="ref"?i=t[d]:c[d]=t[d];var p={type:e,props:c,key:r,ref:i,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:--Hn,__i:-1,__u:0,__source:s,__self:a};if(typeof e=="function"&&(i=e.defaultProps))for(d in i)c[d]===void 0&&(c[d]=i[d]);return x.vnode&&x.vnode(p),p}function h({name:e,class:t="size-4"}){return/(^|\s)size-/.test(t)||(t=`size-4 ${t}`),n("span",{class:`inline-flex shrink-0 ${t}`,"aria-hidden":"true",children:n("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor","stroke-width":"1.75","stroke-linecap":"round","stroke-linejoin":"round",class:"size-full",dangerouslySetInnerHTML:{__html:rn[e]}})})}function Me({class:e,children:t}){return n("span",{class:`inline-flex h-5 shrink-0 items-center rounded-full px-2 text-[11px] font-semibold ${e}`,children:t})}function ae({id:e,label:t,options:r,value:o,onPick:s}){return n("div",{id:e,role:"radiogroup","aria-label":t,class:"inline-flex max-w-full flex-wrap gap-0.5 rounded-[10px] border border-line-strong bg-raised p-0.5",children:r.map(([a,i])=>n("button",{type:"button",role:"radio","aria-checked":a===o?"true":"false",class:`focus-ring min-h-10 cursor-pointer rounded-lg px-3.5 text-sm font-medium transition-colors duration-150 ${a===o?"bg-surface text-ink shadow-card":"text-muted hover:text-ink"}`,onClick:()=>{a!==o&&s(a)},children:i},a))})}function ze({id:e,options:t,value:r,invalid:o,onChange:s}){return n("div",{class:"relative",children:[n("select",{id:e,value:r,class:`input cursor-pointer appearance-none pr-9 font-mono ${o?"input-invalid":""}`,onChange:a=>s(a.currentTarget.value),children:t.map(([a,i])=>n("option",{value:a,children:i},a))}),n(h,{name:"chevronDown",class:"pointer-events-none absolute top-1/2 right-3 size-4 -translate-y-1/2 text-faint"})]})}function se({id:e,values:t,placeholder:r,suggestions:o,onChange:s}){let a=Ce(null),i=c=>{let p=c.trim();a.current&&(a.current.value=""),p&&!t.includes(p)&&s([...t,p])},d=o?`${e}-options`:void 0;return n("div",{class:"flex min-h-11 cursor-text flex-wrap items-center gap-1.5 rounded-lg border border-line-strong bg-sunken px-1.5 py-1.5 transition-[border-color,box-shadow] duration-150 focus-within:border-brand focus-within:ring-4 focus-within:ring-brand-ring sm:py-1",onClick:c=>{c.target===c.currentTarget&&a.current?.focus()},children:[t.map((c,p)=>n("span",{class:"inline-flex h-7 items-center gap-0.5 rounded-md border border-line bg-raised pr-0.5 pl-2 font-mono text-xs text-body sm:h-6",children:[c,n("button",{type:"button","aria-label":`Remove ${c}`,class:"inline-flex size-6 cursor-pointer items-center justify-center rounded text-faint transition-colors hover:bg-hover hover:text-ink sm:size-5",onClick:()=>s(t.filter((m,g)=>g!==p)),children:n(h,{name:"x",class:"size-3"})})]},c)),n("input",{ref:a,id:e,type:"text",list:d,placeholder:t.length?"Add...":r,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:"h-7 min-w-[7rem] flex-1 bg-transparent px-1 font-mono text-base text-ink outline-none placeholder:text-faint sm:text-sm",onKeyDown:c=>{let p=c.currentTarget;(c.key==="Enter"||c.key===",")&&p.value.trim()?(c.preventDefault(),i(p.value)):c.key==="Backspace"&&!p.value&&t.length&&s(t.slice(0,-1))},onChange:c=>{o?.includes(c.currentTarget.value.trim())&&i(c.currentTarget.value)},onBlur:c=>i(c.currentTarget.value)}),o?n("datalist",{id:d,children:o.filter(c=>!t.includes(c)).map(c=>n("option",{value:c},c))}):null]})}function ie({label:e,path:t,help:r,id:o,required:s}){return n("div",{class:"min-w-0",children:[n("label",{for:o,class:"block text-sm font-medium text-ink",children:[e,s?n("span",{class:"text-danger",children:" *"}):null]}),n("div",{class:"mt-0.5 font-mono text-xs text-faint",children:t}),r?n("p",{class:"mt-1.5 text-[13px] leading-5 text-muted",children:r}):null]})}function le({issue:e}){if(!e)return null;let t=e.level==="error";return n("p",{class:`mt-2 flex gap-1.5 text-[13px] leading-5 ${t?"text-danger":"text-warn"}`,role:t?"alert":void 0,children:[n(h,{name:t?"xCircle":"alert",class:"mt-0.5 size-3.5"}),n("span",{children:e.message})]})}function nt({children:e}){return n("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[minmax(0,1fr)_minmax(0,1.1fr)] sm:gap-8",children:e})}function Y({id:e,label:t,path:r,value:o,placeholder:s,onInput:a,issue:i,help:d}){return n(nt,{children:[n(ie,{label:t,path:r,help:d,id:e}),n("div",{class:"min-w-0 sm:pt-0.5",children:[n("input",{id:e,type:"text",value:o??"",placeholder:s,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${i?.level==="error"?"input-invalid":""}`,onInput:c=>a(c.currentTarget.value)}),n(le,{issue:i})]})]})}function rt({name:e,label:t,options:r,value:o,onPick:s}){return n("div",{role:"radiogroup","aria-label":t,class:"grid gap-3 sm:grid-cols-2",children:r.map(([a,i,d])=>{let c=a===o;return n("button",{type:"button",role:"radio","aria-checked":c?"true":"false",id:`${e}-${a}`,class:`focus-ring flex cursor-pointer items-start gap-3 rounded-xl border p-4 text-left transition-colors duration-150 ${c?"border-brand bg-brand-soft":"border-line bg-surface shadow-card hover:border-line-strong"}`,onClick:()=>{c||s(a)},children:[n("span",{class:`mt-0.5 flex size-4 shrink-0 items-center justify-center rounded-full border ${c?"border-brand-solid bg-brand-solid":"border-line-strong"}`,children:c?n("span",{class:"size-1.5 rounded-full bg-white"}):null}),n("span",{class:"min-w-0",children:[n("span",{class:"block text-sm font-medium text-ink",children:i}),n("span",{class:"mt-0.5 block text-[13px] leading-5 text-muted",children:d})]})]},a)})})}function B({title:e,children:t}){return n("section",{children:[e?n("h4",{class:"eyebrow mb-2.5",children:e}):null,t]})}var Ln={info:["info","border-brand/20 bg-brand-soft","text-brand"],warn:["alert","border-warn/25 bg-warn-soft","text-warn"],error:["xCircle","border-danger/25 bg-danger-soft","text-danger"]};function ce({kind:e,title:t,lines:r}){let[o,s,a]=Ln[e];return n("div",{class:`flex gap-3 rounded-xl border px-4 py-3 ${s}`,role:e==="error"?"alert":void 0,children:[n(h,{name:o,class:`mt-0.5 size-4 ${a}`}),n("div",{class:"min-w-0 space-y-1 text-[13px] leading-5 text-body",children:[t?n("p",{class:"font-medium text-ink",children:t}):null,r.map((i,d)=>n("p",{children:i},d))]})]})}function ot({issue:e}){return n(ce,{kind:e.level==="error"?"error":"warn",lines:[n("span",{children:[e.path?n("code",{class:"mr-1.5 font-mono text-xs text-muted",children:e.path}):null,e.message]})]})}function on({icon:e,tint:t,title:r,text:o,children:s}){return n("div",{class:"px-6 py-14 text-center",children:[n("div",{class:`mx-auto flex size-11 items-center justify-center rounded-full bg-raised ${t}`,children:n(h,{name:e,class:"size-5"})}),n("p",{class:"mt-4 text-sm font-semibold text-ink",children:r}),n("p",{class:"mx-auto mt-1 max-w-xs text-sm leading-6 text-muted",children:o}),s]})}function an(){let e=Ce(null);M(()=>{let i=e.current;i&&(l.drawerOpen&&!i.open?i.showModal():!l.drawerOpen&&i.open&&i.close())});let t=l.issues.length>0,r=t&&l.tab==="problems"?"problems":"json",{errors:o,warnings:s}=re(),a=(i,d,c)=>n("button",{type:"button",role:"tab",id:`tab-${i}`,"aria-selected":r===i?"true":"false",class:`focus-ring inline-flex h-10 cursor-pointer items-center gap-2 rounded-lg px-3 text-sm font-medium transition-colors duration-150 ${r===i?"bg-raised text-ink":"text-muted hover:text-ink"}`,onClick:()=>_(p=>{p.tab=i}),children:[d,c]});return n("dialog",{ref:e,"aria-label":"10xgraph.json preview",class:"m-0 ml-auto h-dvh max-h-none w-[min(540px,100vw)] border-0 border-l border-line-strong bg-surface p-0 text-body shadow-pop backdrop:bg-black/45",onClose:()=>{l.drawerOpen&&Oe()},onClick:i=>{i.target===i.currentTarget&&Oe()},children:n("div",{class:"flex h-full flex-col",children:[n("div",{class:"flex h-14 shrink-0 items-center gap-1 border-b border-line px-3",children:[n("div",{role:"tablist",class:"flex flex-1 gap-1",children:[a("json","10xgraph.json",null),t?a("problems","Problems",n("span",{class:`inline-flex h-5 min-w-5 items-center justify-center rounded-full px-1.5 text-[11px] font-semibold ${o?"bg-danger-soft text-danger":"bg-warn-soft text-warn"}`,children:o+s})):null]}),n("button",{type:"button",class:"icon-btn","aria-label":"Close preview",onClick:Oe,children:n(h,{name:"x",class:"size-5"})})]}),n("div",{class:"min-h-0 flex-1 overflow-y-auto overscroll-contain",children:r==="problems"?n(Fn,{}):n(Un,{})})]})})}function Fn(){let{errors:e,warnings:t}=re(),r=l.sections.map(o=>[o,J(o.id)]).filter(([,o])=>o.length);return n("div",{children:[n("div",{class:"border-b border-line px-4 py-3",children:n("p",{class:"text-sm text-muted",children:[n("span",{class:e?"font-medium text-danger":"",children:ne(e,"error")})," \xB7 ",n("span",{class:t?"font-medium text-warn":"",children:ne(t,"warning")})]})}),n("div",{class:"space-y-5 px-3 py-4",children:r.map(([o,s])=>n("div",{children:[n("p",{class:"eyebrow mb-1.5 flex items-center gap-2 px-2",children:[n(h,{name:W[o.id]||"box",class:"size-3.5"}),o.title]}),n("ul",{class:"space-y-0.5",children:s.map((a,i)=>{let d=a.level==="error";return n("li",{children:n("button",{type:"button",onClick:()=>O(o.id,Te(a.path)),class:"focus-ring flex min-h-11 w-full cursor-pointer gap-2.5 rounded-lg px-2 py-2 text-left transition-colors duration-150 hover:bg-raised",children:[n(h,{name:d?"xCircle":"alert",class:`mt-0.5 size-4 ${d?"text-danger":"text-warn"}`}),n("span",{class:"min-w-0",children:[n("span",{class:"block text-[13px] leading-5 text-body",children:a.message}),a.path?n("span",{class:"mt-0.5 block truncate font-mono text-xs text-faint",children:a.path}):null]})]})},i)})})]},o.id))})]})}function qn(e){return e.replace(/&/g,"&").replace(//g,">").replace(/("(\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?)/g,r=>{let o="var(--af-json-number)";return r.startsWith('"')?o=r.endsWith(":")?"var(--af-json-key)":"var(--af-json-string)":(r==="true"||r==="false"||r==="null")&&(o="var(--af-json-literal)"),`${r}`})}function Un(){let e=JSON.stringify(l.config,null,2).split(` -`);return n("div",{children:[n("div",{class:"sticky top-0 flex items-center justify-between gap-3 border-b border-line bg-surface px-4 py-2.5",children:[n("p",{class:"text-xs text-faint",children:R()||!l.exists?"Preview of what Save will write":"Matches the file on disk"}),n("button",{type:"button",class:"btn btn-ghost h-8 px-2.5 text-xs",onClick:nn,children:[n(h,{name:"copy",class:"size-3.5"}),"Copy"]})]}),n("div",{class:"py-3 pr-4 font-mono text-[12.5px] leading-5",children:e.map((t,r)=>n("div",{class:"flex",children:[n("span",{class:"w-10 shrink-0 pr-3 text-right text-faint/70 select-none",children:r+1}),n("span",{class:"min-w-0 flex-1 break-all whitespace-pre-wrap",dangerouslySetInnerHTML:{__html:qn(t)}})]},r))})]})}function st(e){let t=[];for(let r of e){let o=r.group||"",s=t.find(a=>a.name===o);s||t.push(s={name:o,fields:[]}),s.fields.push(r)}return t}var D=(e,t)=>_(r=>{if(t===void 0){Ze(r.config,e);return}let o=e.length>1?r.sections.find(s=>s.optional&&s.key===e[0]):null;o&&(r.config[o.key]===void 0||r.config[o.key]===null)&&(r.config[o.key]=JSON.parse(JSON.stringify(o.default))),Ut(r.config,e,t)}),Wn=e=>`f-${e.path.join("-")}`;function Bn({id:e,value:t,field:r,invalid:o,disabled:s}){let[a,i]=we(String(t??""));return M(()=>{(a===""?t!==void 0:Number(a)!==t)&&i(String(t??""))},[t]),n("input",{id:e,type:"number",inputmode:"decimal",value:a,disabled:s,min:r.min,step:r.step??"any",placeholder:r.placeholder??r.default??"",class:`input max-w-40 font-mono tabular-nums ${o?"input-invalid":""}`,onInput:d=>{let c=d.currentTarget.value;i(c),c===""?D(r.path,void 0):Number.isNaN(Number(c))||D(r.path,Number(c))}})}function at({field:e,value:t}){return e.exclusive_min===void 0||typeof t!="number"||t>e.exclusive_min?null:n("p",{class:"mt-1.5 text-[13px] text-warn",children:["Must be greater than ",e.exclusive_min,"."]})}function Vn({id:e,field:t,value:r,invalid:o}){let s=t.unlimited_value!==void 0&&r===t.unlimited_value,a=n(Bn,{id:e,value:s?void 0:r,field:t,invalid:o,disabled:s});if(t.null_label!==void 0){let i=r===null?"n":r===void 0?"d":"s",d=typeof t.default=="number"?t.default:300;return n("div",{children:[n(ae,{id:e,label:t.label,value:i,options:[["d","Default"],["s","Seconds"],["n",t.null_label]],onPick:c=>D(t.path,c==="d"?void 0:c==="n"?null:d)}),i==="s"?n("div",{class:"mt-2.5",children:[a,n(at,{field:t,value:r})]}):null]})}return t.unlimited_value!==void 0?n("div",{children:[n("div",{class:"flex flex-wrap items-center gap-x-4",children:[a,n("label",{class:"inline-flex min-h-11 cursor-pointer items-center gap-2.5 text-sm text-body",children:[n("input",{type:"checkbox",class:"size-[18px] accent-brand-solid",checked:s,onChange:i=>D(t.path,i.currentTarget.checked?t.unlimited_value:void 0)}),"Unlimited"]})]}),n(at,{field:t,value:r})]}):n("div",{children:[a,n(at,{field:t,value:r})]})}function Gn({id:e,field:t,value:r}){if(t.unset_label!==void 0){let o=[["d","Default"]];return t.allow_true!==!1&&o.push(["t","On"]),o.push(["f","Off"]),n("div",{children:[n(ae,{id:e,label:t.label,value:r===void 0?"d":r?"t":"f",options:o,onPick:s=>D(t.path,s==="d"?void 0:s==="t")}),n("p",{class:"mt-1.5 text-xs text-faint",children:["Default: ",t.unset_label]})]})}return n("label",{class:"inline-flex min-h-11 cursor-pointer items-center gap-2.5 text-sm font-semibold text-ink",children:[n("input",{id:e,type:"checkbox",class:"size-[18px] accent-brand-solid",checked:(r??t.default??!1)===!0,onChange:o=>D(t.path,o.currentTarget.checked)}),t.label]})}function Jn({id:e,field:t,value:r,invalid:o}){if(t.type==="bool")return n(Gn,{id:e,field:t,value:r});if(t.type==="number")return n(Vn,{id:e,field:t,value:r,invalid:o});if(t.type==="list")return n(se,{id:e,values:Array.isArray(r)?r:[],placeholder:t.placeholder,onChange:s=>D(t.path,s.length?s:void 0)});if(t.type==="select"){let s=t.options.map(a=>[a,a]);return t.unset_label!==void 0&&s.unshift(["",t.unset_label]),n(ze,{id:e,options:s,invalid:o,value:r??(t.unset_label!==void 0?"":t.default??t.options[0]),onChange:a=>D(t.path,a===""?void 0:a)})}return n("input",{id:e,type:"text",value:r??"",placeholder:t.placeholder,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${o?"input-invalid":""}`,onInput:s=>D(t.path,s.currentTarget.value===""?void 0:s.currentTarget.value)})}function Ne({field:e}){let t=e.path.join("."),r=Wn(e),o=q(l.config,e.path),s=U(t),a=Qe(e),i=Vt(e),d=e.type==="bool"&&e.unset_label===void 0,c=e.default!==void 0&&!a,p=Array.isArray(e.default)?e.default.join(", "):String(e.default??"");return n("div",{id:`row-${r}`,class:`relative pt-5 pb-1 ${a?"before:absolute before:top-6 before:-left-2.5 before:bottom-2 before:w-[3px] before:rounded before:bg-brand before:content-[''] sm:before:-left-3.5":""}`,children:[n("div",{class:"flex min-h-6 items-center gap-2.5",children:[d?null:n("label",{for:r,class:`text-sm font-semibold text-ink ${i?"opacity-50":""}`,children:[e.label,e.required?n("span",{class:"text-danger",children:" *"}):null]}),d?null:n("code",{class:"hidden font-mono text-[11px] text-faint sm:inline",children:t}),c&&!d?n("span",{class:"rounded border border-line px-1.5 text-[11px] leading-4 font-medium text-faint",children:"default"}):null,n("span",{class:"flex-1"}),a?n("button",{type:"button",class:"focus-ring min-h-8 cursor-pointer rounded px-1 text-[13px] font-medium text-brand hover:underline",title:`Reset to default${e.default!==void 0?` (${p})`:""}`,"aria-label":`Reset ${e.label} to default`,onClick:()=>_(m=>Ze(m.config,e.path)),children:"Reset"}):null]}),e.help?n("p",{class:`mt-0.5 max-w-prose text-[13px] leading-5 text-muted ${i?"opacity-50":""}`,children:e.help}):null,n("fieldset",{disabled:i,class:`m-0 mt-2.5 min-w-0 max-w-[520px] border-0 p-0 ${i?"opacity-50":""}`,children:n(Jn,{id:r,field:e,value:o,invalid:s?.level==="error"})}),i?n("p",{class:"mt-1.5 text-xs text-faint",children:Gt(e)}):null,n(le,{issue:s})]})}function sn(){let e=l.config.auth,t=z(e)?"custom":"jwt";return n("div",{class:"space-y-6",children:[n(B,{title:"Method",children:n(rt,{name:"auth",label:"Authentication method",value:t,options:[["jwt","JWT","Verify bearer tokens signed with JWT_SECRET_KEY."],["custom","Custom backend","Your own BaseAuth subclass decides who is calling."]],onPick:r=>_(o=>{o.config.auth=r==="jwt"?"jwt":{method:"custom",path:""}})})}),t==="custom"?n(B,{title:"Backend",children:n("div",{class:"panel",children:n(Y,{id:"auth-path",label:"Auth class",path:"auth.path",value:e.path,placeholder:"auth.custom:MyAuth",help:"Import path of a BaseAuth subclass, as module:attribute.",issue:U("auth.path"),onInput:r=>_(o=>{o.config.auth={...o.config.auth,method:"custom",path:r}})})})}):null]})}function Kn(e){if(z(e))return"rbac";if(typeof e=="string"){let t=e.trim().toLowerCase();if(t==="ownership")return"ownership";if(["allow_all","default","none"].includes(t))return"allow_all"}return e==null?null:"custom"}var Yn={ownership:()=>"ownership",allow_all:()=>"allow_all",custom:()=>"",rbac:()=>({backend:"rbac",roles:{admin:["*"],member:["graph:invoke","graph:stream","graph:read","checkpointer:read"]},default_scopes:[],isolation:"owner"})};function ln({sec:e}){let t=l.config.authorization,r=Kn(t)??"ownership";return n("div",{class:"space-y-6",children:[n(B,{title:"Policy",children:n(rt,{name:"authz",label:"Authorization policy",value:r,options:[["ownership","Ownership","Users can only reach threads they created."],["allow_all","Allow all","Any authenticated user can do anything."],["rbac","Roles and scopes","Map roles to scopes, on top of owner isolation."],["custom","Custom backend","Your own AuthorizationBackend class."]],onPick:o=>_(s=>{s.config.authorization=Yn[o]()})})}),r==="custom"?n(B,{title:"Backend",children:n("div",{class:"panel",children:n(Y,{id:"authz-path",label:"Backend class",path:"authorization",placeholder:"auth.permissions:MyBackend",value:typeof t=="string"?t:"",issue:U("authorization"),help:"Import path of an AuthorizationBackend, as module:attribute.",onInput:o=>_(s=>{s.config.authorization=o})})})}):null,r==="rbac"?n(Qn,{sec:e,value:t}):null]})}function Zn({id:e,name:t,onRename:r}){let[o,s]=we(t);return M(()=>s(t),[t]),n("div",{children:[n("label",{for:e,class:"sr-only",children:"Role name"}),n("input",{id:e,type:"text",value:o,class:"input font-mono",autocomplete:"off",spellcheck:!1,onInput:a=>s(a.currentTarget.value),onChange:a=>{r(a.currentTarget.value.trim())||s(t)}})]})}function Qn({sec:e,value:t}){let r="roles"in t||!("role_scopes"in t)?"roles":"role_scopes",o=z(t[r])?t[r]:{},s=["*",...e.scopes||[]],a=Object.entries(o),i=c=>_(()=>{t[r]=c}),d=()=>{let c=a.length+1;for(;`role${c}`in o;)c++;i({...o,[`role${c}`]:[]})};return n(w,{children:[n(B,{title:"Roles",children:n("div",{class:"panel",children:[n("div",{class:"hidden grid-cols-[10rem_minmax(0,1fr)_auto] gap-3 border-b border-line px-5 py-2.5 sm:grid",children:[n("span",{class:"text-xs font-medium text-faint",children:"Role"}),n("span",{class:"text-xs font-medium text-faint",children:"Scopes"}),n("span",{class:"w-9"})]}),a.length?n("div",{class:"divide-y divide-line",children:a.map(([c,p],m)=>n("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[10rem_minmax(0,1fr)_auto] sm:items-start",children:[n(Zn,{id:`role-name-${m}`,name:c,onRename:g=>!g||g!==c&&g in o?!1:(i(Object.fromEntries(Object.entries(o).map(([u,f])=>[u===c?g:u,f]))),!0)}),n(se,{id:`role-scopes-${m}`,values:Array.isArray(p)?p:[],placeholder:"Add a scope",suggestions:s,onChange:g=>i({...o,[c]:g})}),n("button",{type:"button",class:"icon-btn justify-self-end hover:text-danger","aria-label":`Remove role ${c}`,onClick:()=>i(Object.fromEntries(Object.entries(o).filter(([g])=>g!==c))),children:n(h,{name:"trash"})})]},m))}):n("p",{class:"px-5 py-6 text-center text-sm text-muted",children:"No roles yet. Users only get the default scopes."}),n("div",{class:"flex items-center justify-between gap-4 border-t border-line px-5 py-3",children:[n("p",{class:"text-[13px] text-muted",children:["Read from the user's roles (or role) claim. ",n("code",{class:"font-mono",children:"*"})," grants every scope."]}),n("button",{type:"button",class:"btn btn-ghost",onClick:d,children:[n(h,{name:"plus"}),"Add role"]})]})]})}),n(B,{title:"Defaults",children:n("div",{class:"panel divide-y divide-line",children:[n("div",{class:"px-5 py-4",children:[n(ie,{label:"Default scopes",path:"authorization.default_scopes",id:"rbac-default-scopes",help:"Granted to every authenticated user, even without a role."}),n("div",{class:"mt-3",children:n(se,{id:"rbac-default-scopes",values:Array.isArray(t.default_scopes)?t.default_scopes:[],placeholder:"graph:read",suggestions:s,onChange:c=>_(()=>{t.default_scopes=c})})})]}),n(nt,{children:[n(ie,{label:"Thread isolation",path:"authorization.isolation",id:"rbac-isolation",help:"owner keeps each user's threads private; none shares them."}),n("div",{class:"min-w-0 sm:pt-0.5",children:n(ze,{id:"rbac-isolation",options:[["owner","owner"],["none","none"]],value:t.isolation??"owner",onChange:c=>_(()=>{t.isolation=c})})})]})]})})]})}function cn(){let e=Array.isArray(l.config.remote_tools)?l.config.remote_tools:[],t=()=>{_(r=>{e.push({node:"",name:"",description:"",parameters:{type:"object",properties:{},required:[]}}),r.config.remote_tools=e}),requestAnimationFrame(()=>document.getElementById(`tool-${e.length-1}-name`)?.focus())};return e.length?n("div",{class:"space-y-4",children:[e.map((r,o)=>n(Xn,{tools:e,tool:r,index:o},o)),n("button",{type:"button",onClick:t,class:"focus-ring flex h-12 w-full cursor-pointer items-center justify-center gap-2 rounded-xl border border-dashed border-line-strong text-sm font-medium text-muted transition-colors duration-150 hover:border-brand hover:text-brand",children:[n(h,{name:"plus"}),"Add remote tool"]})]}):n("div",{class:"rounded-xl border border-dashed border-line-strong bg-surface px-6 py-10 text-center",children:[n("div",{class:"mx-auto flex size-11 items-center justify-center rounded-full bg-raised text-faint",children:n(h,{name:"wrench",class:"size-5"})}),n("p",{class:"mt-4 text-sm font-semibold text-ink",children:"No remote tools yet"}),n("p",{class:"mx-auto mt-1 max-w-sm text-sm leading-6 text-muted",children:"Declare a tool here and the client that calls the API runs it, for example to read the user's location."}),n("button",{type:"button",class:"btn btn-outline mt-5",onClick:t,children:[n(h,{name:"plus"}),"Add remote tool"]})]})}function Xn({tools:e,tool:t,index:r}){let o=l.toolParams[r]??{text:JSON.stringify(t.parameters??{},null,2),error:null},s=i=>_(()=>{e[r]={...e[r],...i}}),a=i=>`remote_tools[${r}].${i}`;return n("section",{class:"panel",children:[n("div",{class:"flex items-center gap-3 border-b border-line px-5 py-3",children:[n(h,{name:"wrench",class:"size-4 text-faint"}),n("span",{class:"min-w-0 flex-1 truncate font-mono text-sm font-medium text-ink",children:t.name||`tool_${r+1}`}),t.node?n("span",{class:"rounded-md border border-line bg-raised px-1.5 py-0.5 font-mono text-xs text-muted",children:t.node}):null,n("button",{type:"button",class:"icon-btn hover:text-danger","aria-label":`Remove ${t.name||"tool"}`,onClick:()=>_(i=>{e.splice(r,1),i.toolParams={}}),children:n(h,{name:"trash"})})]}),n("div",{class:"divide-y divide-line",children:[n(Y,{id:`tool-${r}-name`,label:"Tool name",path:a("name"),value:t.name,placeholder:"get_location",onInput:i=>s({name:i})}),n(Y,{id:`tool-${r}-node`,label:"Graph node",path:a("node"),value:t.node,placeholder:"MAIN",help:"The node whose model may call this tool.",onInput:i=>s({node:i})}),n(Y,{id:`tool-${r}-description`,label:"Description",path:a("description"),value:t.description,placeholder:"Returns the user's current city",help:"What the model reads when deciding to call it.",onInput:i=>s({description:i})}),n("div",{class:"px-5 py-4",children:[n(ie,{label:"Parameters",path:a("parameters"),help:"JSON schema of the arguments. type must be object.",id:`tool-${r}-params`}),n("textarea",{id:`tool-${r}-params`,rows:7,spellcheck:!1,value:o.text,class:`input mt-3 h-auto py-2.5 font-mono text-[13px] leading-5 ${o.error?"input-invalid":""}`,onInput:i=>{let d=i.currentTarget.value;_(c=>{try{e[r]={...e[r],parameters:JSON.parse(d)},c.toolParams[r]={text:d,error:null}}catch(p){c.toolParams[r]={text:d,error:p.message}}})}}),n(le,{issue:o.error?{level:"error",message:o.error}:null})]})]})]})}var ue=({children:e,code:t})=>n("h1",{id:"page-title",tabindex:-1,class:"flex flex-wrap items-baseline gap-x-3 gap-y-1 text-2xl font-semibold tracking-tight text-ink focus:outline-none",children:[e,t?n("code",{class:"rounded border border-line bg-raised px-1.5 font-mono text-xs font-normal text-faint",children:t}):null]}),it=({children:e})=>n("p",{class:"mt-1.5 max-w-[62ch] text-[15px] leading-6 text-muted",children:e}),un=({children:e})=>n("h2",{class:"mt-10 mb-2 eyebrow",children:e});function dn({id:e,text:t,warn:r,compact:o}){let s=oe(e);return s?n("a",{href:`#${e}`,onClick:a=>{a.preventDefault(),O(e)},class:`focus-ring group flex items-center gap-3.5 border-t border-line py-2 last:border-b ${o?"min-h-12":"min-h-14"}`,children:[n(h,{name:W[e]||"box",class:"size-[18px] text-faint"}),n("span",{class:`min-w-0 flex-1 ${o?"flex flex-wrap items-baseline gap-x-4":""}`,children:[n("span",{class:`block text-sm text-ink group-hover:text-brand ${o?"min-w-40 font-medium":"font-semibold"}`,children:s.title}),n("span",{class:`block text-[13px] leading-5 ${r?"text-warn":"text-muted"}`,children:t??s.description})]}),n(h,{name:"chevronRight",class:"size-4 text-faint"})]}):null}function lt(e){return`Not set. ${T(e)?.off_summary||""}`.trim()}function er(){let e=l.config.auth;return e==null?{text:lt("auth")}:e==="jwt"?{text:"JWT bearer tokens."}:e&&typeof e=="object"?{text:`Custom backend${e.path?`: ${e.path}`:"."}`}:{text:String(e)}}function tr(){let e=l.config.rate_limit;if(e==null)return{text:lt("rate_limit")};let t=a=>G(["rate_limit",a]);if(t("enabled")===!1)return{text:"Settings kept, not enforced."};let r={ip:"per IP",user:"per user",global:"across all clients"}[t("by")]||t("by"),o=U("rate_limit.backend")?.level==="warning",s=["requests","window","by"].some(a=>q(l.config,["rate_limit",a])!==void 0&&q(l.config,["rate_limit",a])!==l.fields.get(`rate_limit.${a}`)?.default);return{text:`${t("requests")} requests per ${t("window")} s, ${r}${s?"":" (default)"}.${o?" The memory backend counts per process.":""}`,warn:o}}function nr(){let e=l.config.observability;if(e==null)return{text:lt("observability")};let t=[];return G(["observability","logfire","enabled"])&&t.push("Logfire"),G(["observability","langsmith","enabled"])&&t.push("LangSmith"),{text:`Level ${G(["observability","level"])}${t.length?`, sending to ${t.join(" and ")}`:", no destination chosen yet"}.`}}function pn(){let e=T("core"),t=Jt(),r=e?J("core").filter(a=>!e.fields.some(i=>i.path.join(".")===a.path)):[],o=[["auth",er()],["rate_limit",tr()],["observability",nr()]],s=l.nav.flatMap(a=>a.items).filter(a=>a!=="get-running");return n(w,{children:[n(ue,{children:t?"You're ready to run":"One step to run your agent"}),n(it,{children:N["get-running"].description}),n("div",{class:`mt-4 flex items-center gap-2 font-medium ${t?"text-ok":"text-muted"}`,children:[n(h,{name:t?"checkCircle":"circle",class:"size-[18px]"}),"Required: ",t?"1":"0"," of 1 done"]}),r.length?n("div",{class:"mt-4 space-y-2",children:r.map((a,i)=>n(ot,{issue:a},i))}):null,n("div",{class:"mt-2",children:e?st(e.fields).flatMap(a=>a.fields).map(a=>n(Ne,{field:a},a.path.join("."))):null}),t?n("div",{class:"mt-7 border-t border-line pt-5",children:[n("div",{class:"flex items-center gap-2.5 font-medium text-ok",children:[n(h,{name:"checkCircle",class:"size-[18px]"}),"Ready for 10xgraph dev"]}),n("div",{class:"mt-3 flex max-w-[420px] items-center gap-2 rounded-lg border border-line-strong bg-sunken py-0 pr-1 pl-3.5",children:[n("code",{class:"flex-1 font-mono text-[13px] font-medium text-ink",children:"10xgraph dev"}),l.copied?n("span",{class:"text-xs text-ok",children:"Copied"}):null,n("button",{type:"button",class:"icon-btn",onClick:tn,"aria-label":l.copied?"Copied 10xgraph dev":"Copy command 10xgraph dev",children:n(h,{name:"copy",class:"size-[18px]"})})]}),n("p",{class:"mt-3 text-muted",children:"Everything else already has working defaults."})]}):n("p",{class:"mt-4 text-xs text-faint",children:"Add the agent graph above, then run 10xgraph dev."}),n(un,{children:"Before production (optional)"}),n("div",{children:o.map(([a,i])=>n(dn,{id:a,text:i.text,warn:i.warn},a))}),n(un,{children:"What you can configure"}),n("div",{children:s.map(a=>n(dn,{id:a,compact:!0},a))})]})}function Re({vars:e}){return n("dl",{class:"mb-3 grid gap-x-4 gap-y-1 sm:grid-cols-[minmax(0,260px)_minmax(0,1fr)]",children:e.map(t=>[n("dt",{class:"py-0.5 font-mono text-xs font-medium break-all text-ink",children:t.name},`${t.name}-n`),n("dd",{class:"m-0 pb-2 text-[13px] text-muted sm:py-0.5",children:t.purpose},`${t.name}-p`)])})}function rr({vars:e,open:t}){return e?.length?n("details",{class:"group mt-9 border-t border-line pt-1",open:t,children:[n("summary",{class:"focus-ring flex min-h-11 cursor-pointer list-none items-center gap-2 font-medium text-muted [&::-webkit-details-marker]:hidden",children:[n(h,{name:"chevronRight",class:"size-4 transition-transform group-open:rotate-90"}),"Set in .env"]}),n("p",{class:"mb-2 text-xs text-faint",children:"Read only. The editor never reads .env, so only names are shown."}),n(Re,{vars:e})]}):null}function fn({sec:e}){return e.widget==="auth"?n(sn,{}):e.widget==="authorization"?n(ln,{sec:e}):e.widget==="remote_tools"?n(cn,{}):null}function or(e){let t=new Set([...e.fields.map(r=>r.path.join(".")),...Object.keys(Ke)]);return J(e.id).filter(r=>!t.has(r.path))}function ar({sec:e}){let t=$e(e),r=e.widget&&(t||!e.optional);return n(w,{children:[r?n("div",{class:"mt-6",children:n(fn,{sec:e})}):null,n("div",{class:e.optional&&!t?"opacity-80":"",children:st(e.fields).map((o,s)=>n("div",{children:[o.name?n("h3",{class:`eyebrow ${s===0?"mt-6":"mt-8 border-t border-line pt-4"}`,children:o.name}):null,o.fields.map(a=>n(Ne,{field:a},a.path.join(".")))]},o.name))})]})}function sr({sec:e}){let t=$e(e),r=or(e);return n(w,{children:[n(ue,{code:e.key,children:e.title}),n(it,{children:e.description}),r.length?n("div",{class:"mt-4 space-y-2",children:r.map((o,s)=>n(ot,{issue:o},s))}):null,e.optional?n("div",{class:"mt-5 flex flex-wrap items-center gap-4",children:[n(ae,{label:`Settings for ${e.title}`,value:t?"c":"d",options:[["d","Use defaults"],["c","Customize"]],onPick:o=>Xt(e,o==="c")}),n("p",{class:"m-0 min-w-56 flex-1 text-[13px] text-muted",children:t?`Saved in 10xgraph.json under ${e.key}.`:e.off_summary})]}):null,e.notes?.length?n("div",{class:"mt-5",children:n(ce,{kind:"info",lines:e.notes})}):null,n(ar,{sec:e}),n(rr,{vars:e.env})]})}function ir(){let e=l.schema.environment||[];return n(w,{children:[n(ue,{children:N.environment.title}),n(it,{children:"These variables are set in .env or your deployment environment. The editor never reads that file, so only names and purposes are shown."}),e.map(t=>n("section",{children:[n("h2",{class:"mt-8 mb-2 text-sm font-semibold text-ink",children:t.title}),t.note?n("p",{class:"mb-2 text-xs text-faint",children:t.note}):null,n(Re,{vars:t.vars})]},t.title))]})}var mn=e=>String(e??"").toLowerCase();function lr(e){let t=s=>e.every(a=>mn(s).includes(a)),r=[];for(let s of l.sections){let a=t(`${s.title} ${s.description} ${s.key||""}`),i=s.fields.filter(p=>a||t(`${p.label} ${p.help||""} ${p.path.join(".")}`)),d=(s.env||[]).filter(p=>t(`${p.name} ${p.purpose}`)),c=!!s.widget&&a;(i.length||d.length||c)&&r.push({id:s.id,sec:s,fields:i,env:d,widget:c})}let o=(l.schema.environment||[]).map(s=>({title:s.title,vars:s.vars.filter(a=>t(`${a.name} ${a.purpose} ${s.title}`))})).filter(s=>s.vars.length);return{results:r,envGroups:o}}function cr(){let e=mn(l.query).split(/\s+/).filter(Boolean),{results:t,envGroups:r}=lr(e),o=t.reduce((s,a)=>s+a.fields.length+a.env.length+(a.widget?1:0),0)+r.reduce((s,a)=>s+a.vars.length,0);return o?n(w,{children:[n(ue,{code:l.query,children:[o," ",o===1?"setting matches":"settings match"]}),t.map(({id:s,sec:a,fields:i,env:d,widget:c})=>n("section",{class:"mt-8",children:[n("h2",{class:"eyebrow m-0 flex items-center gap-2",children:[n(h,{name:W[s]||"box",class:"size-4"}),a.title,n("a",{href:`#${s}`,onClick:p=>{p.preventDefault(),O(s)},class:"focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline",children:"Open page"})]}),c?n("div",{class:"mt-4",children:n(fn,{sec:a})}):null,i.map(p=>n(Ne,{field:p},p.path.join("."))),d.length?n("details",{class:"group mt-5",open:!0,children:[n("summary",{class:"focus-ring flex min-h-11 cursor-pointer list-none items-center gap-2 font-medium text-muted [&::-webkit-details-marker]:hidden",children:[n(h,{name:"chevronRight",class:"size-4 transition-transform group-open:rotate-90"}),"Set in .env"]}),n(Re,{vars:d})]}):null]},s)),r.length?n("section",{class:"mt-8",children:[n("h2",{class:"eyebrow m-0 flex items-center gap-2",children:[n(h,{name:"terminal",class:"size-4"}),N.environment.title,n("a",{href:"#environment",onClick:s=>{s.preventDefault(),O("environment")},class:"focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline",children:"Open page"})]}),r.map(s=>n("div",{children:[n("h3",{class:"mt-4 mb-1.5 text-sm font-semibold text-ink",children:s.title}),s.note?n("p",{class:"mb-1.5 text-xs text-faint",children:s.note}):null,n(Re,{vars:s.vars})]},s.title))]}):null]}):n(w,{children:[n(ue,{children:"No matches"}),n("p",{class:"mt-8 text-muted",children:['Nothing matches "',l.query,'". Try a shorter word, or press Esc to go back.']})]})}function hn(){if(l.query.trim())return n(cr,{});if(l.page==="get-running")return n(pn,{});if(l.page==="environment")return n(ir,{});let e=T(l.page);return e?n(sr,{sec:e}):n(pn,{})}function ur(){let e=l.path.split(/[\\/]/).pop()||"10xgraph.json",t=l.path.slice(0,l.path.length-e.length),r=null;return l.schema?l.exists?R()?r=n(Me,{class:"bg-warn-soft text-warn",children:"Unsaved"}):r=n(Me,{class:"bg-ok-soft text-ok",children:"Saved"}):r=n(Me,{class:"bg-brand-soft text-brand",children:"New"}):r=null,n("div",{class:"flex min-w-0 flex-1 items-center gap-2",title:l.path||void 0,children:[n(h,{name:"file",class:"size-4 text-faint"}),n("span",{class:"truncate font-mono text-[13px] font-medium text-ink",children:e}),r,t?n("span",{class:"hidden min-w-0 truncate font-mono text-xs text-faint lg:block",dir:"rtl",children:n("bdi",{children:t})}):null]})}function _n(){let e=!!l.schema&&!l.fatal,t=l.sideOpen?"Hide sidebar":"Show sidebar";return n("header",{class:"sticky top-0 z-40 flex h-14 items-center gap-2 border-b border-line bg-surface px-3 sm:gap-3 sm:px-4",children:[n("button",{type:"button",class:"icon-btn","aria-controls":"side","aria-expanded":l.sideOpen?"true":"false","aria-label":t,title:`${t} (Ctrl/Cmd+B)`,onClick:()=>K(!l.sideOpen),children:n(h,{name:"panel",class:"size-[18px]"})}),n("div",{class:"flex shrink-0 items-center gap-2.5",children:[n("svg",{viewBox:"0 0 48 48",class:"size-7 shrink-0 text-ink","aria-hidden":"true",children:[n("rect",{x:"4",y:"4",width:"40",height:"40",rx:"11",fill:"none",stroke:"currentColor","stroke-width":"4.4"}),n("path",{d:"M13 33L21 23L34 15L34 31L21 23",fill:"none",stroke:"currentColor","stroke-width":"3","stroke-linecap":"round","stroke-linejoin":"round"}),n("g",{fill:"currentColor",children:[n("circle",{cx:"21",cy:"23",r:"4"}),n("circle",{cx:"34",cy:"15",r:"4"}),n("circle",{cx:"34",cy:"31",r:"4"})]}),n("circle",{cx:"13",cy:"33",r:"4.6",class:"fill-brand-amber"})]}),n("div",{class:"hidden items-baseline gap-1.5 sm:flex",children:[n("span",{class:"text-[15px] font-semibold tracking-tight text-ink",children:"10xGraph"}),n("span",{class:"text-[15px] text-faint",children:"Config"})]})]}),n("span",{class:"hidden h-6 w-px shrink-0 bg-line md:block","aria-hidden":"true"}),n(ur,{}),n("button",{type:"button",class:"btn btn-ghost","aria-label":"View 10xgraph.json","aria-haspopup":"dialog",disabled:!e,onClick:()=>en(),children:[n(h,{name:"code",class:"size-[18px]"}),n("span",{class:"hidden md:inline",children:"View 10xgraph.json"})]}),n("button",{type:"button",class:"btn btn-primary",onClick:Ae,disabled:!e||!!l.busy||l.exists&&!R()&&!l.parseError,children:[l.busy==="save"?n(h,{name:"spinner",class:"size-4 animate-spin"}):null,l.exists?"Save":"Create file",n("span",{class:"kbd hidden lg:inline-flex",children:"Ctrl S"})]})]})}function gn(){return l.fatal?n("div",{class:"panel mt-10",children:n(on,{icon:"xCircle",tint:"text-danger",title:l.fatal.title,text:l.fatal.text})}):l.schema?n(w,{children:[l.parseError?n("div",{class:"mb-6",children:n(ce,{kind:"warn",title:"The existing file is not valid JSON",lines:[`${l.parseError}. The form starts from defaults; saving replaces the file and keeps the old one as 10xgraph.json.bak.`]})}):null,n(hn,{})]}):n("div",{class:"animate-pulse space-y-4 pt-2","aria-label":"Loading",children:[n("div",{class:"h-7 w-48 rounded-md bg-raised"}),n("div",{class:"h-4 w-80 max-w-full rounded-md bg-raised"}),[0,1,2].map(e=>n("div",{class:"h-16 rounded-xl bg-raised"},e))]})}function dr({id:e}){let t=!!T(e)&&Wt(T(e)),r=Bt(e);return!t&&!r?null:n("span",{class:"flex shrink-0 gap-1",children:[t?n("span",{class:"size-2 rounded-full bg-brand",title:"Changed from default"}):null,r?n("span",{class:`size-2 rounded-full ${r.level==="error"?"bg-danger":"bg-warn"}`,title:r.level==="error"?"Needs attention":"Has a warning"}):null]})}function pr({id:e}){let t=oe(e);if(!t)return null;let r=!l.query&&l.page===e;return n("a",{href:`#${e}`,"aria-current":r?"page":void 0,onClick:o=>{o.preventDefault(),O(e)},ref:o=>{o&&r&&window.innerWidth<1024&&o.scrollIntoView({block:"nearest",inline:"nearest"})},class:`focus-ring flex min-h-10 items-center gap-2.5 rounded-lg px-3 font-medium transition-colors duration-150 max-lg:min-h-11 max-lg:shrink-0 max-lg:whitespace-nowrap ${r?"bg-brand-soft text-ink":"text-muted hover:bg-raised hover:text-ink"}`,children:[n(h,{name:W[e]||"box",class:`size-[18px] ${r?"text-brand":""}`}),n("span",{class:"min-w-0 flex-1 truncate",children:t.title}),n(dr,{id:e})]})}function xn(){return l.sideRender?n("aside",{id:"side","aria-label":"Settings navigation",class:`contents lg:sticky lg:top-14 lg:block lg:h-[calc(100dvh-3.5rem)] lg:overflow-y-auto lg:border-r lg:border-line lg:bg-surface lg:px-3 lg:pt-4 lg:pb-16 lg:transition-[transform,opacity] lg:duration-200 lg:ease-out ${l.sideShown?"lg:translate-x-0 lg:opacity-100":"lg:-translate-x-6 lg:opacity-0"}`,inert:!l.sideOpen,children:[n("div",{role:"search",class:"relative mx-1 mb-3 max-lg:mx-4 max-lg:mt-3",children:[n(h,{name:"search",class:"pointer-events-none absolute top-1/2 left-3 size-4 -translate-y-1/2 text-faint"}),n("label",{for:"q",class:"sr-only",children:"Search settings"}),n("input",{id:"q",type:"search",value:l.query,autocomplete:"off",spellcheck:!1,placeholder:"Search settings ( / )",class:"input pl-10",onInput:e=>Ie(e.currentTarget.value)})]}),n("nav",{"aria-label":"Pages",class:"max-lg:sticky max-lg:top-14 max-lg:z-20 max-lg:mt-3 max-lg:flex max-lg:gap-0.5 max-lg:overflow-x-auto max-lg:border-b max-lg:border-line max-lg:bg-surface max-lg:px-3 max-lg:py-1.5",children:l.nav.map(e=>n("div",{class:"max-lg:contents",children:[e.label?n("div",{class:"mt-4 mb-1 px-3 text-[11px] font-semibold tracking-wider text-faint uppercase max-lg:hidden",children:e.label}):null,e.items.map(t=>n(pr,{id:t},t))]},e.label||"top"))})]}):null}var fr={success:["checkCircle","text-ok"],warning:["alert","text-warn"],error:["xCircle","text-danger"]};function mr(){return n("div",{class:"pointer-events-none fixed right-4 bottom-4 left-4 z-50 flex flex-col items-end gap-2 sm:left-auto","aria-live":"polite",children:l.toasts.map(e=>{let[t,r]=fr[e.kind];return n("div",{role:e.kind==="error"?"alert":"status",class:"pointer-events-auto flex w-full max-w-sm items-start gap-3 rounded-xl border border-line bg-surface px-4 py-3 text-sm shadow-pop",children:[n(h,{name:t,class:`mt-0.5 size-4 ${r}`}),n("p",{class:"min-w-0 flex-1 leading-5 text-body",children:e.message}),e.action?n("button",{type:"button",class:"shrink-0 font-medium text-brand hover:underline",onClick:()=>{Pe(e.id),e.action.run()},children:e.action.label}):null,n("button",{type:"button",class:"-mr-1 shrink-0 cursor-pointer rounded text-faint hover:text-ink","aria-label":"Dismiss",onClick:()=>Pe(e.id),children:n(h,{name:"x"})})]},e.id)})})}var hr=()=>/^(INPUT|TEXTAREA|SELECT)$/.test(document.activeElement?.tagName)||document.activeElement?.isContentEditable;function _r(){let[,e]=Je(r=>r+1,0);M(()=>{Ft(e);try{localStorage.getItem("tg-sidebar")==="hidden"&&K(!1,{instant:!0})}catch{}let r=a=>{let i=a.ctrlKey||a.metaKey,d=a.key.toLowerCase();i&&d==="s"?(a.preventDefault(),Ae()):i&&d==="b"?(a.preventDefault(),K(!l.sideOpen)):i&&d==="k"?(a.preventDefault(),tt()):a.key==="/"&&!i&&!a.altKey&&!hr()&&!l.drawerOpen?(a.preventDefault(),tt()):a.key==="Escape"&&l.query&&!l.drawerOpen&&Ie("")},o=a=>{l.schema&&R()&&a.preventDefault()},s=()=>Ee();return document.addEventListener("keydown",r),window.addEventListener("beforeunload",o),window.addEventListener("hashchange",s),Zt(),()=>{document.removeEventListener("keydown",r),window.removeEventListener("beforeunload",o),window.removeEventListener("hashchange",s)}},[]),M(()=>{if(!l.schema||l.validated===I())return;let r=setTimeout(Qt,400);return()=>clearTimeout(r)});let t=l.sideRender;return n(w,{children:[n("a",{href:"#main",class:"sr-only z-50 rounded-md bg-brand-solid px-3 py-2 text-white focus:not-sr-only focus:fixed focus:top-3 focus:left-3",children:"Skip to content"}),n(_n,{}),n("div",{class:t?"lg:grid lg:grid-cols-[260px_minmax(0,1fr)] lg:items-start":"",children:[n(xn,{}),n("main",{id:"main",tabindex:-1,class:"min-w-0 px-4 pt-6 pb-24 focus:outline-none sm:px-8 sm:pt-8",children:n("div",{class:"mx-auto w-full max-w-[760px]",children:n(gn,{})})})]}),n(an,{}),n("div",{class:"sr-only",role:"status","aria-live":"polite",children:l.copied?"Copied":""}),n(mr,{})]})}St(n(_r,{}),document.getElementById("app")); +var be,x,mt,bn,H,ut,ht,_t,De,fe,Q,gt,Fe,He,Le,vn,_e={},ge=[],yn=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,ve=Array.isArray;function P(e,t){for(var r in t)e[r]=t[r];return e}function qe(e){e&&e.parentNode&&e.parentNode.removeChild(e)}function kn(e,t,r){var o,s,a,i={};for(a in t)a=="key"?o=t[a]:a=="ref"?s=t[a]:i[a]=t[a];if(arguments.length>2&&(i.children=arguments.length>3?be.call(arguments,2):r),typeof e=="function"&&e.defaultProps!=null)for(a in e.defaultProps)i[a]===void 0&&(i[a]=e.defaultProps[a]);return me(e,i,o,s,null)}function me(e,t,r,o,s){var a={type:e,props:t,key:r,ref:o,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:s??++mt,__i:-1,__u:0};return s==null&&x.vnode!=null&&x.vnode(a),a}function w(e){return e.children}function he(e,t){this.props=e,this.context=t}function F(e,t){if(t==null)return e.__?F(e.__,e.__i+1):null;for(var r;tt&&H.sort(_t),e=H.shift(),t=H.length,wn(e)}finally{H.length=xe.__r=0}}function bt(e,t,r,o,s,a,i,d,c,p,m){var g,u,f,v,$,S,C=o&&o.__k||ge,b=t.length;for(c=Cn(r,t,C,c,b),g=0;g0?i=e.__k[a]=me(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):e.__k[a]=i,c=a+u,i.__=e,i.__b=e.__b+1,d=null,(p=i.__i=Sn(i,r,c,g))!=-1&&(g--,(d=r[p])&&(d.__u|=2)),d==null||d.__v==null?(p==-1&&(s>m?u--:sc?u--:u++,i.__u|=4))):e.__k[a]=null;if(g)for(a=0;a(m?1:0)){for(s=r-1,a=r+1;s>=0||a=0?s--:a++])!=null&&(2&p.__u)==0&&d==p.key&&c==p.type)return i}return-1}function pt(e,t,r){t[0]=="-"?e.setProperty(t,r??""):e[t]=r==null?"":typeof r!="number"||yn.test(t)?r:r+"px"}function pe(e,t,r,o,s){var a,i;e:if(t=="style")if(typeof r=="string")e.style.cssText=r;else{if(typeof o=="string"&&(e.style.cssText=o=""),o)for(t in o)r&&t in r||pt(e.style,t,"");if(r)for(t in r)o&&r[t]==o[t]||pt(e.style,t,r[t])}else if(t[0]=="o"&&t[1]=="n")a=t!=(t=t.replace(gt,"$1")),i=t.toLowerCase(),t=i in e||t=="onFocusOut"||t=="onFocusIn"?i.slice(2):t.slice(2),e.l||(e.l={}),e.l[t+a]=r,r?o?r[Q]=o[Q]:(r[Q]=Fe,e.addEventListener(t,a?Le:He,a)):e.removeEventListener(t,a?Le:He,a);else{if(s=="http://www.w3.org/2000/svg")t=t.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if(t!="width"&&t!="height"&&t!="href"&&t!="list"&&t!="form"&&t!="tabIndex"&&t!="download"&&t!="rowSpan"&&t!="colSpan"&&t!="role"&&t!="popover"&&t in e)try{e[t]=r??"";break e}catch{}typeof r=="function"||(r==null||r===!1&&t[4]!="-"?e.removeAttribute(t):e.setAttribute(t,t=="popover"&&r==1?"":r))}}function ft(e){return function(t){if(this.l){var r=this.l[t.type+e];if(t[fe]==null)t[fe]=Fe++;else if(t[fe]0?e:ve(e)?e.map(wt):e.constructor!==void 0?null:P({},e)}function $n(e,t,r,o,s,a,i,d,c){var p,m,g,u,f,v,$,S=r.props||_e,C=t.props,b=t.type;if(b=="svg"?s="http://www.w3.org/2000/svg":b=="math"?s="http://www.w3.org/1998/Math/MathML":s||(s="http://www.w3.org/1999/xhtml"),a!=null){for(p=0;p=r.__.length&&r.__.push({}),r.__[e]}function we(e){return ke=1,Je(Rt,e)}function Je(e,t,r){var o=Ge(X++,2);if(o.t=e,!o.__c&&(o.__=[r?r(t):Rt(void 0,t),function(d){var c=o.__N?o.__N[0]:o.__[0],p=o.t(c,d);c!==p&&(o.__N=[p,o.__[1]],o.__c.setState({}))}],o.__c=y,!y.__f)){var s=function(d,c,p){if(!o.__c.__H)return!0;var m=!1,g=o.__c.props!==d;if(o.__c.__H.__.some(function(f){if(f.__N){m=!0;var v=f.__[0];f.__=f.__N,f.__N=void 0,v!==f.__[0]&&(g=!0)}}),a){var u=a.call(this,d,c,p);return m?u||g:u}return!m||g};y.__f=!0;var a=y.shouldComponentUpdate,i=y.componentWillUpdate;y.componentWillUpdate=function(d,c,p){if(this.__e){var m=a;a=void 0,s(d,c,p),a=m}i&&i.call(this,d,c,p)},y.shouldComponentUpdate=s}return o.__N||o.__}function M(e,t){var r=Ge(X++,3);!k.__s&&Nt(r.__H,t)&&(r.__=e,r.u=t,y.__H.__h.push(r))}function Ce(e){return ke=5,An(function(){return{current:e}},[])}function An(e,t){var r=Ge(X++,7);return Nt(r.__H,t)&&(r.__=e(),r.__H=t,r.__h=e),r.__}function En(){for(var e;e=zt.shift();){var t=e.__H;if(e.__P&&t)try{t.__h.some(ye),t.__h.some(Ve),t.__h=[]}catch(r){t.__h=[],k.__e(r,e.__v)}}}k.__b=function(e){y=null,Tt&&Tt(e)},k.__=function(e,t){e&&t.__k&&t.__k.__m&&(e.__m=t.__k.__m),Pt&&Pt(e,t)},k.__r=function(e){At&&At(e),X=0;var t=(y=e.__c).__H;t&&(Be===y?(t.__h=[],y.__h=[],t.__.some(function(r){r.__N&&(r.__=r.__N),r.u=r.__N=void 0})):(t.__h.some(ye),t.__h.some(Ve),t.__h=[],X=0)),Be=y},k.diffed=function(e){Et&&Et(e);var t=e.__c;t&&t.__H&&(t.__H.__h.length&&(zt.push(t)!==1&&$t===k.requestAnimationFrame||(($t=k.requestAnimationFrame)||In)(En)),t.__H.__.some(function(r){r.u&&(r.__H=r.u,r.u=void 0)})),Be=y=null},k.__c=function(e,t){t.some(function(r){try{r.__h.some(ye),r.__h=r.__h.filter(function(o){return!o.__||Ve(o)})}catch(o){t.some(function(s){s.__h&&(s.__h=[])}),t=[],k.__e(o,r.__v)}}),It&&It(e,t)},k.unmount=function(e){Ot&&Ot(e);var t,r=e.__c;r&&r.__H&&(r.__H.__.some(function(o){try{ye(o)}catch(s){t=s}}),r.__H=void 0,t&&k.__e(t,r.__v))};var Mt=typeof requestAnimationFrame=="function";function In(e){var t,r=function(){clearTimeout(o),Mt&&cancelAnimationFrame(t),setTimeout(e)},o=setTimeout(r,35);Mt&&(t=requestAnimationFrame(r))}function ye(e){var t=y,r=e.__c;typeof r=="function"&&(e.__c=void 0,r()),y=t}function Ve(e){var t=y;e.__c=e.__(),y=t}function Nt(e,t){return!e||e.length!==t.length||t.some(function(r,o){return r!==e[o]})}function Rt(e,t){return typeof t=="function"?t(e):t}var Se=[{label:null,items:["get-running"]},{label:"Before production",items:["auth","authorization","rate_limit","observability","token_revocation","graph_introspection"]},{label:"Runs and streaming",items:["runs","websocket","ag_ui","remote_tools","media"]},{label:"Project wiring",items:["plugins"]},{label:"Developer tools",items:["test","evaluation"]},{label:"Reference",items:["environment"]}],N={"get-running":{title:"Get running",icon:"flag",description:"Only the agent graph is required. Everything else is optional."},environment:{title:"Server environment (.env)",icon:"terminal",description:"Variables the server reads from .env. The editor never reads that file, so values are not shown."}},Ke={"auth.path":"auth-path",authorization:"authz-path"},jt="tg-config-token",Dt=!1;function On(){let e=new URLSearchParams(location.hash.slice(1)).get("token")||"";try{e&&sessionStorage.setItem(jt,e);let t=sessionStorage.getItem(jt)||"";return Dt=t!==""&&t===(e||t),e||t}catch{return e}}var ee=On(),Ye=e=>Dt||!ee?`#${e}`:`#token=${encodeURIComponent(ee)}&page=${e}`;function Ht(){let e=location.hash.slice(1);if(e.includes("=")){let t=new URLSearchParams(e);return t.get("page")||t.get("section")||""}try{return decodeURIComponent(e)}catch{return e}}var l={schema:null,sections:[],nav:[],fields:new Map,config:{},version:null,exists:!1,path:"",parseError:null,saved:"",validated:null,issues:[],page:"get-running",query:"",sideOpen:!0,sideRender:!0,sideShown:!0,drawerOpen:!1,tab:"json",copied:!1,busy:null,toolParams:{},toasts:[],fatal:null},Lt=()=>{},Ft=e=>{Lt=e};function _(e){e&&e(l),Lt()}var qt=e=>JSON.parse(JSON.stringify(e)),I=()=>JSON.stringify(l.config),z=e=>e!==null&&typeof e=="object"&&!Array.isArray(e),q=(e,t)=>t.reduce((r,o)=>z(r)?r[o]:void 0,e),te=(e,t)=>`${e} ${t}${e===1?"":"s"}`;function Ut(e,t,r){let o=e;for(let s of t.slice(0,-1))z(o[s])||(o[s]={}),o=o[s];o[t[t.length-1]]=r}function Ze(e,t){let r=[],o=e;for(let s of t.slice(0,-1)){if(!z(o))return;r.push([o,s]),o=o[s]}z(o)&&delete o[t[t.length-1]];for(let s=r.length-1;s>=1;s--){let[a,i]=r[s];if(z(a[i])&&Object.keys(a[i]).length===0)delete a[i];else break}}function V(e){let t=q(l.config,e);return t!==void 0?t:l.fields.get(e.join("."))?.default}var Pn=(e,t)=>JSON.stringify(e)===JSON.stringify(t),T=e=>l.sections.find(t=>t.id===e),R=()=>I()!==l.saved,$e=e=>!e.optional||e.key in l.config&&l.config[e.key]!==null,G=e=>l.issues.filter(t=>t.section===e),ne=e=>l.issues.find(t=>t.path===e);function re(e=l.issues){let t=e.filter(r=>r.level==="error").length;return{errors:t,warnings:e.length-t}}var Te=e=>Ke[e]||`f-${e.replaceAll(".","-")}`,Qe=e=>{let t=q(l.config,e.path);return t!==void 0&&!Pn(t,e.default)},Wt=e=>e.id==="core"?!1:e.optional?$e(e):e.fields.some(Qe);function Bt(e){let t=G(e);return t.find(r=>r.level==="error")||t[0]||null}var Vt=e=>!!e.show_if&&V(e.show_if.path)!==e.show_if.equals;function Gt(e){let r=l.fields.get(e.show_if.path.join("."))?.label||e.show_if.path.join("."),o=e.show_if.equals;return`Used when ${r} is ${o===!0?"on":o===!1?"off":String(o)}.`}var oe=e=>{let t=T(e);return N[e]?{id:e,...N[e]}:t?{id:e,title:t.title,description:t.description,icon:null}:null},Jt=()=>/^[\w.]+:[\w.]+$/.test(String(l.config.agent||""));async function et(e,t,r){let o={"X-10xGraph-Token":ee};r!==void 0&&(o["Content-Type"]="application/json");let s=await fetch(t,{method:e,headers:o,body:r===void 0?void 0:JSON.stringify(r)}),a={};try{a=await s.json()}catch{}return{ok:s.ok,status:s.status,data:a}}var Xe=(e,t)=>_(r=>{r.fatal={title:e,text:t}});async function Zt(){if(!ee)return Xe("Open the link printed by 10xgraph config","This page needs the session token included in that link.");let e;try{e=await et("GET","/api/state")}catch{return Xe("Cannot reach the config server","Check that 10xgraph config is still running in your terminal, then reload.")}if(!e.ok)return Xe("This link has expired",e.data.error||"Restart 10xgraph config and open the new link it prints.");let t=e.data;_(r=>{r.schema=t.schema,r.sections=t.schema.sections;let o=new Set(Se.flatMap(i=>i.items)),s=r.sections.map(i=>i.id).filter(i=>i!=="core"&&!o.has(i));r.nav=Se.map((i,d)=>({label:i.label,items:dN[c]||T(c)):[...s,...i.items]}));for(let i of r.sections)for(let d of i.fields)r.fields.set(d.path.join("."),d);r.config=t.config,r.version=t.version,r.exists=t.exists,r.path=t.path,r.parseError=t.parse_error,r.issues=t.issues;let a=t.exists&&!t.parse_error;r.saved=a?I():"",r.validated=a?I():null}),Ee({focus:!1}),history.replaceState(null,"",Ye(l.page))}var Kt=null;async function Qt(){let e=I();if(!(!l.schema||l.validated===e||Kt===e)&&!Object.values(l.toolParams).some(t=>t.error)){Kt=e;try{let t=await et("POST","/api/validate",{config:JSON.parse(e)});if(!t.ok)return;_(r=>{r.issues=t.data.issues,r.validated=e})}catch{}}}async function Ae(){if(!(l.busy||!l.schema)){if(l.exists&&!R()&&!l.parseError)return j("success","No changes to save.");if(Mn()){_(e=>{e.busy="save"});try{let e=await et("POST","/api/save",{config:l.config,version:l.version});if(e.data.issues&&(l.issues=e.data.issues,l.validated=I()),e.ok){l.version=e.data.version,l.config=e.data.config,l.toolParams={},l.validated=I(),l.saved=I(),l.exists=!0,l.parseError=null;let t=e.data.path.split(/[\\/]/).pop();j("success",e.data.backup?`Saved ${t}. The previous version is in ${t}.bak.`:`Created ${t}.`)}else if(e.status===409)j("error",e.data.error,{label:"Reload",run:()=>location.reload()});else if(e.status===422){let{errors:t}=re();j("error",`Not saved: ${te(t,"error")} to fix first.`,{label:"Show",run:zn})}else j("error",e.data.error||"Save failed.")}catch{j("error","Cannot reach the config server.")}finally{_(e=>{e.busy=null})}}}}function Mn(){let e=Object.values(l.toolParams).some(t=>t.error);return e&&(O("remote_tools"),j("error","A remote tool has invalid parameters JSON.")),!e}function Xt(e,t){_(r=>{t?r.config[e.key]=qt(e.default):delete r.config[e.key],e.key==="remote_tools"&&(r.toolParams={})})}function zn(){let e=l.issues.find(t=>t.level==="error")||l.issues[0];e&&O(e.section,Te(e.path))}var Nn=()=>window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth";function Rn(e){return e==="core"?"get-running":N[e]||T(e)?e:"get-running"}function Ee({focus:e=!0}={}){_(t=>{t.page=Rn(Ht()),t.query="",t.drawerOpen=!1}),e&&requestAnimationFrame(()=>{window.scrollTo(0,0),document.getElementById("page-title")?.focus({preventScroll:!0})})}function O(e,t){let r=Ye(e);location.hash!==r&&history.pushState(null,"",r),Ee(),t&&requestAnimationFrame(()=>requestAnimationFrame(()=>{let o=document.getElementById(t);o&&(o.scrollIntoView({block:"center",behavior:Nn()}),o.focus({preventScroll:!0}))}))}function Ie(e){_(t=>{t.query=e})}var jn=()=>window.matchMedia("(prefers-reduced-motion: reduce)").matches,Yt;function J(e,{instant:t=!1}={}){try{localStorage.setItem("tg-sidebar",e?"shown":"hidden")}catch{}clearTimeout(Yt);let r=t||jn();_(o=>{o.sideOpen=e,e?(o.sideRender=!0,o.sideShown=r):o.sideShown=!1,!e&&r&&(o.sideRender=!1)}),e&&!r&&requestAnimationFrame(()=>requestAnimationFrame(()=>_(o=>{o.sideShown=!0}))),!e&&!r&&(Yt=setTimeout(()=>_(o=>{o.sideOpen||(o.sideRender=!1)}),200))}function tt(){l.sideOpen||J(!0,{instant:!0}),requestAnimationFrame(()=>{let e=document.getElementById("q");e?.focus(),e?.select()})}function en(e){_(t=>{e&&(t.tab=e),t.drawerOpen=!0})}var Oe=()=>_(e=>{e.drawerOpen=!1});async function tn(){let e="10xgraph dev";try{await navigator.clipboard.writeText(e)}catch{try{let t=document.createElement("textarea");t.value=e,document.body.append(t),t.select(),document.execCommand("copy"),t.remove()}catch{}}_(t=>{t.copied=!0}),setTimeout(()=>_(t=>{t.copied=!1}),2e3)}async function nn(){try{await navigator.clipboard.writeText(JSON.stringify(l.config,null,2)+` +`),j("success","Copied 10xgraph.json to the clipboard.")}catch{j("error","The clipboard is not available here.")}}var Dn=0;function j(e,t,r){let o=++Dn;_(s=>{s.toasts=[...s.toasts,{id:o,kind:e,message:t,action:r}]}),setTimeout(()=>Pe(o),r?8e3:4e3)}function Pe(e){_(t=>{t.toasts=t.toasts.filter(r=>r.id!==e)})}var rn={box:'',key:'',shield:'',gauge:'',arrows:'',activity:'',layers:'',wrench:'',flask:'',target:'',checkCircle:'',xCircle:'',alert:'',info:'',file:'',copy:'',x:'',plus:'',trash:'',panel:'',chevronDown:'',expand:'',collapse:'',power:'',spinner:'',book:'',flag:'',lock:'',play:'',bolt:'',terminal:'',search:'',chevronRight:'',code:'',circle:'',external:''},U={"get-running":"flag",environment:"terminal",core:"flag",auth:"lock",authorization:"shield",token_revocation:"key",graph_introspection:"code",rate_limit:"gauge",observability:"activity",runs:"play",websocket:"bolt",ag_ui:"external",remote_tools:"wrench",media:"file",plugins:"layers",test:"flask",evaluation:"target"};var Hn=0;function n(e,t,r,o,s,a){t||(t={});var i,d,c=t;if("ref"in c)for(d in c={},t)d=="ref"?i=t[d]:c[d]=t[d];var p={type:e,props:c,key:r,ref:i,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:--Hn,__i:-1,__u:0,__source:s,__self:a};if(typeof e=="function"&&(i=e.defaultProps))for(d in i)c[d]===void 0&&(c[d]=i[d]);return x.vnode&&x.vnode(p),p}function h({name:e,class:t="size-4"}){return/(^|\s)size-/.test(t)||(t=`size-4 ${t}`),n("span",{class:`inline-flex shrink-0 ${t}`,"aria-hidden":"true",children:n("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor","stroke-width":"1.75","stroke-linecap":"round","stroke-linejoin":"round",class:"size-full",dangerouslySetInnerHTML:{__html:rn[e]}})})}function Me({class:e,children:t}){return n("span",{class:`inline-flex h-5 shrink-0 items-center rounded-full px-2 text-[11px] font-semibold ${e}`,children:t})}function ae({id:e,label:t,options:r,value:o,onPick:s}){return n("div",{id:e,role:"radiogroup","aria-label":t,class:"inline-flex max-w-full flex-wrap gap-0.5 rounded-[10px] border border-line-strong bg-raised p-0.5",children:r.map(([a,i])=>n("button",{type:"button",role:"radio","aria-checked":a===o?"true":"false",class:`focus-ring min-h-10 cursor-pointer rounded-lg px-3.5 text-sm font-medium transition-colors duration-150 ${a===o?"bg-surface text-ink shadow-card":"text-muted hover:text-ink"}`,onClick:()=>{a!==o&&s(a)},children:i},a))})}function ze({id:e,options:t,value:r,invalid:o,onChange:s}){return n("div",{class:"relative",children:[n("select",{id:e,value:r,class:`input cursor-pointer appearance-none pr-9 font-mono ${o?"input-invalid":""}`,onChange:a=>s(a.currentTarget.value),children:t.map(([a,i])=>n("option",{value:a,children:i},a))}),n(h,{name:"chevronDown",class:"pointer-events-none absolute top-1/2 right-3 size-4 -translate-y-1/2 text-faint"})]})}function se({id:e,values:t,placeholder:r,suggestions:o,onChange:s}){let a=Ce(null),i=c=>{let p=c.trim();a.current&&(a.current.value=""),p&&!t.includes(p)&&s([...t,p])},d=o?`${e}-options`:void 0;return n("div",{class:"flex min-h-11 cursor-text flex-wrap items-center gap-1.5 rounded-lg border border-line-strong bg-sunken px-1.5 py-1.5 transition-[border-color,box-shadow] duration-150 focus-within:border-brand focus-within:ring-4 focus-within:ring-brand-ring sm:py-1",onClick:c=>{c.target===c.currentTarget&&a.current?.focus()},children:[t.map((c,p)=>n("span",{class:"inline-flex h-7 items-center gap-0.5 rounded-md border border-line bg-raised pr-0.5 pl-2 font-mono text-xs text-body sm:h-6",children:[c,n("button",{type:"button","aria-label":`Remove ${c}`,class:"inline-flex size-6 cursor-pointer items-center justify-center rounded text-faint transition-colors hover:bg-hover hover:text-ink sm:size-5",onClick:()=>s(t.filter((m,g)=>g!==p)),children:n(h,{name:"x",class:"size-3"})})]},c)),n("input",{ref:a,id:e,type:"text",list:d,placeholder:t.length?"Add...":r,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:"h-7 min-w-[7rem] flex-1 bg-transparent px-1 font-mono text-base text-ink outline-none placeholder:text-faint sm:text-sm",onKeyDown:c=>{let p=c.currentTarget;(c.key==="Enter"||c.key===",")&&p.value.trim()?(c.preventDefault(),i(p.value)):c.key==="Backspace"&&!p.value&&t.length&&s(t.slice(0,-1))},onChange:c=>{o?.includes(c.currentTarget.value.trim())&&i(c.currentTarget.value)},onBlur:c=>i(c.currentTarget.value)}),o?n("datalist",{id:d,children:o.filter(c=>!t.includes(c)).map(c=>n("option",{value:c},c))}):null]})}function ie({label:e,path:t,help:r,id:o,required:s}){return n("div",{class:"min-w-0",children:[n("label",{for:o,class:"block text-sm font-medium text-ink",children:[e,s?n("span",{class:"text-danger",children:" *"}):null]}),n("div",{class:"mt-0.5 font-mono text-xs text-faint",children:t}),r?n("p",{class:"mt-1.5 text-[13px] leading-5 text-muted",children:r}):null]})}function le({issue:e}){if(!e)return null;let t=e.level==="error";return n("p",{class:`mt-2 flex gap-1.5 text-[13px] leading-5 ${t?"text-danger":"text-warn"}`,role:t?"alert":void 0,children:[n(h,{name:t?"xCircle":"alert",class:"mt-0.5 size-3.5"}),n("span",{children:e.message})]})}function nt({children:e}){return n("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[minmax(0,1fr)_minmax(0,1.1fr)] sm:gap-8",children:e})}function K({id:e,label:t,path:r,value:o,placeholder:s,onInput:a,issue:i,help:d}){return n(nt,{children:[n(ie,{label:t,path:r,help:d,id:e}),n("div",{class:"min-w-0 sm:pt-0.5",children:[n("input",{id:e,type:"text",value:o??"",placeholder:s,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${i?.level==="error"?"input-invalid":""}`,onInput:c=>a(c.currentTarget.value)}),n(le,{issue:i})]})]})}function rt({name:e,label:t,options:r,value:o,onPick:s}){return n("div",{role:"radiogroup","aria-label":t,class:"grid gap-3 sm:grid-cols-2",children:r.map(([a,i,d])=>{let c=a===o;return n("button",{type:"button",role:"radio","aria-checked":c?"true":"false",id:`${e}-${a}`,class:`focus-ring flex cursor-pointer items-start gap-3 rounded-xl border p-4 text-left transition-colors duration-150 ${c?"border-brand bg-brand-soft":"border-line bg-surface shadow-card hover:border-line-strong"}`,onClick:()=>{c||s(a)},children:[n("span",{class:`mt-0.5 flex size-4 shrink-0 items-center justify-center rounded-full border ${c?"border-brand-solid bg-brand-solid":"border-line-strong"}`,children:c?n("span",{class:"size-1.5 rounded-full bg-white"}):null}),n("span",{class:"min-w-0",children:[n("span",{class:"block text-sm font-medium text-ink",children:i}),n("span",{class:"mt-0.5 block text-[13px] leading-5 text-muted",children:d})]})]},a)})})}function W({title:e,children:t}){return n("section",{children:[e?n("h4",{class:"eyebrow mb-2.5",children:e}):null,t]})}var Ln={info:["info","border-brand/20 bg-brand-soft","text-brand"],warn:["alert","border-warn/25 bg-warn-soft","text-warn"],error:["xCircle","border-danger/25 bg-danger-soft","text-danger"]};function ce({kind:e,title:t,lines:r}){let[o,s,a]=Ln[e];return n("div",{class:`flex gap-3 rounded-xl border px-4 py-3 ${s}`,role:e==="error"?"alert":void 0,children:[n(h,{name:o,class:`mt-0.5 size-4 ${a}`}),n("div",{class:"min-w-0 space-y-1 text-[13px] leading-5 text-body",children:[t?n("p",{class:"font-medium text-ink",children:t}):null,r.map((i,d)=>n("p",{children:i},d))]})]})}function ot({issue:e}){return n(ce,{kind:e.level==="error"?"error":"warn",lines:[n("span",{children:[e.path?n("code",{class:"mr-1.5 font-mono text-xs text-muted",children:e.path}):null,e.message]})]})}function on({icon:e,tint:t,title:r,text:o,children:s}){return n("div",{class:"px-6 py-14 text-center",children:[n("div",{class:`mx-auto flex size-11 items-center justify-center rounded-full bg-raised ${t}`,children:n(h,{name:e,class:"size-5"})}),n("p",{class:"mt-4 text-sm font-semibold text-ink",children:r}),n("p",{class:"mx-auto mt-1 max-w-xs text-sm leading-6 text-muted",children:o}),s]})}function an(){let e=Ce(null);M(()=>{let i=e.current;i&&(l.drawerOpen&&!i.open?i.showModal():!l.drawerOpen&&i.open&&i.close())});let t=l.issues.length>0,r=t&&l.tab==="problems"?"problems":"json",{errors:o,warnings:s}=re(),a=(i,d,c)=>n("button",{type:"button",role:"tab",id:`tab-${i}`,"aria-selected":r===i?"true":"false",class:`focus-ring inline-flex h-10 cursor-pointer items-center gap-2 rounded-lg px-3 text-sm font-medium transition-colors duration-150 ${r===i?"bg-raised text-ink":"text-muted hover:text-ink"}`,onClick:()=>_(p=>{p.tab=i}),children:[d,c]});return n("dialog",{ref:e,"aria-label":"10xgraph.json preview",class:"m-0 ml-auto h-dvh max-h-none w-[min(540px,100vw)] border-0 border-l border-line-strong bg-surface p-0 text-body shadow-pop backdrop:bg-black/45",onClose:()=>{l.drawerOpen&&Oe()},onClick:i=>{i.target===i.currentTarget&&Oe()},children:n("div",{class:"flex h-full flex-col",children:[n("div",{class:"flex h-14 shrink-0 items-center gap-1 border-b border-line px-3",children:[n("div",{role:"tablist",class:"flex flex-1 gap-1",children:[a("json","10xgraph.json",null),t?a("problems","Problems",n("span",{class:`inline-flex h-5 min-w-5 items-center justify-center rounded-full px-1.5 text-[11px] font-semibold ${o?"bg-danger-soft text-danger":"bg-warn-soft text-warn"}`,children:o+s})):null]}),n("button",{type:"button",class:"icon-btn","aria-label":"Close preview",onClick:Oe,children:n(h,{name:"x",class:"size-5"})})]}),n("div",{class:"min-h-0 flex-1 overflow-y-auto overscroll-contain",children:r==="problems"?n(Fn,{}):n(Un,{})})]})})}function Fn(){let{errors:e,warnings:t}=re(),r=l.sections.map(o=>[o,G(o.id)]).filter(([,o])=>o.length);return n("div",{children:[n("div",{class:"border-b border-line px-4 py-3",children:n("p",{class:"text-sm text-muted",children:[n("span",{class:e?"font-medium text-danger":"",children:te(e,"error")})," \xB7 ",n("span",{class:t?"font-medium text-warn":"",children:te(t,"warning")})]})}),n("div",{class:"space-y-5 px-3 py-4",children:r.map(([o,s])=>n("div",{children:[n("p",{class:"eyebrow mb-1.5 flex items-center gap-2 px-2",children:[n(h,{name:U[o.id]||"box",class:"size-3.5"}),o.title]}),n("ul",{class:"space-y-0.5",children:s.map((a,i)=>{let d=a.level==="error";return n("li",{children:n("button",{type:"button",onClick:()=>O(o.id,Te(a.path)),class:"focus-ring flex min-h-11 w-full cursor-pointer gap-2.5 rounded-lg px-2 py-2 text-left transition-colors duration-150 hover:bg-raised",children:[n(h,{name:d?"xCircle":"alert",class:`mt-0.5 size-4 ${d?"text-danger":"text-warn"}`}),n("span",{class:"min-w-0",children:[n("span",{class:"block text-[13px] leading-5 text-body",children:a.message}),a.path?n("span",{class:"mt-0.5 block truncate font-mono text-xs text-faint",children:a.path}):null]})]})},i)})})]},o.id))})]})}function qn(e){return e.replace(/&/g,"&").replace(//g,">").replace(/("(\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?)/g,r=>{let o="var(--af-json-number)";return r.startsWith('"')?o=r.endsWith(":")?"var(--af-json-key)":"var(--af-json-string)":(r==="true"||r==="false"||r==="null")&&(o="var(--af-json-literal)"),`${r}`})}function Un(){let e=JSON.stringify(l.config,null,2).split(` +`);return n("div",{children:[n("div",{class:"sticky top-0 flex items-center justify-between gap-3 border-b border-line bg-surface px-4 py-2.5",children:[n("p",{class:"text-xs text-faint",children:R()||!l.exists?"Preview of what Save will write":"Matches the file on disk"}),n("button",{type:"button",class:"btn btn-ghost h-8 px-2.5 text-xs",onClick:nn,children:[n(h,{name:"copy",class:"size-3.5"}),"Copy"]})]}),n("div",{class:"py-3 pr-4 font-mono text-[12.5px] leading-5",children:e.map((t,r)=>n("div",{class:"flex",children:[n("span",{class:"w-10 shrink-0 pr-3 text-right text-faint/70 select-none",children:r+1}),n("span",{class:"min-w-0 flex-1 break-all whitespace-pre-wrap",dangerouslySetInnerHTML:{__html:qn(t)}})]},r))})]})}function st(e){let t=[];for(let r of e){let o=r.group||"",s=t.find(a=>a.name===o);s||t.push(s={name:o,fields:[]}),s.fields.push(r)}return t}var D=(e,t)=>_(r=>{if(t===void 0){Ze(r.config,e);return}let o=e.length>1?r.sections.find(s=>s.optional&&s.key===e[0]):null;o&&(r.config[o.key]===void 0||r.config[o.key]===null)&&(r.config[o.key]=JSON.parse(JSON.stringify(o.default))),Ut(r.config,e,t)}),Wn=e=>`f-${e.path.join("-")}`;function Bn({id:e,value:t,field:r,invalid:o,disabled:s}){let[a,i]=we(String(t??""));return M(()=>{(a===""?t!==void 0:Number(a)!==t)&&i(String(t??""))},[t]),n("input",{id:e,type:"number",inputmode:"decimal",value:a,disabled:s,min:r.min,step:r.step??"any",placeholder:r.placeholder??r.default??"",class:`input max-w-40 font-mono tabular-nums ${o?"input-invalid":""}`,onInput:d=>{let c=d.currentTarget.value;i(c),c===""?D(r.path,void 0):Number.isNaN(Number(c))||D(r.path,Number(c))}})}function at({field:e,value:t}){return e.exclusive_min===void 0||typeof t!="number"||t>e.exclusive_min?null:n("p",{class:"mt-1.5 text-[13px] text-warn",children:["Must be greater than ",e.exclusive_min,"."]})}function Vn({id:e,field:t,value:r,invalid:o}){let s=t.unlimited_value!==void 0&&r===t.unlimited_value,a=n(Bn,{id:e,value:s?void 0:r,field:t,invalid:o,disabled:s});if(t.null_label!==void 0){let i=r===null?"n":r===void 0?"d":"s",d=typeof t.default=="number"?t.default:300;return n("div",{children:[n(ae,{id:e,label:t.label,value:i,options:[["d","Default"],["s","Seconds"],["n",t.null_label]],onPick:c=>D(t.path,c==="d"?void 0:c==="n"?null:d)}),i==="s"?n("div",{class:"mt-2.5",children:[a,n(at,{field:t,value:r})]}):null]})}return t.unlimited_value!==void 0?n("div",{children:[n("div",{class:"flex flex-wrap items-center gap-x-4",children:[a,n("label",{class:"inline-flex min-h-11 cursor-pointer items-center gap-2.5 text-sm text-body",children:[n("input",{type:"checkbox",class:"size-[18px] accent-brand-solid",checked:s,onChange:i=>D(t.path,i.currentTarget.checked?t.unlimited_value:void 0)}),"Unlimited"]})]}),n(at,{field:t,value:r})]}):n("div",{children:[a,n(at,{field:t,value:r})]})}function Gn({id:e,field:t,value:r}){if(t.unset_label!==void 0){let o=[["d","Default"]];return t.allow_true!==!1&&o.push(["t","On"]),o.push(["f","Off"]),n("div",{children:[n(ae,{id:e,label:t.label,value:r===void 0?"d":r?"t":"f",options:o,onPick:s=>D(t.path,s==="d"?void 0:s==="t")}),n("p",{class:"mt-1.5 text-xs text-faint",children:["Default: ",t.unset_label]})]})}return n("label",{class:"inline-flex min-h-11 cursor-pointer items-center gap-2.5 text-sm font-semibold text-ink",children:[n("input",{id:e,type:"checkbox",class:"size-[18px] accent-brand-solid",checked:(r??t.default??!1)===!0,onChange:o=>D(t.path,o.currentTarget.checked)}),t.label]})}function Jn({id:e,field:t,value:r,invalid:o}){if(t.type==="bool")return n(Gn,{id:e,field:t,value:r});if(t.type==="number")return n(Vn,{id:e,field:t,value:r,invalid:o});if(t.type==="list")return n(se,{id:e,values:Array.isArray(r)?r:[],placeholder:t.placeholder,onChange:s=>D(t.path,s.length?s:void 0)});if(t.type==="select"){let s=t.options.map(a=>[a,a]);return t.unset_label!==void 0&&s.unshift(["",t.unset_label]),n(ze,{id:e,options:s,invalid:o,value:r??(t.unset_label!==void 0?"":t.default??t.options[0]),onChange:a=>D(t.path,a===""?void 0:a)})}return n("input",{id:e,type:"text",value:r??"",placeholder:t.placeholder,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${o?"input-invalid":""}`,onInput:s=>D(t.path,s.currentTarget.value===""?void 0:s.currentTarget.value)})}function Ne({field:e}){let t=e.path.join("."),r=Wn(e),o=q(l.config,e.path),s=ne(t),a=Qe(e),i=Vt(e),d=e.type==="bool"&&e.unset_label===void 0,c=e.default!==void 0&&!a,p=Array.isArray(e.default)?e.default.join(", "):String(e.default??"");return n("div",{id:`row-${r}`,class:`relative pt-5 pb-1 ${a?"before:absolute before:top-6 before:-left-2.5 before:bottom-2 before:w-[3px] before:rounded before:bg-brand before:content-[''] sm:before:-left-3.5":""}`,children:[n("div",{class:"flex min-h-6 items-center gap-2.5",children:[d?null:n("label",{for:r,class:`text-sm font-semibold text-ink ${i?"opacity-50":""}`,children:[e.label,e.required?n("span",{class:"text-danger",children:" *"}):null]}),d?null:n("code",{class:"hidden font-mono text-[11px] text-faint sm:inline",children:t}),c&&!d?n("span",{class:"rounded border border-line px-1.5 text-[11px] leading-4 font-medium text-faint",children:"default"}):null,n("span",{class:"flex-1"}),a?n("button",{type:"button",class:"focus-ring min-h-8 cursor-pointer rounded px-1 text-[13px] font-medium text-brand hover:underline",title:`Reset to default${e.default!==void 0?` (${p})`:""}`,"aria-label":`Reset ${e.label} to default`,onClick:()=>_(m=>Ze(m.config,e.path)),children:"Reset"}):null]}),e.help?n("p",{class:`mt-0.5 max-w-prose text-[13px] leading-5 text-muted ${i?"opacity-50":""}`,children:e.help}):null,n("fieldset",{disabled:i,class:`m-0 mt-2.5 min-w-0 max-w-[520px] border-0 p-0 ${i?"opacity-50":""}`,children:n(Jn,{id:r,field:e,value:o,invalid:s?.level==="error"})}),i?n("p",{class:"mt-1.5 text-xs text-faint",children:Gt(e)}):null,n(le,{issue:s})]})}function sn(){let e=l.config.auth,t=z(e)?"custom":"jwt";return n("div",{class:"space-y-6",children:[n(W,{title:"Method",children:n(rt,{name:"auth",label:"Authentication method",value:t,options:[["jwt","JWT","Verify bearer tokens signed with JWT_SECRET_KEY."],["custom","Custom backend","Your own BaseAuth subclass decides who is calling."]],onPick:r=>_(o=>{o.config.auth=r==="jwt"?"jwt":{method:"custom",path:""}})})}),t==="custom"?n(W,{title:"Backend",children:n("div",{class:"panel",children:n(K,{id:"auth-path",label:"Auth class",path:"auth.path",value:e.path,placeholder:"auth.custom:MyAuth",help:"Import path of a BaseAuth subclass, as module:attribute.",issue:ne("auth.path"),onInput:r=>_(o=>{o.config.auth={...o.config.auth,method:"custom",path:r}})})})}):null]})}function Kn(e){if(z(e))return"rbac";if(typeof e=="string"){let t=e.trim().toLowerCase();if(t==="ownership")return"ownership";if(["allow_all","default","none"].includes(t))return"allow_all"}return e==null?null:"custom"}var Yn={ownership:()=>"ownership",allow_all:()=>"allow_all",custom:()=>"",rbac:()=>({backend:"rbac",roles:{admin:["*"],member:["graph:invoke","graph:stream","graph:read","checkpointer:read"]},default_scopes:[],isolation:"owner"})};function ln({sec:e}){let t=l.config.authorization,r=Kn(t)??"ownership";return n("div",{class:"space-y-6",children:[n(W,{title:"Policy",children:n(rt,{name:"authz",label:"Authorization policy",value:r,options:[["ownership","Ownership","Users can only reach threads they created."],["allow_all","Allow all","Any authenticated user can do anything."],["rbac","Roles and scopes","Map roles to scopes, on top of owner isolation."],["custom","Custom backend","Your own AuthorizationBackend class."]],onPick:o=>_(s=>{s.config.authorization=Yn[o]()})})}),r==="custom"?n(W,{title:"Backend",children:n("div",{class:"panel",children:n(K,{id:"authz-path",label:"Backend class",path:"authorization",placeholder:"auth.permissions:MyBackend",value:typeof t=="string"?t:"",issue:ne("authorization"),help:"Import path of an AuthorizationBackend, as module:attribute.",onInput:o=>_(s=>{s.config.authorization=o})})})}):null,r==="rbac"?n(Qn,{sec:e,value:t}):null]})}function Zn({id:e,name:t,onRename:r}){let[o,s]=we(t);return M(()=>s(t),[t]),n("div",{children:[n("label",{for:e,class:"sr-only",children:"Role name"}),n("input",{id:e,type:"text",value:o,class:"input font-mono",autocomplete:"off",spellcheck:!1,onInput:a=>s(a.currentTarget.value),onChange:a=>{r(a.currentTarget.value.trim())||s(t)}})]})}function Qn({sec:e,value:t}){let r="roles"in t||!("role_scopes"in t)?"roles":"role_scopes",o=z(t[r])?t[r]:{},s=["*",...e.scopes||[]],a=Object.entries(o),i=c=>_(()=>{t[r]=c}),d=()=>{let c=a.length+1;for(;`role${c}`in o;)c++;i({...o,[`role${c}`]:[]})};return n(w,{children:[n(W,{title:"Roles",children:n("div",{class:"panel",children:[n("div",{class:"hidden grid-cols-[10rem_minmax(0,1fr)_auto] gap-3 border-b border-line px-5 py-2.5 sm:grid",children:[n("span",{class:"text-xs font-medium text-faint",children:"Role"}),n("span",{class:"text-xs font-medium text-faint",children:"Scopes"}),n("span",{class:"w-9"})]}),a.length?n("div",{class:"divide-y divide-line",children:a.map(([c,p],m)=>n("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[10rem_minmax(0,1fr)_auto] sm:items-start",children:[n(Zn,{id:`role-name-${m}`,name:c,onRename:g=>!g||g!==c&&g in o?!1:(i(Object.fromEntries(Object.entries(o).map(([u,f])=>[u===c?g:u,f]))),!0)}),n(se,{id:`role-scopes-${m}`,values:Array.isArray(p)?p:[],placeholder:"Add a scope",suggestions:s,onChange:g=>i({...o,[c]:g})}),n("button",{type:"button",class:"icon-btn justify-self-end hover:text-danger","aria-label":`Remove role ${c}`,onClick:()=>i(Object.fromEntries(Object.entries(o).filter(([g])=>g!==c))),children:n(h,{name:"trash"})})]},m))}):n("p",{class:"px-5 py-6 text-center text-sm text-muted",children:"No roles yet. Users only get the default scopes."}),n("div",{class:"flex items-center justify-between gap-4 border-t border-line px-5 py-3",children:[n("p",{class:"text-[13px] text-muted",children:["Read from the user's roles (or role) claim. ",n("code",{class:"font-mono",children:"*"})," grants every scope."]}),n("button",{type:"button",class:"btn btn-ghost",onClick:d,children:[n(h,{name:"plus"}),"Add role"]})]})]})}),n(W,{title:"Defaults",children:n("div",{class:"panel divide-y divide-line",children:[n("div",{class:"px-5 py-4",children:[n(ie,{label:"Default scopes",path:"authorization.default_scopes",id:"rbac-default-scopes",help:"Granted to every authenticated user, even without a role."}),n("div",{class:"mt-3",children:n(se,{id:"rbac-default-scopes",values:Array.isArray(t.default_scopes)?t.default_scopes:[],placeholder:"graph:read",suggestions:s,onChange:c=>_(()=>{t.default_scopes=c})})})]}),n(nt,{children:[n(ie,{label:"Thread isolation",path:"authorization.isolation",id:"rbac-isolation",help:"owner keeps each user's threads private; none shares them."}),n("div",{class:"min-w-0 sm:pt-0.5",children:n(ze,{id:"rbac-isolation",options:[["owner","owner"],["none","none"]],value:t.isolation??"owner",onChange:c=>_(()=>{t.isolation=c})})})]})]})})]})}function cn(){let e=Array.isArray(l.config.remote_tools)?l.config.remote_tools:[],t=()=>{_(r=>{e.push({node:"",name:"",description:"",parameters:{type:"object",properties:{},required:[]}}),r.config.remote_tools=e}),requestAnimationFrame(()=>document.getElementById(`tool-${e.length-1}-name`)?.focus())};return e.length?n("div",{class:"space-y-4",children:[e.map((r,o)=>n(Xn,{tools:e,tool:r,index:o},o)),n("button",{type:"button",onClick:t,class:"focus-ring flex h-12 w-full cursor-pointer items-center justify-center gap-2 rounded-xl border border-dashed border-line-strong text-sm font-medium text-muted transition-colors duration-150 hover:border-brand hover:text-brand",children:[n(h,{name:"plus"}),"Add remote tool"]})]}):n("div",{class:"rounded-xl border border-dashed border-line-strong bg-surface px-6 py-10 text-center",children:[n("div",{class:"mx-auto flex size-11 items-center justify-center rounded-full bg-raised text-faint",children:n(h,{name:"wrench",class:"size-5"})}),n("p",{class:"mt-4 text-sm font-semibold text-ink",children:"No remote tools yet"}),n("p",{class:"mx-auto mt-1 max-w-sm text-sm leading-6 text-muted",children:"Declare a tool here and the client that calls the API runs it, for example to read the user's location."}),n("button",{type:"button",class:"btn btn-outline mt-5",onClick:t,children:[n(h,{name:"plus"}),"Add remote tool"]})]})}function Xn({tools:e,tool:t,index:r}){let o=l.toolParams[r]??{text:JSON.stringify(t.parameters??{},null,2),error:null},s=i=>_(()=>{e[r]={...e[r],...i}}),a=i=>`remote_tools[${r}].${i}`;return n("section",{class:"panel",children:[n("div",{class:"flex items-center gap-3 border-b border-line px-5 py-3",children:[n(h,{name:"wrench",class:"size-4 text-faint"}),n("span",{class:"min-w-0 flex-1 truncate font-mono text-sm font-medium text-ink",children:t.name||`tool_${r+1}`}),t.node?n("span",{class:"rounded-md border border-line bg-raised px-1.5 py-0.5 font-mono text-xs text-muted",children:t.node}):null,n("button",{type:"button",class:"icon-btn hover:text-danger","aria-label":`Remove ${t.name||"tool"}`,onClick:()=>_(i=>{e.splice(r,1),i.toolParams={}}),children:n(h,{name:"trash"})})]}),n("div",{class:"divide-y divide-line",children:[n(K,{id:`tool-${r}-name`,label:"Tool name",path:a("name"),value:t.name,placeholder:"get_location",onInput:i=>s({name:i})}),n(K,{id:`tool-${r}-node`,label:"Graph node",path:a("node"),value:t.node,placeholder:"MAIN",help:"The node whose model may call this tool.",onInput:i=>s({node:i})}),n(K,{id:`tool-${r}-description`,label:"Description",path:a("description"),value:t.description,placeholder:"Returns the user's current city",help:"What the model reads when deciding to call it.",onInput:i=>s({description:i})}),n("div",{class:"px-5 py-4",children:[n(ie,{label:"Parameters",path:a("parameters"),help:"JSON schema of the arguments. type must be object.",id:`tool-${r}-params`}),n("textarea",{id:`tool-${r}-params`,rows:7,spellcheck:!1,value:o.text,class:`input mt-3 h-auto py-2.5 font-mono text-[13px] leading-5 ${o.error?"input-invalid":""}`,onInput:i=>{let d=i.currentTarget.value;_(c=>{try{e[r]={...e[r],parameters:JSON.parse(d)},c.toolParams[r]={text:d,error:null}}catch(p){c.toolParams[r]={text:d,error:p.message}}})}}),n(le,{issue:o.error?{level:"error",message:o.error}:null})]})]})]})}var ue=({children:e,code:t})=>n("h1",{id:"page-title",tabindex:-1,class:"flex flex-wrap items-baseline gap-x-3 gap-y-1 text-2xl font-semibold tracking-tight text-ink focus:outline-none",children:[e,t?n("code",{class:"rounded border border-line bg-raised px-1.5 font-mono text-xs font-normal text-faint",children:t}):null]}),it=({children:e})=>n("p",{class:"mt-1.5 max-w-[62ch] text-[15px] leading-6 text-muted",children:e}),un=({children:e})=>n("h2",{class:"mt-10 mb-2 eyebrow",children:e});function dn({id:e,text:t,warn:r,compact:o}){let s=oe(e);return s?n("a",{href:`#${e}`,onClick:a=>{a.preventDefault(),O(e)},class:`focus-ring group flex items-center gap-3.5 border-t border-line py-2 last:border-b ${o?"min-h-12":"min-h-14"}`,children:[n(h,{name:U[e]||"box",class:"size-[18px] text-faint"}),n("span",{class:`min-w-0 flex-1 ${o?"flex flex-wrap items-baseline gap-x-4":""}`,children:[n("span",{class:`block text-sm text-ink group-hover:text-brand ${o?"min-w-40 font-medium":"font-semibold"}`,children:s.title}),n("span",{class:`block text-[13px] leading-5 ${r?"text-warn":"text-muted"}`,children:t??s.description})]}),n(h,{name:"chevronRight",class:"size-4 text-faint"})]}):null}function lt(e){return`Not set. ${T(e)?.off_summary||""}`.trim()}function er(){let e=l.config.auth;return e==null?{text:lt("auth")}:e==="jwt"?{text:"JWT bearer tokens."}:e&&typeof e=="object"?{text:`Custom backend${e.path?`: ${e.path}`:"."}`}:{text:String(e)}}function tr(){let e=l.config.rate_limit;if(e==null)return{text:lt("rate_limit")};let t=s=>V(["rate_limit",s]);if(t("enabled")===!1)return{text:"Settings kept, not enforced."};let r={ip:"per IP",user:"per user",global:"across all clients"}[t("by")]||t("by"),o=["requests","window","by"].some(s=>q(l.config,["rate_limit",s])!==void 0&&q(l.config,["rate_limit",s])!==l.fields.get(`rate_limit.${s}`)?.default);return{text:`${t("requests")} requests per ${t("window")} s, ${r}${o?"":" (default)"}.`}}function nr(){let e=l.config.observability;if(e==null)return{text:lt("observability")};let t=[];return V(["observability","logfire","enabled"])&&t.push("Logfire"),V(["observability","langsmith","enabled"])&&t.push("LangSmith"),{text:`Level ${V(["observability","level"])}${t.length?`, sending to ${t.join(" and ")}`:", no destination chosen yet"}.`}}function pn(){let e=T("core"),t=Jt(),r=e?G("core").filter(a=>!e.fields.some(i=>i.path.join(".")===a.path)):[],o=[["auth",er()],["rate_limit",tr()],["observability",nr()]],s=l.nav.flatMap(a=>a.items).filter(a=>a!=="get-running");return n(w,{children:[n(ue,{children:t?"You're ready to run":"One step to run your agent"}),n(it,{children:N["get-running"].description}),n("div",{class:`mt-4 flex items-center gap-2 font-medium ${t?"text-ok":"text-muted"}`,children:[n(h,{name:t?"checkCircle":"circle",class:"size-[18px]"}),"Required: ",t?"1":"0"," of 1 done"]}),r.length?n("div",{class:"mt-4 space-y-2",children:r.map((a,i)=>n(ot,{issue:a},i))}):null,n("div",{class:"mt-2",children:e?st(e.fields).flatMap(a=>a.fields).map(a=>n(Ne,{field:a},a.path.join("."))):null}),t?n("div",{class:"mt-7 border-t border-line pt-5",children:[n("div",{class:"flex items-center gap-2.5 font-medium text-ok",children:[n(h,{name:"checkCircle",class:"size-[18px]"}),"Ready for 10xgraph dev"]}),n("div",{class:"mt-3 flex max-w-[420px] items-center gap-2 rounded-lg border border-line-strong bg-sunken py-0 pr-1 pl-3.5",children:[n("code",{class:"flex-1 font-mono text-[13px] font-medium text-ink",children:"10xgraph dev"}),l.copied?n("span",{class:"text-xs text-ok",children:"Copied"}):null,n("button",{type:"button",class:"icon-btn",onClick:tn,"aria-label":l.copied?"Copied 10xgraph dev":"Copy command 10xgraph dev",children:n(h,{name:"copy",class:"size-[18px]"})})]}),n("p",{class:"mt-3 text-muted",children:"Everything else already has working defaults."})]}):n("p",{class:"mt-4 text-xs text-faint",children:"Add the agent graph above, then run 10xgraph dev."}),n(un,{children:"Before production (optional)"}),n("div",{children:o.map(([a,i])=>n(dn,{id:a,text:i.text,warn:i.warn},a))}),n(un,{children:"What you can configure"}),n("div",{children:s.map(a=>n(dn,{id:a,compact:!0},a))})]})}function Re({vars:e}){return n("dl",{class:"mb-3 grid gap-x-4 gap-y-1 sm:grid-cols-[minmax(0,260px)_minmax(0,1fr)]",children:e.map(t=>[n("dt",{class:"py-0.5 font-mono text-xs font-medium break-all text-ink",children:t.name},`${t.name}-n`),n("dd",{class:"m-0 pb-2 text-[13px] text-muted sm:py-0.5",children:t.purpose},`${t.name}-p`)])})}function rr({vars:e,open:t}){return e?.length?n("details",{class:"group mt-9 border-t border-line pt-1",open:t,children:[n("summary",{class:"focus-ring flex min-h-11 cursor-pointer list-none items-center gap-2 font-medium text-muted [&::-webkit-details-marker]:hidden",children:[n(h,{name:"chevronRight",class:"size-4 transition-transform group-open:rotate-90"}),"Set in .env"]}),n("p",{class:"mb-2 text-xs text-faint",children:"Read only. The editor never reads .env, so only names are shown."}),n(Re,{vars:e})]}):null}function fn({sec:e}){return e.widget==="auth"?n(sn,{}):e.widget==="authorization"?n(ln,{sec:e}):e.widget==="remote_tools"?n(cn,{}):null}function or(e){let t=new Set([...e.fields.map(r=>r.path.join(".")),...Object.keys(Ke)]);return G(e.id).filter(r=>!t.has(r.path))}function ar({sec:e}){let t=$e(e),r=e.widget&&(t||!e.optional);return n(w,{children:[r?n("div",{class:"mt-6",children:n(fn,{sec:e})}):null,n("div",{class:e.optional&&!t?"opacity-80":"",children:st(e.fields).map((o,s)=>n("div",{children:[o.name?n("h3",{class:`eyebrow ${s===0?"mt-6":"mt-8 border-t border-line pt-4"}`,children:o.name}):null,o.fields.map(a=>n(Ne,{field:a},a.path.join(".")))]},o.name))})]})}function sr({sec:e}){let t=$e(e),r=or(e);return n(w,{children:[n(ue,{code:e.key,children:e.title}),n(it,{children:e.description}),r.length?n("div",{class:"mt-4 space-y-2",children:r.map((o,s)=>n(ot,{issue:o},s))}):null,e.optional?n("div",{class:"mt-5 flex flex-wrap items-center gap-4",children:[n(ae,{label:`Settings for ${e.title}`,value:t?"c":"d",options:[["d","Use defaults"],["c","Customize"]],onPick:o=>Xt(e,o==="c")}),n("p",{class:"m-0 min-w-56 flex-1 text-[13px] text-muted",children:t?`Saved in 10xgraph.json under ${e.key}.`:e.off_summary})]}):null,e.notes?.length?n("div",{class:"mt-5",children:n(ce,{kind:"info",lines:e.notes})}):null,n(ar,{sec:e}),n(rr,{vars:e.env})]})}function ir(){let e=l.schema.environment||[];return n(w,{children:[n(ue,{children:N.environment.title}),n(it,{children:"These variables are set in .env or your deployment environment. The editor never reads that file, so only names and purposes are shown."}),e.map(t=>n("section",{children:[n("h2",{class:"mt-8 mb-2 text-sm font-semibold text-ink",children:t.title}),t.note?n("p",{class:"mb-2 text-xs text-faint",children:t.note}):null,n(Re,{vars:t.vars})]},t.title))]})}var mn=e=>String(e??"").toLowerCase();function lr(e){let t=s=>e.every(a=>mn(s).includes(a)),r=[];for(let s of l.sections){let a=t(`${s.title} ${s.description} ${s.key||""}`),i=s.fields.filter(p=>a||t(`${p.label} ${p.help||""} ${p.path.join(".")}`)),d=(s.env||[]).filter(p=>t(`${p.name} ${p.purpose}`)),c=!!s.widget&&a;(i.length||d.length||c)&&r.push({id:s.id,sec:s,fields:i,env:d,widget:c})}let o=(l.schema.environment||[]).map(s=>({title:s.title,vars:s.vars.filter(a=>t(`${a.name} ${a.purpose} ${s.title}`))})).filter(s=>s.vars.length);return{results:r,envGroups:o}}function cr(){let e=mn(l.query).split(/\s+/).filter(Boolean),{results:t,envGroups:r}=lr(e),o=t.reduce((s,a)=>s+a.fields.length+a.env.length+(a.widget?1:0),0)+r.reduce((s,a)=>s+a.vars.length,0);return o?n(w,{children:[n(ue,{code:l.query,children:[o," ",o===1?"setting matches":"settings match"]}),t.map(({id:s,sec:a,fields:i,env:d,widget:c})=>n("section",{class:"mt-8",children:[n("h2",{class:"eyebrow m-0 flex items-center gap-2",children:[n(h,{name:U[s]||"box",class:"size-4"}),a.title,n("a",{href:`#${s}`,onClick:p=>{p.preventDefault(),O(s)},class:"focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline",children:"Open page"})]}),c?n("div",{class:"mt-4",children:n(fn,{sec:a})}):null,i.map(p=>n(Ne,{field:p},p.path.join("."))),d.length?n("details",{class:"group mt-5",open:!0,children:[n("summary",{class:"focus-ring flex min-h-11 cursor-pointer list-none items-center gap-2 font-medium text-muted [&::-webkit-details-marker]:hidden",children:[n(h,{name:"chevronRight",class:"size-4 transition-transform group-open:rotate-90"}),"Set in .env"]}),n(Re,{vars:d})]}):null]},s)),r.length?n("section",{class:"mt-8",children:[n("h2",{class:"eyebrow m-0 flex items-center gap-2",children:[n(h,{name:"terminal",class:"size-4"}),N.environment.title,n("a",{href:"#environment",onClick:s=>{s.preventDefault(),O("environment")},class:"focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline",children:"Open page"})]}),r.map(s=>n("div",{children:[n("h3",{class:"mt-4 mb-1.5 text-sm font-semibold text-ink",children:s.title}),s.note?n("p",{class:"mb-1.5 text-xs text-faint",children:s.note}):null,n(Re,{vars:s.vars})]},s.title))]}):null]}):n(w,{children:[n(ue,{children:"No matches"}),n("p",{class:"mt-8 text-muted",children:['Nothing matches "',l.query,'". Try a shorter word, or press Esc to go back.']})]})}function hn(){if(l.query.trim())return n(cr,{});if(l.page==="get-running")return n(pn,{});if(l.page==="environment")return n(ir,{});let e=T(l.page);return e?n(sr,{sec:e}):n(pn,{})}function ur(){let e=l.path.split(/[\\/]/).pop()||"10xgraph.json",t=l.path.slice(0,l.path.length-e.length),r=null;return l.schema?l.exists?R()?r=n(Me,{class:"bg-warn-soft text-warn",children:"Unsaved"}):r=n(Me,{class:"bg-ok-soft text-ok",children:"Saved"}):r=n(Me,{class:"bg-brand-soft text-brand",children:"New"}):r=null,n("div",{class:"flex min-w-0 flex-1 items-center gap-2",title:l.path||void 0,children:[n(h,{name:"file",class:"size-4 text-faint"}),n("span",{class:"truncate font-mono text-[13px] font-medium text-ink",children:e}),r,t?n("span",{class:"hidden min-w-0 truncate font-mono text-xs text-faint lg:block",dir:"rtl",children:n("bdi",{children:t})}):null]})}function _n(){let e=!!l.schema&&!l.fatal,t=l.sideOpen?"Hide sidebar":"Show sidebar";return n("header",{class:"sticky top-0 z-40 flex h-14 items-center gap-2 border-b border-line bg-surface px-3 sm:gap-3 sm:px-4",children:[n("button",{type:"button",class:"icon-btn","aria-controls":"side","aria-expanded":l.sideOpen?"true":"false","aria-label":t,title:`${t} (Ctrl/Cmd+B)`,onClick:()=>J(!l.sideOpen),children:n(h,{name:"panel",class:"size-[18px]"})}),n("div",{class:"flex shrink-0 items-center gap-2.5",children:[n("svg",{viewBox:"0 0 48 48",class:"size-7 shrink-0 text-ink","aria-hidden":"true",children:[n("rect",{x:"4",y:"4",width:"40",height:"40",rx:"11",fill:"none",stroke:"currentColor","stroke-width":"4.4"}),n("path",{d:"M13 33L21 23L34 15L34 31L21 23",fill:"none",stroke:"currentColor","stroke-width":"3","stroke-linecap":"round","stroke-linejoin":"round"}),n("g",{fill:"currentColor",children:[n("circle",{cx:"21",cy:"23",r:"4"}),n("circle",{cx:"34",cy:"15",r:"4"}),n("circle",{cx:"34",cy:"31",r:"4"})]}),n("circle",{cx:"13",cy:"33",r:"4.6",class:"fill-brand-amber"})]}),n("div",{class:"hidden items-baseline gap-1.5 sm:flex",children:[n("span",{class:"text-[15px] font-semibold tracking-tight text-ink",children:"10xGraph"}),n("span",{class:"text-[15px] text-faint",children:"Config"})]})]}),n("span",{class:"hidden h-6 w-px shrink-0 bg-line md:block","aria-hidden":"true"}),n(ur,{}),n("button",{type:"button",class:"btn btn-ghost","aria-label":"View 10xgraph.json","aria-haspopup":"dialog",disabled:!e,onClick:()=>en(),children:[n(h,{name:"code",class:"size-[18px]"}),n("span",{class:"hidden md:inline",children:"View 10xgraph.json"})]}),n("button",{type:"button",class:"btn btn-primary",onClick:Ae,disabled:!e||!!l.busy||l.exists&&!R()&&!l.parseError,children:[l.busy==="save"?n(h,{name:"spinner",class:"size-4 animate-spin"}):null,l.exists?"Save":"Create file",n("span",{class:"kbd hidden lg:inline-flex",children:"Ctrl S"})]})]})}function gn(){return l.fatal?n("div",{class:"panel mt-10",children:n(on,{icon:"xCircle",tint:"text-danger",title:l.fatal.title,text:l.fatal.text})}):l.schema?n(w,{children:[l.parseError?n("div",{class:"mb-6",children:n(ce,{kind:"warn",title:"The existing file is not valid JSON",lines:[`${l.parseError}. The form starts from defaults; saving replaces the file and keeps the old one as 10xgraph.json.bak.`]})}):null,n(hn,{})]}):n("div",{class:"animate-pulse space-y-4 pt-2","aria-label":"Loading",children:[n("div",{class:"h-7 w-48 rounded-md bg-raised"}),n("div",{class:"h-4 w-80 max-w-full rounded-md bg-raised"}),[0,1,2].map(e=>n("div",{class:"h-16 rounded-xl bg-raised"},e))]})}function dr({id:e}){let t=!!T(e)&&Wt(T(e)),r=Bt(e);return!t&&!r?null:n("span",{class:"flex shrink-0 gap-1",children:[t?n("span",{class:"size-2 rounded-full bg-brand",title:"Changed from default"}):null,r?n("span",{class:`size-2 rounded-full ${r.level==="error"?"bg-danger":"bg-warn"}`,title:r.level==="error"?"Needs attention":"Has a warning"}):null]})}function pr({id:e}){let t=oe(e);if(!t)return null;let r=!l.query&&l.page===e;return n("a",{href:`#${e}`,"aria-current":r?"page":void 0,onClick:o=>{o.preventDefault(),O(e)},ref:o=>{o&&r&&window.innerWidth<1024&&o.scrollIntoView({block:"nearest",inline:"nearest"})},class:`focus-ring flex min-h-10 items-center gap-2.5 rounded-lg px-3 font-medium transition-colors duration-150 max-lg:min-h-11 max-lg:shrink-0 max-lg:whitespace-nowrap ${r?"bg-brand-soft text-ink":"text-muted hover:bg-raised hover:text-ink"}`,children:[n(h,{name:U[e]||"box",class:`size-[18px] ${r?"text-brand":""}`}),n("span",{class:"min-w-0 flex-1 truncate",children:t.title}),n(dr,{id:e})]})}function xn(){return l.sideRender?n("aside",{id:"side","aria-label":"Settings navigation",class:`contents lg:sticky lg:top-14 lg:block lg:h-[calc(100dvh-3.5rem)] lg:overflow-y-auto lg:border-r lg:border-line lg:bg-surface lg:px-3 lg:pt-4 lg:pb-16 lg:transition-[transform,opacity] lg:duration-200 lg:ease-out ${l.sideShown?"lg:translate-x-0 lg:opacity-100":"lg:-translate-x-6 lg:opacity-0"}`,inert:!l.sideOpen,children:[n("div",{role:"search",class:"relative mx-1 mb-3 max-lg:mx-4 max-lg:mt-3",children:[n(h,{name:"search",class:"pointer-events-none absolute top-1/2 left-3 size-4 -translate-y-1/2 text-faint"}),n("label",{for:"q",class:"sr-only",children:"Search settings"}),n("input",{id:"q",type:"search",value:l.query,autocomplete:"off",spellcheck:!1,placeholder:"Search settings ( / )",class:"input pl-10",onInput:e=>Ie(e.currentTarget.value)})]}),n("nav",{"aria-label":"Pages",class:"max-lg:sticky max-lg:top-14 max-lg:z-20 max-lg:mt-3 max-lg:flex max-lg:gap-0.5 max-lg:overflow-x-auto max-lg:border-b max-lg:border-line max-lg:bg-surface max-lg:px-3 max-lg:py-1.5",children:l.nav.map(e=>n("div",{class:"max-lg:contents",children:[e.label?n("div",{class:"mt-4 mb-1 px-3 text-[11px] font-semibold tracking-wider text-faint uppercase max-lg:hidden",children:e.label}):null,e.items.map(t=>n(pr,{id:t},t))]},e.label||"top"))})]}):null}var fr={success:["checkCircle","text-ok"],warning:["alert","text-warn"],error:["xCircle","text-danger"]};function mr(){return n("div",{class:"pointer-events-none fixed right-4 bottom-4 left-4 z-50 flex flex-col items-end gap-2 sm:left-auto","aria-live":"polite",children:l.toasts.map(e=>{let[t,r]=fr[e.kind];return n("div",{role:e.kind==="error"?"alert":"status",class:"pointer-events-auto flex w-full max-w-sm items-start gap-3 rounded-xl border border-line bg-surface px-4 py-3 text-sm shadow-pop",children:[n(h,{name:t,class:`mt-0.5 size-4 ${r}`}),n("p",{class:"min-w-0 flex-1 leading-5 text-body",children:e.message}),e.action?n("button",{type:"button",class:"shrink-0 font-medium text-brand hover:underline",onClick:()=>{Pe(e.id),e.action.run()},children:e.action.label}):null,n("button",{type:"button",class:"-mr-1 shrink-0 cursor-pointer rounded text-faint hover:text-ink","aria-label":"Dismiss",onClick:()=>Pe(e.id),children:n(h,{name:"x"})})]},e.id)})})}var hr=()=>/^(INPUT|TEXTAREA|SELECT)$/.test(document.activeElement?.tagName)||document.activeElement?.isContentEditable;function _r(){let[,e]=Je(r=>r+1,0);M(()=>{Ft(e);try{localStorage.getItem("tg-sidebar")==="hidden"&&J(!1,{instant:!0})}catch{}let r=a=>{let i=a.ctrlKey||a.metaKey,d=a.key.toLowerCase();i&&d==="s"?(a.preventDefault(),Ae()):i&&d==="b"?(a.preventDefault(),J(!l.sideOpen)):i&&d==="k"?(a.preventDefault(),tt()):a.key==="/"&&!i&&!a.altKey&&!hr()&&!l.drawerOpen?(a.preventDefault(),tt()):a.key==="Escape"&&l.query&&!l.drawerOpen&&Ie("")},o=a=>{l.schema&&R()&&a.preventDefault()},s=()=>Ee();return document.addEventListener("keydown",r),window.addEventListener("beforeunload",o),window.addEventListener("hashchange",s),Zt(),()=>{document.removeEventListener("keydown",r),window.removeEventListener("beforeunload",o),window.removeEventListener("hashchange",s)}},[]),M(()=>{if(!l.schema||l.validated===I())return;let r=setTimeout(Qt,400);return()=>clearTimeout(r)});let t=l.sideRender;return n(w,{children:[n("a",{href:"#main",class:"sr-only z-50 rounded-md bg-brand-solid px-3 py-2 text-white focus:not-sr-only focus:fixed focus:top-3 focus:left-3",children:"Skip to content"}),n(_n,{}),n("div",{class:t?"lg:grid lg:grid-cols-[260px_minmax(0,1fr)] lg:items-start":"",children:[n(xn,{}),n("main",{id:"main",tabindex:-1,class:"min-w-0 px-4 pt-6 pb-24 focus:outline-none sm:px-8 sm:pt-8",children:n("div",{class:"mx-auto w-full max-w-[760px]",children:n(gn,{})})})]}),n(an,{}),n("div",{class:"sr-only",role:"status","aria-live":"polite",children:l.copied?"Copied":""}),n(mr,{})]})}St(n(_r,{}),document.getElementById("app")); diff --git a/tenxgraph_api/cli/config_editor/validation.py b/tenxgraph_api/cli/config_editor/validation.py index ce3c7f9f..a89fbfbe 100644 --- a/tenxgraph_api/cli/config_editor/validation.py +++ b/tenxgraph_api/cli/config_editor/validation.py @@ -117,8 +117,9 @@ def production_readiness_warnings(config: dict) -> list[Issue]: if rate_limit is None or (isinstance(rate_limit, dict) and rate_limit.get("enabled") is False): out.warning( "rate_limit", - "Rate limiting is off, so one caller can exhaust the service. Enable it, with the " - "redis backend for more than one worker, before production.", + "Rate limiting is off, so one caller can exhaust the service. Enable it, with a " + "shared cache store (redis, REDIS_URL or cache_store) for more than one worker, " + "before production.", ) elif isinstance(rate_limit, dict) and rate_limit.get("enabled") is not False: if auth and rate_limit.get("by", "ip") == "ip": @@ -134,11 +135,15 @@ def production_readiness_warnings(config: dict) -> list[Issue]: 'The ReDoc page is served at /redocs, not /redoc; "/redoc" excludes nothing.', ) - if auth and not config.get("token_revocation"): + revocation = config.get("token_revocation") + if auth and ( + not revocation or (isinstance(revocation, dict) and revocation.get("enabled") is False) + ): out.warning( "token_revocation", "Token revocation is off, so a leaked or logged-out token works until it expires. " - 'Set "token_revocation": "redis" and revoke with `10xgraph revoke`.', + 'Set "token_revocation": true with a shared cache store (redis, REDIS_URL or ' + "cache_store) and revoke with `10xgraph revoke`.", ) return out.issues @@ -243,33 +248,8 @@ def _check_rate_limit(config: dict, out: _Collector) -> None: from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig - probe = dict(data) - redis_value = probe.get("redis") - redis_url = redis_value.get("url") if isinstance(redis_value, dict) else redis_value - if isinstance(redis_url, str) and _has_unresolved_env(redis_url): - # The server expands $VAR at startup; it is fine for it to be unset while editing. - out.warning( - "rate_limit.redis.url", - f"{redis_url} is not set in this shell. Make sure it is set when the API runs.", - ) - placeholder = "redis://placeholder" - probe["redis"] = ( - {**redis_value, "url": placeholder} if isinstance(redis_value, dict) else placeholder - ) - - config_obj = _quietly(lambda: RateLimitConfig.from_dict(probe), out, "rate_limit") - if config_obj is None: - return - if config_obj.backend == "redis" and not redis_url: - out.warning( - "rate_limit.redis.url", - "The redis backend needs a URL unless a Redis client is bound in InjectQ.", - ) - if config_obj.enabled and config_obj.backend == "memory": - out.warning( - "rate_limit.backend", - "The memory backend counts per process. Use redis with more than one worker.", - ) + # rate_limit.backend and rate_limit.redis were removed: the parser names the replacement. + _quietly(lambda: RateLimitConfig.from_dict(data), out, "rate_limit") def _check_websocket(config: dict, out: _Collector) -> None: @@ -282,17 +262,26 @@ def _check_websocket(config: dict, out: _Collector) -> None: def _check_token_revocation(config: dict, out: _Collector, base_dir: Path | None) -> None: - value = config.get("token_revocation") - if value is None or value in ("", "redis", "memory"): - if value == "redis" and not (config.get("redis") or os.environ.get("REDIS_URL")): - out.warning("token_revocation", "redis needs the redis key or REDIS_URL.") - if value and not config.get("auth"): - out.warning("token_revocation", "Auth is off, so there are no tokens to revoke.") + if config.get("token_revocation") is None: return - if not isinstance(value, str): - out.error("token_revocation", "token_revocation must be redis, memory or module:attr.") + from tenxgraph_api.src.app.core.config.graph_config import TokenRevocationConfig + + # The old string values (redis, memory, module:attr) are errors naming the replacement. + parsed = _quietly( + lambda: TokenRevocationConfig.parse(config.get("token_revocation")), + out, + "token_revocation", + ) + if parsed is None: return - _check_import_path("token_revocation", value, out, base_dir) + if not config.get("auth"): + out.warning("token_revocation", "Auth is off, so there are no tokens to revoke.") + elif not (config.get("redis") or config.get("cache_store") or os.environ.get("REDIS_URL")): + out.warning( + "token_revocation", + "No shared cache store (redis, REDIS_URL or cache_store): a revocation reaches " + "only the process that recorded it, and `10xgraph revoke` cannot reach it.", + ) def _check_graph_introspection(config: dict, out: _Collector) -> None: @@ -372,7 +361,7 @@ def _check_observability(config: dict, out: _Collector) -> None: def _check_plugins(config: dict, out: _Collector, base_dir: Path | None) -> None: - for key in ("injectq", "store", "checkpointer", "thread_name_generator"): + for key in ("injectq", "store", "checkpointer", "thread_name_generator", "cache_store"): value = config.get(key) if value is None or value == "": continue @@ -389,8 +378,11 @@ def _check_plugins(config: dict, out: _Collector, base_dir: Path | None) -> None return if not isinstance(redis, str): out.error("redis", "redis must be a URL string.") - elif "$" in redis: - out.warning("redis", "Top-level redis is used as-is; $VAR is not expanded.") + elif _has_unresolved_env(redis): + # The server expands $VAR at startup; it is fine for it to be unset while editing. + out.warning( + "redis", f"{redis} is not set in this shell. Make sure it is set when the API runs." + ) def _check_remote_tools(config: dict, out: _Collector) -> None: @@ -459,8 +451,8 @@ def _check_import_path(path: str, value: Any, out: _Collector, base_dir: Path | def _quietly(parse: Callable[[], Any], out: _Collector, path: str) -> Any: """Run a server-side parser, turning its exceptions into an error issue. - The parsers log operational warnings (for example about the memory rate-limit - backend); those are reported as issues here, so the logger is muted meanwhile. + The parsers may log operational warnings; those are reported as issues here, so the + logger is muted meanwhile. """ api_logger = logging.getLogger("tenxgraph_api") previous = api_logger.disabled diff --git a/tenxgraph_api/cli/main.py b/tenxgraph_api/cli/main.py index e29c4912..0072a7f6 100644 --- a/tenxgraph_api/cli/main.py +++ b/tenxgraph_api/cli/main.py @@ -48,8 +48,7 @@ def execute(self, *args: Any, **kwargs: Any) -> int: f"The {feature} command could not load its required dependencies: {exc}", hints=( "Run `10xgraph audit` to inspect installed package compatibility.", - "Upgrade with `python -m pip install --upgrade " - '"10xgraph>=1.0.0rc3,<2"`.', + 'Upgrade with `python -m pip install --upgrade "10xgraph>=1.0.0rc3,<2"`.', ), ) from exc implementation = getattr(module, class_name) @@ -431,13 +430,15 @@ def revoke( None, "--expires-at", help="With --jti: the token's exp (epoch seconds), to drop the entry." ), redis_url: str | None = typer.Option( - None, "--redis-url", help="Redis URL (default: REDIS_URL, then `redis` in 10xgraph.json)." + None, + "--redis-url", + help="Redis URL to write to instead of the cache store the server uses.", ), config_file: str = typer.Option( DEFAULT_CONFIG_FILE, "--config", "-c", help="Project configuration file." ), ) -> None: - """Revoke a token or all of a user's tokens (needs "token_revocation": "redis").""" + """Revoke a token or all of a user's tokens (needs "token_revocation": true).""" try: sys.exit( RevokeCommand(output).execute( @@ -515,7 +516,10 @@ def init( # noqa: PLR0913 rate_limit: str | None = typer.Option( None, "--rate-limit", - help="Rate limiting: none, memory, or redis.", + help=( + "Rate limiting: none (off), memory (on, no shared cache store), or redis (on, " + "Redis cache store from REDIS_URL, plus token revocation with auth)." + ), ), provider: str | None = typer.Option( None, diff --git a/tenxgraph_api/cli/templates/prod/.env.example b/tenxgraph_api/cli/templates/prod/.env.example index d9c95005..1aacf135 100644 --- a/tenxgraph_api/cli/templates/prod/.env.example +++ b/tenxgraph_api/cli/templates/prod/.env.example @@ -66,7 +66,8 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/agent" ################################# ###### Redis Config ############# ################################# -# Hot thread cache, rate limiting and run locks. Local example value; replace it in production. +# The checkpointer's hot thread cache and the API's cache store (rate limiting, run locks and +# caps, token revocation). Local example value; replace it in production. REDIS_URL="redis://localhost:6379/0" ##[IF_JWT]## diff --git a/tenxgraph_api/cli/templates/prod/10xgraph.json b/tenxgraph_api/cli/templates/prod/10xgraph.json index 3b683722..9821dc5d 100644 --- a/tenxgraph_api/cli/templates/prod/10xgraph.json +++ b/tenxgraph_api/cli/templates/prod/10xgraph.json @@ -5,7 +5,8 @@ "method": "custom", "path": "auth.agent_auth:AgentAuth" }, - "token_revocation": "redis", + "redis": "${REDIS_URL}", + "token_revocation": true, "remote_tools": [], "ag_ui": { "enabled": false @@ -14,11 +15,6 @@ "injectq": "graph.agent:container", "rate_limit": { "enabled": true, - "backend": "redis", - "redis": { - "url": "${REDIS_URL}", - "prefix": "10xgraph:rate-limit" - }, "requests": 100, "window": 60, "by": "user", diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md index 664191b1..568ed4f4 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md @@ -23,10 +23,11 @@ Common full shape: "authorization": "graph.auth:my_authorization_backend", "env": ".env", "auth": "jwt", + "redis": "${REDIS_URL}", + "token_revocation": true, "remote_tools": [], "rate_limit": { "enabled": true, - "backend": "memory", "requests": 100, "window": 60, "by": "ip", @@ -46,14 +47,26 @@ Common full shape: - `env`: optional `.env` path loaded before graph import. - `auth`: `null`, `"jwt"`, or `{"method": "custom", "path": "module:backend"}`. - `remote_tools`: validated client-executed tool schemas, attached once at startup. +- `redis`: Redis URL (supports `${ENV_VAR}`) of the API's cache store, a `RedisCacheStore`. Unset falls back to `REDIS_URL`, then to an in-memory store per process. +- `cache_store`: optional `module:attr` of your own `BaseCacheStore` (an instance or zero-argument factory); wins over `redis`. The cache store holds everything the API shares between workers: run locks and caps, idempotency records, the ownership cache, token revocation and rate-limit counters. +- `token_revocation`: `true`, or `{"enabled": true, "token_lifetime": 3600, "user_ttl": 2592000}`. Revocations live in the cache store; fails closed (503) when it is unreachable. The old string values (`"redis"`, `"memory"`, `module:attr`) are config errors. - `rate_limit`: optional sliding-window rate limiter config object; omit or set to `null` to disable. See `references/rate-limiting.md` for the full field reference. +## Startup check and readiness + +Before serving, the server calls `aping()` on the checkpointer, the store and the cache store +(`STARTUP_CHECKS=false` skips it) and refuses to start while one stays unreachable for +`STARTUP_CHECK_TIMEOUT_SECONDS`. The cache store counts only when a fail-closed feature uses it +(`token_revocation`, or `rate_limit` with `fail_open: false`); otherwise an unreachable one is +one warning and the server starts with per-process locks and caps. `GET /health/ready` reports +`checkpointer`, `store` and `cache` with the same rule. + ## Loading Order 1. Read `10xgraph.json`. 2. Load `.env` when configured. 3. Import the compiled graph from `agent`. -4. Attach `remote_tools`, then import and bind `store`, `injectq`, `thread_name_generator`, and `authorization` when configured. +4. Attach `remote_tools`, then import and bind `store`, `injectq`, `thread_name_generator`, the cache store, and `authorization` when configured. 5. Configure auth. 6. Start FastAPI routes and services. diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md index 3c43af6e..c547b9ad 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md @@ -34,33 +34,45 @@ and takes three args: def authenticate(self, request, response, credential) -> dict | None: ... ``` -- `request` may be a `Request` or a `WebSocket` (both are `HTTPConnection`) — read +- `request` may be a `Request` or a `WebSocket` (both are `HTTPConnection`): read `request.headers`, do not assume Request-only APIs. - `credential` is the parsed bearer token (`HTTPAuthorizationCredentials | None`). For API-key/cookie schemes, ignore it and read headers directly. - Return a dict with at least `user_id` (keys flow into `config["user"]`); `{}`/`None` = anonymous; raise `UserAccountError`/`HTTPException` to reject (401 HTTP, 1008 close on WebSocket). - `user_id` must not start with `"__system__"` (`SYSTEM_USER_PREFIX` in core): `system_user_id(name)` = `"__system__:"` owns shared data (each agent's memory, RAG knowledge bases). The API refuses any identity whose id starts with that prefix, from any auth backend, with 401 "This user id is reserved" (WebSocket close 1008) and a warning in the log. Auth-off development still uses `"anonymous"`. -- Do NOT declare it `async def` — an un-awaited coroutine breaks auth silently. +- Do NOT declare it `async def`: an un-awaited coroutine breaks auth silently. - Include `roles` / `scopes` in the returned dict to feed authorization. ## Authorization Configured via the `authorization` key; separate from `auth`. Values: -- `null` (unset): mode-based default — `"ownership"` in production, `"allow_all"` in development. +- `null` (unset): mode-based default: `"ownership"` in production, `"allow_all"` in development. - `"ownership"`: owner-only. A thread is readable/runnable/deletable only by its creator; a foreign `invoke`/`stream` is rejected up front (403). Enforced on every thread-touching step. - `"allow_all"` (aliases `"default"`, `"none"`): any authenticated user may do anything. - `"module:attr"`: custom `AuthorizationBackend`. - `{ "backend": "rbac", ... }`: role-based access control (below). -Ownership is scalable: ownership is immutable, so it is cached (in-process LRU + optional shared -Redis when `redis`/`REDIS_URL` is set) via `ThreadOwnershipResolver` over +Ownership is scalable: ownership is immutable, so it is cached (in-process LRU + the shared +cache store when it is not in-memory: `cache_store`, `redis` or `REDIS_URL`) via +`ThreadOwnershipResolver` over `BaseCheckpointer.aget_thread_owner`. After the first lookup a check is an in-memory hit, not a per-request DB call; the cache is evicted on thread deletion. Ownership needs a persistent checkpointer (pg/sqlite/in-memory); with none, requests pass through. +### Token revocation + +`"token_revocation": true` (or `{"enabled": true, "token_lifetime": 3600, "user_ttl": 2592000}`) +rejects revoked tokens with 401 before they expire: by `jti` (`10xgraph revoke --jti`) or every +token of a user issued before a cutoff (`10xgraph revoke --user`). Revocations live in the API's +cache store, so share them between workers with `redis`, `REDIS_URL` or a custom +`cache_store`; with the in-memory store a revocation reaches one process and `10xgraph revoke` +refuses to run. It fails closed: while the cache store is unreachable, authenticated requests +get 503 and the server refuses to start. The old string values (`"redis"`, `"memory"`, +`module:attr`) are config errors naming `cache_store`. + ### Custom backend interface ```python @@ -115,7 +127,7 @@ API checks decide access; the storage layer (checkpointer, store) enforces isola trusted policy. After a successful check, `RequirePermission` stamps `user["authz"] = {user_id, scope, scopes}` where `scope` = backend `isolation_scope()`. Services copy the trusted `user` into `config["user"]`, so the core library reads `config["user"]["authz"]` -and scopes rows to the caller (`owner`) or not (`none`). Set server-side only — not client-forgeable. +and scopes rows to the caller (`owner`) or not (`none`). Set server-side only; not client-forgeable. Server-owned config keys (`RESERVED_CONFIG_KEYS` in `core/auth/request_config.py`: `authz`, `user`, `user_id`, `remote_tools`, `file_tool_root`, `workspace_root`, `tool_timeout`, `node_timeout`, `max_parallel_tool_calls`, `collection`, plus any `_`-prefixed key) are dropped from client config by `client_config` on every route. They are ignored, not refused (debug log). A client never picks a Qdrant `collection`; see memory-and-store.md. diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md index 99f17229..fddd99be 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md @@ -5,18 +5,15 @@ Use this when adding, configuring, or debugging 10xGraph's built-in sliding-wind ## Overview 10xGraph provides a sliding-window rate limiter configured via the `rate_limit` block in -`10xgraph.json`. The limiter is disabled by default — add the block to activate it. +`10xgraph.json`. The limiter is disabled by default; add the block to activate it. ## Quick Start -In-memory backend for local development or single-process services: - ```json { "agent": "graph.react:app", "rate_limit": { "enabled": true, - "backend": "memory", "requests": 100, "window": 60, "by": "ip", @@ -27,58 +24,50 @@ In-memory backend for local development or single-process services: Each client IP may make `requests` calls every `window` seconds. -## Redis Backend (Production) +## Where the counters live -Redis stores counters centrally so the limit is enforced across multiple workers, -containers, or servers. Install the optional extra first: +The counters live in the API's cache store (`BaseCacheStore`), the same one that holds the run +locks and caps, idempotency records, the ownership cache and token revocation. There is no +separate rate-limit backend. The store is chosen once at boot: -```bash -pip install "10xgraph-api[redis]" -``` +1. `"cache_store": "module:attr"` in `10xgraph.json` (your own `BaseCacheStore`). +2. Else `"redis"` in `10xgraph.json` (supports `${ENV_VAR}`), else `REDIS_URL`: + `RedisCacheStore` (also Valkey, Dragonfly, KeyDB). Needs `pip install "10xgraph-api[redis]"`. +3. Else `InMemoryCacheStore`: counters per process, fine for one worker. + +Shared counters across workers, containers or servers: ```json { "agent": "graph.react:app", + "redis": "${REDIS_URL}", "rate_limit": { "enabled": true, - "backend": "redis", "requests": 1000, "window": 60, "by": "ip", "trusted_proxy_headers": true, "exclude_paths": ["/ping", "/metrics", "/docs", "/redocs", "/openapi.json"], - "redis": { - "url": "${RATE_LIMIT_REDIS_URL}", - "prefix": "10xgraph:rate-limit" - }, "fail_open": true } } ``` -Set the environment variable: - -```bash -RATE_LIMIT_REDIS_URL=redis://localhost:6379/0 -``` - -The Redis backend uses an atomic Lua script with sorted sets — check and record happen as a -single Redis operation, which prevents concurrent requests from racing past the limit. +Check and record are one atomic store operation (a Lua script on Redis), so concurrent requests +cannot race past the limit. `rate_limit.backend` and `rate_limit.redis` were removed in +1.0.0rc4: either key is a config error naming `cache_store`. ## Configuration Reference | Field | Type | Default | Description | | --- | --- | --- | --- | | `enabled` | boolean | `true` | Enables the middleware when the `rate_limit` block exists. | -| `backend` | string | `"memory"` | `memory`, `redis`, or `custom`. | | `requests` | integer | `100` | Maximum requests allowed in each window. | | `window` | integer | `60` | Window size in seconds. | -| `by` | string | `"ip"` | Limit by client IP (`"ip"`) or one shared quota (`"global"`). | +| `by` | string | `"ip"` | Limit by client IP (`"ip"`), verified user (`"user"`) or one shared quota (`"global"`). | | `exclude_paths` | string array | `[]` | Paths that bypass rate limiting entirely. | | `trusted_proxy_headers` | boolean | `false` | Use `X-Forwarded-For` as the client IP (only behind a trusted proxy). | -| `redis.url` | string | `null` | Redis URL; required for the Redis backend. Supports `${ENV_VAR}` expansion. | -| `redis.prefix` | string | `"10xgraph:rate-limit"` | Prefix for all Redis keys. | -| `fail_open` | boolean | `true` | On Redis errors, allow (`true`) or deny (`false`, 503) requests. | +| `fail_open` | boolean | `true` | On cache store errors, allow (`true`) or deny (`false`, 503) requests. With `false` the server also refuses to start while the cache store is unreachable. | ## Identity Modes @@ -127,7 +116,7 @@ When the limit is exceeded, 10xGraph returns `429 Too Many Requests`: } ``` -When the backend cannot be reached and `fail_open` is `false`, the request is denied with +When the cache store cannot be reached and `fail_open` is `false`, the request is denied with `503 Service Unavailable` and `Retry-After: 5`, not 429, so clients can tell an outage from a limit hit: @@ -145,71 +134,30 @@ limit hit: } ``` -## Custom Backend - -Implement `BaseRateLimitBackend` and bind the instance through InjectQ, then set -`"backend": "custom"` in `10xgraph.json`. - -```python -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - BaseRateLimitBackend, - RateLimitDecision, -) - - -class MyRateLimitBackend(BaseRateLimitBackend): - async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: - allowed = True - remaining = limit - 1 - reset_after = window - return RateLimitDecision( - allowed=allowed, - remaining=remaining, - reset_after=reset_after, - ) - - async def close(self) -> None: - return None -``` - -When the backend's store cannot be reached and the request must not pass, return -`RateLimitDecision.backend_unavailable()`; the middleware answers it with the 503 above. - -```json -{ - "rate_limit": { - "enabled": true, - "backend": "custom", - "requests": 100, - "window": 60, - "by": "ip" - } -} -``` - -## Choosing a Backend +## Choosing a Cache Store -| Scenario | Backend | +| Scenario | Cache store | | --- | --- | -| Local development, tests, demos | `memory` | -| Single-process production | `memory` | -| Gunicorn/Uvicorn with multiple workers | `redis` | -| Docker / Kubernetes (multiple replicas) | `redis` | -| Custom storage or quotas | `custom` | +| Local development, tests, demos | in-memory (set nothing) | +| Single-process production | in-memory | +| Gunicorn/Uvicorn with multiple workers | Redis (`redis` or `REDIS_URL`) | +| Docker / Kubernetes (multiple replicas) | Redis (`redis` or `REDIS_URL`) | +| Storage that does not speak Redis | `cache_store` (your own `BaseCacheStore`) | ## Rules - Do not enable `trusted_proxy_headers` unless a reverse proxy strips untrusted forwarding headers. - Always add health-check and observability paths to `exclude_paths`. -- In production with Redis set `fail_open: false` only when hard enforcement is required; otherwise `true` prevents availability issues during Redis outages. -- The `redis` backend requires the `redis` extra: `pip install "10xgraph-api[redis]"`. -- The `RATE_LIMIT_REDIS_URL` value supports `${ENV_VAR}` expansion — keep secrets out of committed config. +- Set `fail_open: false` only when hard enforcement is required; otherwise `true` prevents availability issues during cache store outages. +- A Redis cache store requires the `redis` extra: `pip install "10xgraph-api[redis]"`. +- The `redis` value supports `${ENV_VAR}` expansion; keep secrets out of committed config. ## Source Map - Middleware: https://github.com/10xGraph/10xgraph-api/tree/main/tenxgraph_api/src/app/core/middleware/rate_limit -- Base class: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/core/middleware/rate_limit/base.py +- Limiter on the cache store: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/core/middleware/rate_limit/limiter.py +- Cache store: https://github.com/10xGraph/10xgraph-api/tree/main/tenxgraph_api/src/app/core/cache_store - Rate-limit config model: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/core/config/graph_config.py - Middleware setup: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/core/config/setup_middleware.py - Docs: https://github.com/10xGraph/10xgraph-api/blob/main/docs/rate-limiting.md -- Configuration reference: https://github.com/10xGraph/10xgraph-api/blob/main/docs/configuration.md — "Rate Limiting" section +- Configuration reference: https://github.com/10xGraph/10xgraph-api/blob/main/docs/configuration.md, "Rate Limiting" section diff --git a/tenxgraph_api/src/app/core/auth/authorization.py b/tenxgraph_api/src/app/core/auth/authorization.py index 739483d2..4c20c289 100644 --- a/tenxgraph_api/src/app/core/auth/authorization.py +++ b/tenxgraph_api/src/app/core/auth/authorization.py @@ -7,7 +7,11 @@ import logging from abc import ABC, abstractmethod -from typing import Any +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from tenxgraph_api.src.app.core.cache_store import BaseCacheStore logger = logging.getLogger("tenxgraph_api.authorization") @@ -235,8 +239,9 @@ class OwnershipAuthorizationBackend(AuthorizationBackend): Ownership is resolved through a :class:`ThreadOwnershipResolver` in front of :meth:`BaseCheckpointer.aget_thread_owner`. Ownership is immutable, so the resolver - caches it (in-process, plus optional shared Redis) -- after the first lookup a check is - an in-memory hit, not a database round-trip per request. + caches it (in-process, plus the shared cache store when one is configured) -- after the + first lookup a check is an in-memory hit, not a database round-trip per request. Pass the + API's ``BaseCacheStore`` as ``cache``; the backend uses it but never closes it. This backend fails closed: with no checkpointer, or one that cannot resolve ownership (``aget_thread_owner`` raises ``NotImplementedError``), every thread-scoped request is @@ -269,12 +274,12 @@ def __init__( checkpointer: Any | None = None, *, resolver: Any | None = None, - redis: object | None = None, + cache: "BaseCacheStore | None" = None, server_thread_ids: bool = False, ) -> None: self._checkpointer = checkpointer self._resolver = resolver - self._redis = redis + self._cache = cache self._resolver_built = resolver is not None # ``runs.thread_ids == "server"``: a request may only name a thread that exists. self.server_thread_ids = server_thread_ids @@ -308,7 +313,7 @@ def _get_resolver(self) -> Any | None: ThreadOwnershipResolver, ) - self._resolver = ThreadOwnershipResolver(cp.aget_thread_owner, redis=self._redis) + self._resolver = ThreadOwnershipResolver(cp.aget_thread_owner, cache=self._cache) return self._resolver def evict(self, thread_id: str | int, owner: str | int | None = None) -> Any: @@ -323,15 +328,6 @@ def evict(self, thread_id: str | int, owner: str | int | None = None) -> Any: return None return resolver.evict(thread_id, owner) - async def aclose(self) -> None: - """Close the L2 Redis client this backend created (called on app shutdown).""" - client = self._redis - if client is not None and hasattr(client, "aclose"): - try: - await client.aclose() - except Exception as exc: # pragma: no cover - best-effort shutdown - logger.debug("Ownership Redis client close failed: %s", exc) - async def authorize( self, user: dict[str, Any], @@ -513,13 +509,13 @@ def __init__( isolation: str = "owner", checkpointer: Any | None = None, resolver: Any | None = None, - redis: object | None = None, + cache: "BaseCacheStore | None" = None, server_thread_ids: bool = False, ) -> None: super().__init__( checkpointer=checkpointer, resolver=resolver, - redis=redis, + cache=cache, server_thread_ids=server_thread_ids, ) self._role_scopes = {str(r): list(s) for r, s in (role_scopes or {}).items()} diff --git a/tenxgraph_api/src/app/core/auth/ownership_resolver.py b/tenxgraph_api/src/app/core/auth/ownership_resolver.py index 9e94e5cd..0e6a6821 100644 --- a/tenxgraph_api/src/app/core/auth/ownership_resolver.py +++ b/tenxgraph_api/src/app/core/auth/ownership_resolver.py @@ -8,16 +8,19 @@ Cache tiers: - **L1**: a bounded in-process LRU (per worker) with a short TTL. -- **L2** (optional): a shared Redis, so workers/instances do not each re-hit the database. +- **L2** (optional): the API's shared cache store (``BaseCacheStore``, Redis by default), so + workers/instances do not each re-hit the database. With an in-memory cache store there is + no L2: it would only cache the same thing twice in this process. Deleting a thread evicts it from this worker's L1 and from L2, but another worker's L1 cannot be reached. A deleted id could be created again by a different user, and a stale L1 entry on another worker would then grant the old owner access to the new owner's thread. So a delete -also leaves a **tombstone** for one L1 TTL (in L1, and in L2 when Redis is configured), and -:meth:`ThreadOwnershipResolver.recently_deleted` lets the authorization backend refuse to -create a thread under that id until every worker's stale entry has expired. Without Redis the -tombstone only exists on the worker that handled the delete; run multi-worker deployments with -the shared L2 tier. The L2 TTL covers an eviction that failed to reach Redis. +also leaves a **tombstone** for one L1 TTL (in L1, and in L2 when a shared cache store is +configured), and :meth:`ThreadOwnershipResolver.recently_deleted` lets the authorization +backend refuse to create a thread under that id until every worker's stale entry has expired. +Without a shared cache store the tombstone only exists on the worker that handled the delete; +run multi-worker deployments with the shared L2 tier. The L2 TTL covers an eviction that +failed to reach the cache store. Only *positive* results (a known owner) are cached. A ``None`` result (the thread does not exist yet) is **never** cached: a nonexistent thread can become owned by the very next @@ -33,7 +36,9 @@ from collections import OrderedDict from collections.abc import Awaitable, Callable -from tenxgraph_api.src.app.core.redis_client import key_part, prefixed +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore +from tenxgraph_api.src.app.core.cache_store.loader import is_in_memory +from tenxgraph_api.src.app.core.redis_client import key_part logger = logging.getLogger("tenxgraph_api.authorization") @@ -54,30 +59,28 @@ class ThreadOwnershipResolver: lookup: async ``(thread_id) -> user_id | None`` backing lookup (typically ``checkpointer.aget_thread_owner``). May raise ``NotImplementedError`` if the checkpointer cannot resolve ownership; that propagates uncached. - redis: optional async Redis-like client (``get``/``set``/``delete`` coroutines). - When absent the resolver is L1-only. Redis errors degrade to the DB lookup and - never fail a request. + cache: the API's cache store, the shared L2 tier (keys ``owner:id:`` and + ``owner:tomb:``). None or an ``InMemoryCacheStore`` means L1 only. Cache + errors degrade to the DB lookup and never fail a request. The resolver never + closes it. max_size: L1 LRU capacity (entries). - redis_prefix: key namespace for L2 entries (default ``af:authz:owner`` behind - ``REDIS_KEY_PREFIX``). l1_ttl: seconds an L1 entry is trusted; bounds staleness after a delete elsewhere. - l2_ttl: seconds an L2 entry lives in Redis. + l2_ttl: seconds an L2 entry lives in the cache store. """ def __init__( self, lookup: OwnerLookup, *, - redis: object | None = None, + cache: BaseCacheStore | None = None, max_size: int = 10_000, - redis_prefix: str | None = None, l1_ttl: float = DEFAULT_L1_TTL_SECONDS, l2_ttl: int = DEFAULT_L2_TTL_SECONDS, ) -> None: self._lookup = lookup - self._redis = redis + # An in-memory store lives in this process, like L1: caching in it too adds nothing. + self._cache = None if is_in_memory(cache) else cache self._max_size = max(1, max_size) - self._prefix = redis_prefix if redis_prefix is not None else prefixed("af:authz:owner") self._l1_ttl = l1_ttl self._l2_ttl = l2_ttl # key -> (owner, monotonic expiry) @@ -100,10 +103,10 @@ async def owner_of(self, thread_id: str | int) -> str | None: if cached is not None: return cached - from_redis = await self._redis_get(key) - if from_redis is not None: - await self._l1_put(key, from_redis) - return from_redis + from_l2 = await self._l2_get(key) + if from_l2 is not None: + await self._l1_put(key, from_l2) + return from_l2 owner = await self._lookup(key) # may raise NotImplementedError -> propagate if owner is None: @@ -111,7 +114,7 @@ async def owner_of(self, thread_id: str | int) -> str | None: owner_str = str(owner) await self._l1_put(key, owner_str) - await self._redis_set(key, owner_str) + await self._l2_set(key, owner_str) return owner_str async def evict(self, thread_id: str | int, owner: str | int | None = None) -> None: @@ -130,8 +133,8 @@ async def evict(self, thread_id: str | int, owner: str | int | None = None) -> N self._tombstones.move_to_end(key) while len(self._tombstones) > self._max_size: self._tombstones.popitem(last=False) - await self._redis_delete(key) - await self._redis_tombstone(key, owner_str) + await self._l2_delete(key) + await self._l2_tombstone(key, owner_str) async def recently_deleted(self, thread_id: str | int, user_id: str | None = None) -> bool: """Whether ``thread_id`` was deleted within the last L1 TTL by someone other than @@ -150,7 +153,7 @@ async def recently_deleted(self, thread_id: str | int, user_id: str | None = Non if time.monotonic() < expires_at: return not (owner and user_id is not None and owner == str(user_id)) del self._tombstones[key] - shared_owner = await self._redis_tombstone_owner(key) + shared_owner = await self._l2_tombstone_owner(key) if shared_owner is None: return False return not (shared_owner and user_id is not None and shared_owner == str(user_id)) @@ -176,69 +179,73 @@ async def _l1_put(self, key: str, owner: str) -> None: while len(self._l1) > self._max_size: self._l1.popitem(last=False) # evict least-recently-used - # ------------------------------------------------------------------ L2 (Redis) + # ------------------------------------------------------------------ L2 (cache store) # Owner entries and delete markers live under different fixed segments, so no thread id # (not even one like "deleted:X") can make an owner key collide with a marker key. - def _redis_key(self, key: str) -> str: - return f"{self._prefix}:id:{key_part(key)}" + @staticmethod + def _owner_key(key: str) -> str: + return f"owner:id:{key_part(key)}" - async def _redis_get(self, key: str) -> str | None: - if self._redis is None: + @staticmethod + def _tombstone_key(key: str) -> str: + return f"owner:tomb:{key_part(key)}" + + async def _l2_get(self, key: str) -> str | None: + if self._cache is None: return None try: - value = await self._redis.get(self._redis_key(key)) # type: ignore[attr-defined] + value = await self._cache.get(self._owner_key(key)) except Exception as exc: # degrade to DB; never fail the request logger.debug("Ownership L2 get failed for %s: %s", key, exc) return None if value is None: return None - return value.decode() if isinstance(value, bytes | bytearray) else str(value) + return _text(value) - async def _redis_set(self, key: str, owner: str) -> None: - if self._redis is None: + async def _l2_set(self, key: str, owner: str) -> None: + if self._cache is None: return try: # evict() removes the entry on delete; the TTL covers an eviction that failed. - await self._redis.set( # type: ignore[attr-defined] - self._redis_key(key), owner, ex=self._l2_ttl - ) + await self._cache.set(self._owner_key(key), owner.encode(), ttl=self._l2_ttl) except Exception as exc: logger.debug("Ownership L2 set failed for %s: %s", key, exc) - def _tombstone_key(self, key: str) -> str: - return f"{self._prefix}:tomb:{key_part(key)}" - - async def _redis_tombstone(self, key: str, owner: str) -> None: - if self._redis is None: + async def _l2_tombstone(self, key: str, owner: str) -> None: + if self._cache is None: return try: - await self._redis.set( # type: ignore[attr-defined] + await self._cache.set( self._tombstone_key(key), owner.encode() or b"-", - ex=max(1, int(self._l1_ttl + 0.999)), + ttl=max(1, int(self._l1_ttl + 0.999)), ) except Exception as exc: logger.debug("Ownership L2 tombstone failed for %s: %s", key, exc) - async def _redis_tombstone_owner(self, key: str) -> str | None: + async def _l2_tombstone_owner(self, key: str) -> str | None: """The deleting owner in a shared tombstone ("" when unknown), or None when none.""" - if self._redis is None: + if self._cache is None: return None try: - value = await self._redis.get(self._tombstone_key(key)) # type: ignore[attr-defined] + value = await self._cache.get(self._tombstone_key(key)) except Exception as exc: # degrade to "not deleted"; L1 already answered for us logger.debug("Ownership L2 tombstone read failed for %s: %s", key, exc) return None if value is None: return None - text = value.decode() if isinstance(value, bytes | bytearray) else str(value) + text = _text(value) return "" if text == "-" else text - async def _redis_delete(self, key: str) -> None: - if self._redis is None: + async def _l2_delete(self, key: str) -> None: + if self._cache is None: return try: - await self._redis.delete(self._redis_key(key)) # type: ignore[attr-defined] + await self._cache.delete(self._owner_key(key)) except Exception as exc: logger.debug("Ownership L2 delete failed for %s: %s", key, exc) + + +def _text(value: object) -> str: + return value.decode() if isinstance(value, bytes | bytearray) else str(value) diff --git a/tenxgraph_api/src/app/core/auth/permissions.py b/tenxgraph_api/src/app/core/auth/permissions.py index 0fca0ce5..1696f926 100644 --- a/tenxgraph_api/src/app/core/auth/permissions.py +++ b/tenxgraph_api/src/app/core/auth/permissions.py @@ -177,10 +177,10 @@ def _authenticate( async def _reject_if_revoked(connection: HTTPConnection, user: dict[str, Any]) -> None: - """Refuse a credential the token revocation backend reports as revoked (401). + """Refuse a credential that was revoked (401), when ``token_revocation`` is on. - When the backend cannot answer, refuse with 503 instead of serving a token that might have - been revoked. + When the cache store cannot answer, refuse with 503 instead of serving a token that might + have been revoked. """ try: revoked = await token_revoked(user) diff --git a/tenxgraph_api/src/app/core/auth/revocation.py b/tenxgraph_api/src/app/core/auth/revocation.py index 548323b5..17fdbf96 100644 --- a/tenxgraph_api/src/app/core/auth/revocation.py +++ b/tenxgraph_api/src/app/core/auth/revocation.py @@ -1,62 +1,63 @@ """Token revocation: reject a credential before it expires. -JWTs are self-contained, so a stolen or logged-out token stays valid until its ``exp``. A -:class:`TokenRevocationBackend` closes that gap: ``RequirePermission`` asks it about every -authenticated request, and WebSocket runs ask again before each run, so a socket opened with -a token that is later revoked stops working too. +JWTs are self-contained, so a stolen or logged-out token stays valid until its ``exp``. +Revocation closes that gap: ``RequirePermission`` checks every authenticated request, and +WebSocket runs check again before each run, so a socket opened with a token that is later +revoked stops working too. -Two kinds of revocation are supported by the built-in backends: +Two kinds of revocation are supported: -- **one token**, by its ``jti`` claim (``revoke_token``), kept until the token would have +- **one token**, by its ``jti`` claim (:func:`revoke_token`), kept until the token would have expired anyway; -- **every token of a user issued before a moment** (``revoke_user``), compared against the - token's ``iat`` claim. Use it for "log out everywhere" or after a password change. A token - without ``iat`` cannot be dated, so it counts as revoked while the user entry exists; give the - backend your ``token_lifetime`` and such a token is dated from its ``exp`` instead, so one - minted after the revoke is accepted. Tokens should carry ``iat``. - -To revoke from your own code use :func:`revoke_token` and :func:`revoke_user`; from a shell -use ``10xgraph revoke``. A newer ``revoke_user`` cutoff is never replaced by an older one. - -Enable it in ``10xgraph.json`` with ``"token_revocation": "redis"`` (shared by every worker; -uses the ``redis`` key or ``REDIS_URL``), ``"token_revocation": "memory"`` (one process; tests -and single-worker setups), or ``"token_revocation": "module:attr"`` for your own backend. A -``TokenRevocationBackend`` bound in your InjectQ container works too. - -A backend error fails closed: the request is refused with 503 rather than served with a token -that might have been revoked. +- **every token of a user issued before a moment** (:func:`revoke_user`), compared against + the token's ``iat`` claim. Use it for "log out everywhere" or after a password change. A + token without ``iat`` cannot be dated, so it counts as revoked while the user entry exists; + set ``token_lifetime`` and such a token is dated from its ``exp`` instead, so one minted + after the revoke is accepted. Tokens should carry ``iat``. + +Revocations are kept in the API's cache store (``BaseCacheStore``): keys +``revoked:jti:`` (expires with the token) and ``revoked:user:`` (holds the +cutoff timestamp). A newer ``revoke_user`` cutoff is never replaced by an older one +(``set(mode="if_greater")``). Times come from the store's clock, so pods with skewed clocks +agree. From your own code call :func:`revoke_token` / :func:`revoke_user` with the app's +cache store; from a shell use ``10xgraph revoke``. + +Enable it in ``10xgraph.json`` with ``"token_revocation": true`` (or an object with +``token_lifetime`` / ``user_ttl``, see ``TokenRevocationConfig``). With a shared cache store +(Redis) a revocation reaches every worker at once; with the in-memory store only this process. + +A cache store error fails closed: the request is refused with 503 rather than served with a +token that might have been revoked. """ from __future__ import annotations import asyncio -import importlib -import inspect import logging -import time -from abc import ABC, abstractmethod -from typing import Any +from typing import TYPE_CHECKING, Any -from tenxgraph_api.src.app.core.redis_client import key_part, prefixed +from tenxgraph_api.src.app.core.config.graph_config import ( + DEFAULT_USER_REVOCATION_TTL, + TokenRevocationConfig, +) +from tenxgraph_api.src.app.core.redis_client import key_part -logger = logging.getLogger("tenxgraph_api.revocation") - +if TYPE_CHECKING: + from tenxgraph_api.src.app.core.cache_store import BaseCacheStore -class TokenRevocationBackend(ABC): - """Decides whether an authenticated identity's credential has been revoked. - ``identity`` is what the auth backend returned: for ``"auth": "jwt"`` the decoded claims - (``user_id``, ``jti``, ``iat``, ``exp``, ...). Custom auth backends should include the - claims their revocation backend needs. - """ - - @abstractmethod - async def is_revoked(self, identity: dict[str, Any]) -> bool: - """Return True when this credential must no longer be accepted.""" +logger = logging.getLogger("tenxgraph_api.revocation") - async def aclose(self) -> None: # noqa: B027 - optional hook, no-op by default - """Release resources (called on app shutdown).""" +__all__ = [ + "DEFAULT_USER_REVOCATION_TTL", + "TokenRevocationConfig", + "TokenRevocationUnavailableError", + "is_revoked", + "revoke_token", + "revoke_user", + "token_revoked", +] def _number(value: Any) -> float | None: @@ -77,11 +78,6 @@ def _user_key_of(identity: dict[str, Any]) -> str | None: return None -# A revoke_user entry is kept this long unless told otherwise. It has to outlive the longest -# token you issue, or an old token comes back to life when the entry goes. -DEFAULT_USER_REVOCATION_TTL = 30 * 24 * 3600 - - def _revoked_by_user_cutoff( identity: dict[str, Any], cutoff: float | None, token_lifetime: float | None = None ) -> bool: @@ -95,239 +91,116 @@ def _revoked_by_user_cutoff( return issued_at is None or issued_at <= cutoff -async def _redis_now(client: Any) -> float: - """The Redis server's clock, so every pod agrees; this process's clock without ``TIME``.""" - server_time = getattr(client, "time", None) - if not callable(server_time): - return time.time() - seconds, microseconds = await server_time() - return float(seconds) + float(microseconds) / 1_000_000 - - -# Keeps the newest cutoff: an older ``revoke_user(before=...)`` must not un-revoke tokens a -# newer call already revoked. An empty cutoff means "now" on the Redis clock. -_REVOKE_USER_LUA = """ -local cutoff = tonumber(ARGV[1]) -if cutoff == nil then - local t = redis.call('TIME') - cutoff = tonumber(t[1]) + tonumber(t[2]) / 1000000 -end -local current = tonumber(redis.call('GET', KEYS[1]) or '') -if current ~= nil and current >= cutoff then - return 0 -end -redis.call('SET', KEYS[1], string.format('%.6f', cutoff)) -local ttl = tonumber(ARGV[2]) -if ttl ~= nil and ttl > 0 then - redis.call('EXPIRE', KEYS[1], ttl) -end -return 1 -""" - - -class InMemoryTokenRevocation(TokenRevocationBackend): - """Revocation list held in this process. Fine for tests and single-worker servers.""" +def jti_key(jti: Any) -> str: + """Cache store key of a revoked token.""" + return f"revoked:jti:{key_part(jti)}" - def __init__(self, token_lifetime: float | None = None) -> None: - self._token_lifetime = token_lifetime - self._tokens: dict[str, float] = {} # jti -> expiry (epoch seconds) - self._users: dict[str, float] = {} # user_id -> revoke tokens issued at or before - def revoke_token(self, jti: str, expires_at: float | None = None) -> None: - """Revoke one token. ``expires_at`` (its ``exp``) lets the entry be dropped later.""" - self._tokens[str(jti)] = float(expires_at) if expires_at is not None else float("inf") +def user_key(user_id: Any) -> str: + """Cache store key of a user's revocation cutoff.""" + return f"revoked:user:{key_part(user_id)}" - def revoke_user(self, user_id: str, before: float | None = None) -> None: - """Revoke every token of ``user_id`` issued at or before ``before`` (default: now).""" - cutoff = time.time() if before is None else float(before) - key = str(user_id) - self._users[key] = max(cutoff, self._users.get(key, cutoff)) - async def is_revoked(self, identity: dict[str, Any]) -> bool: - now = time.time() - self._tokens = {jti: exp for jti, exp in self._tokens.items() if exp > now} - jti = identity.get("jti") - if jti is not None and str(jti) in self._tokens: - return True - user_id = _user_key_of(identity) - cutoff = self._users.get(user_id) if user_id is not None else None - return _revoked_by_user_cutoff(identity, cutoff, self._token_lifetime) +async def revoke_token(cache: BaseCacheStore, jti: str, expires_at: float | None = None) -> None: + """Revoke one token by its ``jti`` claim. + ``expires_at`` is its ``exp`` (epoch seconds); the entry is dropped then. Without it the + entry is kept until deleted. + """ + ttl = None + if expires_at is not None: + ttl = max(1, int(float(expires_at) - await cache.now())) + await cache.set(jti_key(jti), b"1", ttl) -class RedisTokenRevocation(TokenRevocationBackend): - """Revocation list in Redis, shared by every worker and instance. - Keys: ``:jti:`` (expires with the token) and ``:user:`` - (holds the cutoff timestamp). A check reads both keys with two parallel ``GET`` calls (a - single ``MGET`` over two keys fails with CROSSSLOT on Redis Cluster). Times come from the - Redis clock, so pods with skewed clocks agree. +async def revoke_user( + cache: BaseCacheStore, + user_id: str, + before: float | None = None, + ttl: float | None = DEFAULT_USER_REVOCATION_TTL, +) -> None: + """Revoke every token of ``user_id`` issued at or before ``before`` (default: now). - ``token_lifetime`` is the longest lifetime of the tokens you issue, in seconds. It dates - a token that has no ``iat`` from its ``exp``. ``user_ttl`` is how long a ``revoke_user`` - entry is kept when the call gives no ``ttl`` (``None`` keeps it forever). + ``now`` is the cache store's clock. ``ttl`` (seconds) expires the entry once every older + token has expired; None keeps it until deleted. An older ``before`` than the one already + stored changes nothing (nor its TTL). """ - - def __init__( - self, - url: str | None = None, - *, - client: Any | None = None, - prefix: str | None = None, - token_lifetime: float | None = None, - user_ttl: int | None = DEFAULT_USER_REVOCATION_TTL, - ) -> None: - self._token_lifetime = token_lifetime - self._user_ttl = user_ttl - if client is None: - if not url: - raise ValueError("RedisTokenRevocation needs a Redis url or client") - from tenxgraph_api.src.app.core.redis_client import create_redis - - try: - client = create_redis(url) - except ImportError as exc: # pragma: no cover - depends on installed extras - raise ImportError( - 'token_revocation "redis" needs the redis package: ' - 'pip install "10xgraph-api[redis]"' - ) from exc - self._client = client - self._prefix = prefix if prefix is not None else prefixed("10xgraph:revoked") - - def _jti_key(self, jti: Any) -> str: - return f"{self._prefix}:jti:{key_part(jti)}" - - def _user_key(self, user_id: Any) -> str: - return f"{self._prefix}:user:{key_part(user_id)}" - - async def revoke_token(self, jti: str, expires_at: float | None = None) -> None: - ttl = None - if expires_at is not None: - ttl = max(1, int(expires_at - await _redis_now(self._client))) - await self._client.set(self._jti_key(jti), b"1", ex=ttl) - - async def revoke_user( - self, user_id: str, before: float | None = None, ttl: int | None = None - ) -> None: - """Revoke tokens of ``user_id`` issued at or before ``before`` (default: now). - - ``ttl`` (seconds) expires the entry once every older token has expired; it defaults to - ``user_ttl``. An older ``before`` than the one already stored changes nothing. - """ - cutoff = "" if before is None else repr(float(before)) - keep_for = self._user_ttl if ttl is None else ttl - await self._client.eval( - _REVOKE_USER_LUA, 1, self._user_key(user_id), cutoff, str(keep_for or 0) - ) - - async def is_revoked(self, identity: dict[str, Any]) -> bool: - jti, user_id = identity.get("jti"), _user_key_of(identity) - if jti is None and user_id is None: - return False - # Two plain GETs, not one MGET: the keys hash to different slots on Redis Cluster. - jti_value, raw = await asyncio.gather( - self._client.get(self._jti_key(jti)) if jti is not None else _none(), - self._client.get(self._user_key(user_id)) if user_id is not None else _none(), - ) - if jti_value is not None: - return True - if raw is None: - return False - cutoff = float(raw.decode() if isinstance(raw, bytes | bytearray) else raw) - return _revoked_by_user_cutoff(identity, cutoff, self._token_lifetime) - - async def aclose(self) -> None: - close = getattr(self._client, "aclose", None) - if callable(close): - await close() + cutoff = await cache.now() if before is None else float(before) + await cache.set(user_key(user_id), b"%.6f" % cutoff, ttl, mode="if_greater") async def _none() -> None: return None -def build_token_revocation(spec: Any, redis_url: str | None = None) -> TokenRevocationBackend: - """Build the backend named by the ``token_revocation`` key of 10xgraph.json.""" - if spec == "memory": - return InMemoryTokenRevocation() - if spec == "redis": - if not redis_url: - raise ValueError( - 'token_revocation "redis" needs a Redis URL: set "redis" in 10xgraph.json ' - "or REDIS_URL" - ) - return RedisTokenRevocation(redis_url) - if isinstance(spec, str) and ":" in spec: - module_name, attr = spec.split(":", 1) - obj = getattr(importlib.import_module(module_name), attr) - if inspect.isclass(obj) and issubclass(obj, TokenRevocationBackend): - obj = obj() - if not isinstance(obj, TokenRevocationBackend): - raise TypeError(f"{spec} is not a TokenRevocationBackend") - return obj - raise ValueError( - f"Unsupported token_revocation {spec!r}. Use 'redis', 'memory' or 'module:attr'." +async def is_revoked( + cache: BaseCacheStore, identity: dict[str, Any], token_lifetime: float | None = None +) -> bool: + """Whether ``identity``'s token was revoked (by ``jti`` or by a user cutoff). + + Raises whatever the cache store raises; :func:`token_revoked` turns that into + :class:`TokenRevocationUnavailableError`. + """ + jti, user_id = identity.get("jti"), _user_key_of(identity) + if jti is None and user_id is None: + return False + # Two plain GETs, not one MGET: the keys hash to different slots on Redis Cluster. + jti_value, raw = await asyncio.gather( + cache.get(jti_key(jti)) if jti is not None else _none(), + cache.get(user_key(user_id)) if user_id is not None else _none(), ) + if jti_value is not None: + return True + if raw is None: + return False + cutoff = float(raw.decode() if isinstance(raw, bytes | bytearray) else raw) + return _revoked_by_user_cutoff(identity, cutoff, token_lifetime) -def _bound_backend() -> TokenRevocationBackend | None: - """The backend bound in the app's container (the request's, inside a request).""" - from tenxgraph_api.src.app.core.utils.container import current_container +class TokenRevocationUnavailableError(RuntimeError): + """The cache store could not answer a revocation check; the request must not be served.""" - return current_container().try_get(TokenRevocationBackend) +def _bound() -> tuple[TokenRevocationConfig | None, BaseCacheStore | None]: + """The revocation settings and cache store bound in the app's container.""" + from tenxgraph_api.src.app.core.cache_store import BaseCacheStore + from tenxgraph_api.src.app.core.utils.container import current_container -class TokenRevocationUnavailableError(RuntimeError): - """The revocation backend could not answer; the request must not be served.""" + container = current_container() + # ``in`` first: try_get builds an unbound class whose arguments all have defaults, which + # would switch revocation on for every app. + config = ( + container.try_get(TokenRevocationConfig) if TokenRevocationConfig in container else None + ) + return config, container.try_get(BaseCacheStore) async def token_revoked( - identity: dict[str, Any], backend: TokenRevocationBackend | None = None + identity: dict[str, Any], + cache: BaseCacheStore | None = None, + config: TokenRevocationConfig | None = None, ) -> bool: """Whether ``identity``'s credential has been revoked. - False when no backend is configured or the identity is not a mapping. The ``jti`` is - checked even when ``user_id`` is missing or odd; per-user revocation applies when - ``user_id`` is a valid string. Raises :class:`TokenRevocationUnavailableError` when the - backend fails (fail closed). + ``cache`` and ``config`` default to the ones bound in the app's container. False when + token revocation is off or the identity is not a mapping. The ``jti`` is checked even when + ``user_id`` is missing or odd; per-user revocation applies when ``user_id`` is a valid + string. Raises :class:`TokenRevocationUnavailableError` when the cache store fails or none + is bound (fail closed). """ if not isinstance(identity, dict): return False - backend = backend or _bound_backend() - if backend is None: + if cache is None or config is None: + bound_config, bound_cache = _bound() + config = config or bound_config + cache = cache or bound_cache + if config is None: return False + if cache is None: + logger.error("Token revocation is on but no cache store is bound") + raise TokenRevocationUnavailableError("Token revocation check unavailable") try: - return bool(await backend.is_revoked(identity)) + return await is_revoked(cache, identity, config.token_lifetime) except Exception as exc: logger.error("Token revocation check failed: %s", exc) raise TokenRevocationUnavailableError("Token revocation check unavailable") from exc - - -async def _call(method: Any, *args: Any, **kwargs: Any) -> None: - """Call a backend method that may be sync (in-memory) or async (Redis).""" - result = method(*args, **kwargs) - if inspect.isawaitable(result): - await result - - -def _backend_or_raise(backend: TokenRevocationBackend | None) -> Any: - backend = backend or _bound_backend() - if backend is None: - raise RuntimeError( - 'No token revocation backend is configured. Set "token_revocation": "redis" in ' - "10xgraph.json (with Redis configured), or bind a TokenRevocationBackend." - ) - return backend - - -async def revoke_token( - jti: str, expires_at: float | None = None, *, backend: TokenRevocationBackend | None = None -) -> None: - """Revoke one token by its ``jti`` claim. ``expires_at`` is its ``exp`` (epoch seconds).""" - await _call(_backend_or_raise(backend).revoke_token, jti, expires_at) - - -async def revoke_user( - user_id: str, before: float | None = None, *, backend: TokenRevocationBackend | None = None -) -> None: - """Revoke every token of ``user_id`` issued at or before ``before`` (default: now).""" - await _call(_backend_or_raise(backend).revoke_user, user_id, before) diff --git a/tenxgraph_api/src/app/core/cache_store/__init__.py b/tenxgraph_api/src/app/core/cache_store/__init__.py new file mode 100644 index 00000000..a05d7fab --- /dev/null +++ b/tenxgraph_api/src/app/core/cache_store/__init__.py @@ -0,0 +1,13 @@ +"""Shared state for the API's workers: one ``BaseCacheStore`` (see ``base.py``).""" + +from .base import BaseCacheStore, SlotResult +from .memory_store import InMemoryCacheStore +from .redis_store import RedisCacheStore + + +__all__ = [ + "BaseCacheStore", + "InMemoryCacheStore", + "RedisCacheStore", + "SlotResult", +] diff --git a/tenxgraph_api/src/app/core/cache_store/base.py b/tenxgraph_api/src/app/core/cache_store/base.py new file mode 100644 index 00000000..04ab6753 --- /dev/null +++ b/tenxgraph_api/src/app/core/cache_store/base.py @@ -0,0 +1,122 @@ +"""The one extension point for state the API shares between its workers. + +Run locks, run and WebSocket caps, idempotency records, the ownership cache, token revocation +and rate limiting all keep their shared state in a :class:`BaseCacheStore`. Two primitives +cover every one of them: + +- **Timed slots** (``acquire`` / ``renew`` / ``release``): a set of expiring slots per key, + capped at ``limit``. A lock is a slot set with ``limit=1``; a rate limit is a slot set whose + slots are never released and expire after the window. +- **Keys with a TTL** (``get`` / ``set`` / ``delete``), with ``if_absent`` and ``if_greater`` + writes. + +``RedisCacheStore`` (also Valkey, Dragonfly and KeyDB) shares the state between workers; +``InMemoryCacheStore`` keeps it in one process. Implement this class for a store that does not +speak the Redis protocol. + +Keys: callers pass ``":"`` (for example ``"concurrency:runs:thread:42"``). The +store may map the feature segment to its own namespace and add a deployment prefix, but two +different keys must never land on the same entry, and any string (``:``, spaces, unicode) is +a valid key. A key holds either slots or a value, never both. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Literal + + +SetMode = Literal["always", "if_absent", "if_greater"] + + +@dataclass(frozen=True) +class SlotResult: + """Outcome of :meth:`BaseCacheStore.acquire`.""" + + granted: bool + # Live slots after this call, including ours when granted. + count: int + # Expiry time of the oldest live slot, on the store's clock; None when no slot is live. + oldest_expiry: float | None + # The new slot's id when granted (pass it to renew / release); None when denied. + slot_id: str | None + + +class BaseCacheStore(ABC): + """Shared state for the API: timed slots and keys with a TTL. + + The contract every implementation keeps (``tenxgraph_api.testing.cache_store_contract`` + checks it): + + - ``acquire`` is atomic: drop expired slots, and only if the live count is below + ``limit``, add a new slot expiring at ``now() + ttl``. A denied call adds nothing. Two + callers can never both take the last slot. ``limit < 1`` always denies. + - ``renew`` sets a held slot's expiry to ``now() + ttl``; returns False when the slot no + longer exists (expired or released). It never re-creates a slot. + - ``release`` removes a slot; releasing a slot that is gone is not an error. + - ``set(mode="always")`` writes and returns True. ``set(mode="if_absent")`` writes only + when the key does not exist; returns whether it wrote. + - ``set(mode="if_greater")`` treats the value as a decimal number and writes only when it + is greater than the stored one (or nothing is stored); atomic. Used by token revocation + so an older cutoff never overwrites a newer one. A value that is not a decimal number + raises ``ValueError``. + - ``ttl`` is in seconds and must be positive; ``set(ttl=None)`` keeps the key until it is + deleted. A write replaces the TTL of the key it writes. + - TTLs are honoured: a key or slot must not be dropped before its TTL. A store that evicts + data under memory pressure breaks locks (two workers run the same thread). Redis with a + ``maxmemory-policy`` other than ``noeviction`` or ``volatile-*`` is a deployment mistake. + (``InMemoryCacheStore`` bounds its memory and documents its one exception.) + - Every method touches exactly one key (Redis Cluster safe). + - Errors: raise any exception when the backend fails. The features decide whether to fail + open or closed. The store itself never swallows errors. + - All times are the store's clock (``now()``), so pods with skewed clocks agree. + """ + + @abstractmethod + async def aping(self) -> None: + """Return when the store answers; raise otherwise.""" + + @abstractmethod + async def now(self) -> float: + """The store's clock, in seconds since the epoch.""" + + @abstractmethod + async def acquire(self, key: str, limit: int, ttl: float) -> SlotResult: + """Take one of ``limit`` slots for ``key``, expiring ``ttl`` seconds from now.""" + + @abstractmethod + async def renew(self, key: str, slot_id: str, ttl: float) -> bool: + """Move a held slot's expiry to ``now() + ttl``; False if the slot is gone.""" + + @abstractmethod + async def release(self, key: str, slot_id: str) -> None: + """Remove a slot (no error when it is already gone).""" + + @abstractmethod + async def get(self, key: str) -> bytes | None: + """The value stored under ``key``, or None.""" + + @abstractmethod + async def set( + self, + key: str, + value: bytes, + ttl: float | None, + mode: SetMode = "always", + ) -> bool: + """Store ``value`` under ``key`` (see the class docstring for ``mode``); True if written.""" + + @abstractmethod + async def delete(self, key: str) -> None: + """Remove ``key`` (no error when it does not exist).""" + + def describe(self) -> str | None: + """Where the store lives (for example ``host:port``) for logs and startup checks. + + Never includes credentials. None when there is nothing useful to name. + """ + return None + + async def aclose(self) -> None: # noqa: B027 - optional hook + """Release resources (called once on app shutdown).""" diff --git a/tenxgraph_api/src/app/core/cache_store/loader.py b/tenxgraph_api/src/app/core/cache_store/loader.py new file mode 100644 index 00000000..09a7ac19 --- /dev/null +++ b/tenxgraph_api/src/app/core/cache_store/loader.py @@ -0,0 +1,110 @@ +"""Pick the API's cache store from 10xgraph.json and the environment, and bind it. + +Selection order (the first one set wins): + +1. ``"cache_store": "module:attr"`` in 10xgraph.json: a ``BaseCacheStore`` instance, or a + zero-argument factory (a class or function) that returns one. +2. ``"redis"`` in 10xgraph.json: ``RedisCacheStore.from_url(...)``. +3. ``REDIS_URL`` in the environment: ``RedisCacheStore.from_url(...)``. +4. Otherwise ``InMemoryCacheStore()``: correct for one process, not shared between workers. + +The store is bound in the InjectQ container as ``BaseCacheStore`` together with the +``LeaseManager`` that the run locks, run and WebSocket caps and idempotency use. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from .base import BaseCacheStore +from .memory_store import InMemoryCacheStore +from .redis_store import RedisCacheStore + + +if TYPE_CHECKING: + from injectq import InjectQ + + from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + + +logger = logging.getLogger("tenxgraph_api.cache_store") + + +def build_cache_store( + graph_config: GraphConfig | None, settings: Any +) -> tuple[BaseCacheStore, str]: + """The configured cache store and where it came from (for logs; never credentials). + + ``graph_config`` may be None (no 10xgraph.json). ``settings`` supplies ``REDIS_URL``. + """ + path = graph_config.cache_store if graph_config is not None else None + if path: + from tenxgraph_api.src.app.loader import load_cache_store + + return load_cache_store(path), f"from cache_store {path}" + + redis_url = graph_config.redis_url if graph_config is not None else None + source = "redis in 10xgraph.json" + if not redis_url: + redis_url = getattr(settings, "REDIS_URL", None) + source = "REDIS_URL" + if redis_url: + try: + return RedisCacheStore.from_url(redis_url), f"from {source}" + except ImportError: + logger.warning( + "%s is set but the 'redis' package is not installed; the cache store is " + "in-memory (per process). Install the redis extra to share it across workers.", + source, + ) + return InMemoryCacheStore(), f"{source} set but the redis package is missing" + + return InMemoryCacheStore(), "no cache_store, redis or REDIS_URL set" + + +@dataclass(frozen=True) +class CacheStoreSource: + """Where the bound cache store came from (``from REDIS_URL``), for the startup check.""" + + text: str | None = None + + +def describe_cache_store(store: BaseCacheStore, source: str | None = None) -> str: + """``RedisCacheStore (from REDIS_URL, localhost:6379)``: the store, its source, endpoint. + + The endpoint is the store's ``describe()``; without one (or without a source) only what is + known is named, at least the class. + """ + try: + where = store.describe() + except Exception: # a custom store's describe must not break boot + where = None + detail = ", ".join(part for part in (source, where) if part) + name = type(store).__name__ + return f"{name} ({detail})" if detail else name + + +def bind_cache_store(container: InjectQ, store: BaseCacheStore, source: str | None = None) -> None: + """Bind ``store`` as ``BaseCacheStore``, its ``CacheStoreSource`` and a ``LeaseManager``.""" + from tenxgraph_api.src.app.core.leases import LeaseManager + + container.bind_instance(BaseCacheStore, store) + container.bind_instance(CacheStoreSource, CacheStoreSource(source)) + container.bind_instance(LeaseManager, LeaseManager(store)) + + +def load_and_bind_cache_store( + container: InjectQ, graph_config: GraphConfig | None, settings: Any +) -> BaseCacheStore: + """Build the cache store (see the module docstring), bind it and log one INFO line.""" + store, source = build_cache_store(graph_config, settings) + bind_cache_store(container, store, source) + logger.info("Cache store: %s", describe_cache_store(store, source)) + return store + + +def is_in_memory(store: BaseCacheStore | None) -> bool: + """Whether ``store`` keeps its state in this process (or no store is bound).""" + return store is None or isinstance(store, InMemoryCacheStore) diff --git a/tenxgraph_api/src/app/core/cache_store/memory_store.py b/tenxgraph_api/src/app/core/cache_store/memory_store.py new file mode 100644 index 00000000..095a1db0 --- /dev/null +++ b/tenxgraph_api/src/app/core/cache_store/memory_store.py @@ -0,0 +1,326 @@ +"""``BaseCacheStore`` held in this process: correct for one worker, not shared between them. + +Each operation runs under the ``asyncio.Lock`` of its key's stripe and has no ``await`` inside +that lock, so it is one atomic step on the event loop. Slot sets and plain keys live in two +separate tables, each in least-recently-used order. + +Bounded memory, and the one exception to "never drop before TTL". Unbounded tables would let +a flood of distinct rate-limit keys (one per client IP) grow the process without limit, so +each table has a cap (``max_slot_keys``, 50,000 like the old memory rate limiter; +``max_values``, 10,000 like the old in-memory idempotency records). At the cap the store first +drops expired entries, then evicts the least recently used entry that is not protected, and +logs a warning (at most once a minute). Protected entries are never evicted while live: keys +whose feature segment is in ``protected_features`` (default ``concurrency``, every lease: +thread locks, run and WebSocket caps, idempotency running state; and ``revoked``, so a revoked +token cannot come back). They are bounded by real work in flight and by admin calls, not by +anonymous traffic. If only protected entries are left, the table grows past its cap (with the +warning) rather than break a lock. Everything else (rate-limit windows, idempotency records, +ownership cache entries) may be dropped before its TTL at the cap. + +Expired entries are swept in the background every ``SWEEP_EVERY_OPS`` operations or +``SWEEP_INTERVAL_SECONDS`` seconds, whichever comes first, in batches that yield to the loop. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import heapq +import logging +import math +import time +import uuid +from collections import OrderedDict + +from .base import BaseCacheStore, SetMode, SlotResult +from .redis_store import decimal_text + + +logger = logging.getLogger("tenxgraph_api.cache_store") + +_NUM_STRIPES = 64 + +DEFAULT_MAX_SLOT_KEYS = 50_000 +# Same as the old in-memory idempotency record cap (routers/graph/idempotency.py). +DEFAULT_MAX_VALUES = 10_000 +DEFAULT_PROTECTED_FEATURES = ("concurrency", "revoked") + + +def _check_ttl(ttl: float) -> None: + if not ttl > 0 or math.isinf(ttl): + raise ValueError(f"ttl must be a positive number of seconds, got {ttl!r}") + + +class _SlotSet: + """Live slots of one key: slot id -> expiry, plus a min-heap of expiries. + + The heap may hold stale entries (a renewed or released slot); they are skipped when they + reach the top and the heap is rebuilt when they outnumber the live slots. + """ + + __slots__ = ("expiries", "heap") + + def __init__(self) -> None: + self.expiries: dict[str, float] = {} + self.heap: list[tuple[float, str]] = [] + + def _clean_top(self) -> None: + heap, expiries = self.heap, self.expiries + while heap and expiries.get(heap[0][1]) != heap[0][0]: + heapq.heappop(heap) + + def drop_expired(self, now: float) -> None: + heap, expiries = self.heap, self.expiries + while heap and heap[0][0] <= now: + expiry, slot_id = heapq.heappop(heap) + if expiries.get(slot_id) == expiry: + del expiries[slot_id] + self._clean_top() + + def oldest(self) -> float | None: + self._clean_top() + return self.heap[0][0] if self.heap else None + + def put(self, slot_id: str, expiry: float) -> None: + self.expiries[slot_id] = expiry + heapq.heappush(self.heap, (expiry, slot_id)) + if len(self.heap) > 2 * len(self.expiries) + 16: + self.heap = [(e, s) for s, e in self.expiries.items()] + heapq.heapify(self.heap) + + def remove(self, slot_id: str) -> None: + self.expiries.pop(slot_id, None) + self._clean_top() + + +class InMemoryCacheStore(BaseCacheStore): + """Per-process cache store (see the module docstring for its memory bounds).""" + + SWEEP_EVERY_OPS = 2000 + SWEEP_INTERVAL_SECONDS = 60.0 + SWEEP_BATCH = 1000 + CAP_WARNING_INTERVAL = 60.0 + + def __init__( + self, + *, + max_slot_keys: int = DEFAULT_MAX_SLOT_KEYS, + max_values: int = DEFAULT_MAX_VALUES, + protected_features: tuple[str, ...] = DEFAULT_PROTECTED_FEATURES, + ) -> None: + self._slots: OrderedDict[str, _SlotSet] = OrderedDict() + # key -> (value, expiry or None) + self._values: OrderedDict[str, tuple[bytes, float | None]] = OrderedDict() + self._max_slot_keys = max_slot_keys + self._max_values = max_values + self._protected = tuple(f"{feature}:" for feature in protected_features) + self._locks = [asyncio.Lock() for _ in range(_NUM_STRIPES)] + self._ops = 0 + self._last_sweep = time.monotonic() + self._sweep_task: asyncio.Task[None] | None = None + self._last_cap_warning: dict[str, float] = {} + + def _stripe_lock(self, key: str) -> asyncio.Lock: + return self._locks[hash(key) % _NUM_STRIPES] + + def _protected_key(self, key: str) -> bool: + return key.startswith(self._protected) + + # Sweeping and caps + + def _tick(self) -> None: + self._ops += 1 + due = self._ops >= self.SWEEP_EVERY_OPS + if not due and time.monotonic() - self._last_sweep < self.SWEEP_INTERVAL_SECONDS: + return + if self._sweep_task is not None and not self._sweep_task.done(): + return + self._ops = 0 + self._last_sweep = time.monotonic() + self._sweep_task = asyncio.get_running_loop().create_task(self._sweep()) + + async def _sweep(self) -> None: + removed = 0 + for index, key in enumerate(list(self._slots)): + if index and index % self.SWEEP_BATCH == 0: + await asyncio.sleep(0) + slots = self._slots.get(key) + if slots is not None: + slots.drop_expired(time.time()) + if not slots.expiries: + del self._slots[key] + removed += 1 + for index, key in enumerate(list(self._values)): + if index and index % self.SWEEP_BATCH == 0: + await asyncio.sleep(0) + entry = self._values.get(key) + if entry is not None and entry[1] is not None and entry[1] <= time.time(): + del self._values[key] + removed += 1 + if removed: + logger.debug("Cache store sweep removed %d expired entries", removed) + + def _warn_cap(self, table: str, cap: int, detail: str) -> None: + now = time.monotonic() + last = self._last_cap_warning.get(table) + if last is not None and now - last < self.CAP_WARNING_INTERVAL: + return + self._last_cap_warning[table] = now + logger.warning("In-memory cache store %s cap (%d) reached; %s", table, cap, detail) + + def _make_room_for_slots(self, now: float) -> None: + """Free one entry in the slot table before a new key is added, if it is full.""" + if len(self._slots) < self._max_slot_keys: + return + victim = None + for key, slots in self._slots.items(): # least recently used first + slots.drop_expired(now) + if not slots.expiries or not self._protected_key(key): + victim = key + break + if victim is None: + self._warn_cap("slot", self._max_slot_keys, "only live leases are left, growing") + return + if self._slots[victim].expiries: + self._warn_cap("slot", self._max_slot_keys, "evicting least recently used keys") + del self._slots[victim] + + def _make_room_for_values(self, now: float) -> None: + if len(self._values) < self._max_values: + return + victim = None + for key, (_, expiry) in self._values.items(): + expired = expiry is not None and expiry <= now + if expired or not self._protected_key(key): + victim = key + break + if victim is None: + self._warn_cap("key", self._max_values, "only protected keys are left, growing") + return + expiry = self._values[victim][1] + if expiry is None or expiry > now: + self._warn_cap("key", self._max_values, "evicting least recently used keys") + del self._values[victim] + + # BaseCacheStore + + async def aping(self) -> None: + return None + + async def now(self) -> float: + return time.time() + + async def acquire(self, key: str, limit: int, ttl: float) -> SlotResult: + _check_ttl(ttl) + self._tick() + async with self._stripe_lock(key): + # No await below: the check and the add are one step. + now = time.time() + slots = self._slots.get(key) + if slots is not None: + slots.drop_expired(now) + self._slots.move_to_end(key) + count = len(slots.expiries) if slots is not None else 0 + if count >= limit: + if slots is not None and not slots.expiries: + del self._slots[key] + return SlotResult(False, 0, None, None) + return SlotResult(False, count, slots.oldest() if slots else None, None) + if slots is None: + self._make_room_for_slots(now) + slots = self._slots[key] = _SlotSet() + slot_id = uuid.uuid4().hex + slots.put(slot_id, now + ttl) + return SlotResult(True, count + 1, slots.oldest(), slot_id) + + async def renew(self, key: str, slot_id: str, ttl: float) -> bool: + _check_ttl(ttl) + self._tick() + async with self._stripe_lock(key): + now = time.time() + slots = self._slots.get(key) + if slots is None: + return False + slots.drop_expired(now) + if slot_id not in slots.expiries: + if not slots.expiries: + del self._slots[key] + return False + slots.put(slot_id, now + ttl) + self._slots.move_to_end(key) + return True + + async def release(self, key: str, slot_id: str) -> None: + async with self._stripe_lock(key): + slots = self._slots.get(key) + if slots is None: + return + slots.remove(slot_id) + if not slots.expiries: + del self._slots[key] + + async def get(self, key: str) -> bytes | None: + self._tick() + async with self._stripe_lock(key): + entry = self._values.get(key) + if entry is None: + return None + if entry[1] is not None and entry[1] <= time.time(): + del self._values[key] + return None + self._values.move_to_end(key) + return entry[0] + + async def set( + self, + key: str, + value: bytes, + ttl: float | None, + mode: SetMode = "always", + ) -> bool: + if ttl is not None: + _check_ttl(ttl) + if mode not in ("always", "if_absent", "if_greater"): + raise ValueError(f"Unknown set mode {mode!r}") + new_number = 0.0 + if mode == "if_greater": + value = decimal_text(value).encode() + new_number = float(value) + self._tick() + async with self._stripe_lock(key): + now = time.time() + entry = self._values.get(key) + if entry is not None and entry[1] is not None and entry[1] <= now: + del self._values[key] + entry = None + if mode == "if_absent" and entry is not None: + return False + if mode == "if_greater" and entry is not None: + current = _number(entry[0]) + if current is not None and current >= new_number: + return False + if entry is None: + self._make_room_for_values(now) + self._values[key] = (bytes(value), None if ttl is None else now + ttl) + self._values.move_to_end(key) + return True + + async def delete(self, key: str) -> None: + async with self._stripe_lock(key): + self._values.pop(key, None) + self._slots.pop(key, None) + + async def aclose(self) -> None: + task, self._sweep_task = self._sweep_task, None + if task is not None and not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +def _number(raw: bytes) -> float | None: + """A stored value as a number, or None when it is not one (then it counts as absent).""" + try: + return float(decimal_text(raw)) + except ValueError: + return None diff --git a/tenxgraph_api/src/app/core/cache_store/redis_store.py b/tenxgraph_api/src/app/core/cache_store/redis_store.py new file mode 100644 index 00000000..a948cbe3 --- /dev/null +++ b/tenxgraph_api/src/app/core/cache_store/redis_store.py @@ -0,0 +1,327 @@ +"""``BaseCacheStore`` on Redis (also Valkey, Dragonfly and KeyDB). + +Slots are a sorted set per key (member = slot id, score = expiry in seconds on the Redis +clock). ``acquire`` and ``renew`` run as one Lua script each, so they are atomic and read the +clock with ``TIME`` inside the script: pods with skewed clocks agree on every expiry. +``if_greater`` is a Lua script too. Every command and script touches exactly one key. + +Key layout. Callers pass ``":"``; the store maps the feature segment to a Redis +namespace and puts the deployment prefix (``REDIS_KEY_PREFIX``, via ``prefixed``) in front. +The namespaces are the ones the API used before the cache store, so old and new pods share +the same locks, records and revocations during a rolling deploy: + +============================== ================================================== +Caller key Redis key (``P:`` only when a prefix is set) +============================== ================================================== +``concurrency:`` ``P:10xgraph:concurrency:`` +``idempotency:`` ``P:10xgraph:idempotency:`` +``owner:id:`` ``P:af:authz:owner:id:`` +``owner:tomb:`` ``P:af:authz:owner:tomb:`` +``revoked:jti:`` ``P:10xgraph:revoked:jti:`` +``revoked:user:`` ``P:10xgraph:revoked:user:`` +``ratelimit:`` ``P:10xgraph:ratelimit2:`` +anything else ``P:10xgraph:cache:`` +============================== ================================================== + +Lease keys keep their old shape too (``runs:thread:``, ``runs:user:``, +``runs:idem:``, ``ws:all``, ``ws:user:``), with scores that are expiry seconds as +the API's Redis leases wrote them before the cache store. The rate limiter gets a new +namespace because its scores used to be request times and are now expiries; its counters +reset once at deploy. + +Redis must not evict keys that have a TTL under memory pressure (``maxmemory-policy`` must be +``noeviction`` or ``volatile-*`` with room to spare): an evicted lease lets two workers run the +same thread. +""" + +from __future__ import annotations + +import hashlib +import math +import re +import uuid +from typing import Any +from urllib.parse import urlsplit + +from tenxgraph_api.src.app.core.redis_client import create_redis, prefixed + +from .base import BaseCacheStore, SetMode, SlotResult + + +# Feature segment of a caller key -> the Redis namespace it has always used. +FEATURE_NAMESPACES: dict[str, str] = { + "concurrency": "10xgraph:concurrency", + "idempotency": "10xgraph:idempotency", + "owner": "af:authz:owner", + "revoked": "10xgraph:revoked", + "ratelimit": "10xgraph:ratelimit2", +} +# Keys whose feature is not listed above. +DEFAULT_NAMESPACE = "10xgraph:cache" + +# The key outlives its newest slot by this much, like ``expire(ttl + 1)`` did before. +_KEY_TTL_MARGIN_MS = 1000 + +_DECIMAL = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?") + +# Effects replication is the default since Redis 5 and the only mode in 7; asking for it keeps +# a write after TIME legal on older servers. +_PREAMBLE = """ +if redis.replicate_commands then redis.replicate_commands() end +local t = redis.call('TIME') +local now = tonumber(t[1]) + tonumber(t[2]) / 1000000 +local function touch_key() + local newest = redis.call('ZRANGE', KEYS[1], -1, -1, 'WITHSCORES') + if newest[2] then + local at = math.ceil(tonumber(newest[2]) * 1000) + tonumber(ARGV[4]) + redis.call('PEXPIREAT', KEYS[1], string.format('%.0f', at)) + end +end +""" + +# ARGV: limit, ttl seconds, slot id, key ttl margin ms. +# Returns {granted, live count, oldest expiry or ''}. Scores travel as strings: Lua numbers +# in a reply are truncated to integers. +_ACQUIRE_LUA = ( + _PREAMBLE + + """ +redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', string.format('%.6f', now)) +local count = redis.call('ZCARD', KEYS[1]) +local granted = 0 +if count < tonumber(ARGV[1]) then + redis.call('ZADD', KEYS[1], string.format('%.6f', now + tonumber(ARGV[2])), ARGV[3]) + count = count + 1 + granted = 1 + touch_key() +end +local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES') +return {granted, count, oldest[2] or ''} +""" +) + +# ARGV: unused, ttl seconds, slot id, key ttl margin ms. Returns 1 when renewed, 0 when the +# slot is gone. An expired slot that is still in the set counts as gone and is removed; ``XX`` +# makes sure nothing is ever added. +_RENEW_LUA = ( + _PREAMBLE + + """ +local score = redis.call('ZSCORE', KEYS[1], ARGV[3]) +if not score then + return 0 +end +if tonumber(score) <= now then + redis.call('ZREM', KEYS[1], ARGV[3]) + return 0 +end +redis.call('ZADD', KEYS[1], 'XX', string.format('%.6f', now + tonumber(ARGV[2])), ARGV[3]) +touch_key() +return 1 +""" +) + +# ARGV: value, ttl ms ('' for none). Keeps the larger number; a stored value that is not a +# number counts as absent (the old token revocation script behaved the same way). +_IF_GREATER_LUA = """ +local current = tonumber(redis.call('GET', KEYS[1]) or '') +if current ~= nil and current >= tonumber(ARGV[1]) then + return 0 +end +local ttl = tonumber(ARGV[2]) +if ttl ~= nil and ttl > 0 then + redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2]) +else + redis.call('SET', KEYS[1], ARGV[1]) +end +return 1 +""" + + +class _Script: + __slots__ = ("sha", "source") + + def __init__(self, source: str) -> None: + self.source = source + self.sha = hashlib.sha1(source.encode(), usedforsecurity=False).hexdigest() + + +_ACQUIRE = _Script(_ACQUIRE_LUA) +_RENEW = _Script(_RENEW_LUA) +_IF_GREATER = _Script(_IF_GREATER_LUA) + + +def _ttl_ms(ttl: float) -> int: + if not ttl > 0 or math.isinf(ttl): + raise ValueError(f"ttl must be a positive number of seconds, got {ttl!r}") + return max(1, math.ceil(ttl * 1000)) + + +def decimal_text(value: bytes) -> str: + """``value`` as decimal text for ``if_greater``; ``ValueError`` when it is not one.""" + try: + text = value.decode("ascii").strip() + except (UnicodeDecodeError, AttributeError): + raise ValueError("if_greater needs a decimal number as bytes") + if not _DECIMAL.fullmatch(text) or not math.isfinite(float(text)): + raise ValueError(f"if_greater needs a decimal number, got {text!r}") + return text + + +def _endpoint(url: str | None) -> str | None: + """``host:port`` of a URL, never user or password.""" + if not isinstance(url, str) or "://" not in url: + return None + try: + parts = urlsplit(url) + host, port = parts.hostname, parts.port + except ValueError: + return None + if not host: + return None + if ":" in host: + host = f"[{host}]" + return f"{host}:{port}" if port else host + + +def _is_noscript(exc: Exception) -> bool: + return type(exc).__name__ == "NoScriptError" or str(exc).startswith("NOSCRIPT") + + +class RedisCacheStore(BaseCacheStore): + """Shared cache store on Redis; see the module docstring for the key layout. + + Pass a ``url`` (the store builds and later closes its client with ``create_redis``, so + the timeouts and pool bound in the settings apply) or an existing async ``client`` (left + open on ``aclose``; its owner closes it). ``prefix`` overrides ``REDIS_KEY_PREFIX``; ``""`` + means no prefix. + """ + + def __init__( + self, + url: str | None = None, + *, + client: Any = None, + prefix: str | None = None, + ) -> None: + if (url is None) == (client is None): + raise ValueError("RedisCacheStore needs exactly one of url or client") + self._url = url + self._owns_client = client is None + if client is None: + try: + client = create_redis(url) # type: ignore[arg-type] + except ImportError: + raise ImportError( + 'RedisCacheStore needs the redis package: pip install "10xgraph-api[redis]"' + ) + self._client = client + if prefix is None: + self._namespaces = {f: prefixed(ns) for f, ns in FEATURE_NAMESPACES.items()} + self._default = prefixed(DEFAULT_NAMESPACE) + else: + lead = f"{prefix}:" if prefix else "" + self._namespaces = {f: lead + ns for f, ns in FEATURE_NAMESPACES.items()} + self._default = lead + DEFAULT_NAMESPACE + + @classmethod + def from_url(cls, url: str, **kwargs: Any) -> RedisCacheStore: + return cls(url, **kwargs) + + @property + def client(self) -> Any: + """The Redis client (for diagnostics; features go through the store's methods).""" + return self._client + + def redis_key(self, key: str) -> str: + """The Redis key a caller key is stored under.""" + feature, sep, rest = key.partition(":") + namespace = self._namespaces.get(feature) if sep else None + if namespace is None: + return f"{self._default}:{key}" + return f"{namespace}:{rest}" + + async def _script(self, script: _Script, key: str, *args: Any) -> Any: + try: + return await self._client.evalsha(script.sha, 1, key, *args) + except Exception as exc: + if not _is_noscript(exc): + raise + return await self._client.eval(script.source, 1, key, *args) + + async def aping(self) -> None: + await self._client.ping() + + async def now(self) -> float: + seconds, microseconds = await self._client.time() + return float(seconds) + float(microseconds) / 1_000_000 + + async def acquire(self, key: str, limit: int, ttl: float) -> SlotResult: + _ttl_ms(ttl) + slot_id = uuid.uuid4().hex + reply = await self._script( + _ACQUIRE, self.redis_key(key), int(limit), repr(float(ttl)), slot_id, _KEY_TTL_MARGIN_MS + ) + granted, count, oldest = int(reply[0]), int(reply[1]), reply[2] + if isinstance(oldest, bytes | bytearray): + oldest = oldest.decode() + return SlotResult( + granted=bool(granted), + count=count, + oldest_expiry=float(oldest) if oldest not in ("", None) else None, + slot_id=slot_id if granted else None, + ) + + async def renew(self, key: str, slot_id: str, ttl: float) -> bool: + _ttl_ms(ttl) + reply = await self._script( + _RENEW, self.redis_key(key), "", repr(float(ttl)), slot_id, _KEY_TTL_MARGIN_MS + ) + return bool(int(reply)) + + async def release(self, key: str, slot_id: str) -> None: + await self._client.zrem(self.redis_key(key), slot_id) + + async def get(self, key: str) -> bytes | None: + value = await self._client.get(self.redis_key(key)) + if value is None or isinstance(value, bytes): + return value + return value.encode() if isinstance(value, str) else bytes(value) + + async def set( + self, + key: str, + value: bytes, + ttl: float | None, + mode: SetMode = "always", + ) -> bool: + px = None if ttl is None else _ttl_ms(ttl) + redis_key = self.redis_key(key) + if mode == "if_greater": + reply = await self._script( + _IF_GREATER, redis_key, decimal_text(value), "" if px is None else str(px) + ) + return bool(int(reply)) + if mode == "if_absent": + return bool(await self._client.set(redis_key, value, px=px, nx=True)) + if mode == "always": + await self._client.set(redis_key, value, px=px) + return True + raise ValueError(f"Unknown set mode {mode!r}") + + async def delete(self, key: str) -> None: + await self._client.delete(self.redis_key(key)) + + def describe(self) -> str | None: + if self._url is not None: + return _endpoint(self._url) + pool = getattr(self._client, "connection_pool", None) + kwargs = getattr(pool, "connection_kwargs", None) + if not isinstance(kwargs, dict) or not kwargs.get("host"): + return None + return f"{kwargs['host']}:{kwargs.get('port', 6379)}" + + async def aclose(self) -> None: + if not self._owns_client: + return + close = getattr(self._client, "aclose", None) + if callable(close): + await close() diff --git a/tenxgraph_api/src/app/core/concurrency.py b/tenxgraph_api/src/app/core/concurrency.py deleted file mode 100644 index bf8e9966..00000000 --- a/tenxgraph_api/src/app/core/concurrency.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Concurrency caps that can be shared by every worker. - -The per-user caps on HTTP graph runs (``runs.max_concurrent_per_user``) and on WebSocket -connections (``websocket.max_connections`` / ``max_connections_per_user``) count holders of a -slot. In one process a dict does that. With several workers each process would count only its -own holders, so the real cap would be ``limit x workers``. - -:class:`RedisConcurrencyLimiter` keeps the holders in Redis instead, as a sorted set of leases -per key (member = lease id, score = expiry time). A held lease is refreshed in the background -and released on exit; if a worker dies its leases expire on their own, so a crash cannot leak -slots. The loader binds it whenever Redis is configured (the ``redis`` key in 10xgraph.json or -``REDIS_URL``), like the ownership cache. - -Acquiring runs one transaction (drop expired leases, add ours, count, set a key TTL) and backs -out if the count is over the limit. Two callers racing for the last slot may both back out, -but the cap is never exceeded. Expiry times use the Redis clock (``TIME``), not each pod's -own, so clock skew between pods cannot trim a live lease. A Redis error fails open to this -process's own count: the error is logged and the slot is taken from an in-process counter, -because refusing every request while Redis is down would be an outage of its own. The cap then -holds per process, not across workers, until Redis is back (so the per-thread run lock still -stops a double run in one worker). -""" - -from __future__ import annotations - -import asyncio -import contextlib -import logging -import time -import uuid -from abc import ABC, abstractmethod -from typing import Any - -from tenxgraph_api.src.app.core.redis_client import prefixed -from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe - - -logger = logging.getLogger("tenxgraph_api.concurrency") - - -# Release coroutines scheduled from sync code, referenced until they finish. -_pending_releases: set[asyncio.Task[Any]] = set() - -# Longest ``aclose`` waits for those releases before it closes the client anyway. -SHUTDOWN_RELEASE_WAIT_SECONDS = 5.0 - - -class Lease: - """A held slot. ``release`` is idempotent and safe to call from sync code.""" - - def __init__(self, release: Any = None) -> None: - self._release = release - self._released = release is None - # Set when a shared lease was found missing on refresh (expired or evicted while held). - self.lost = False - - @property - def released(self) -> bool: - return self._released - - def release(self) -> None: - if self._released: - return - self._released = True - result = self._release() - if asyncio.iscoroutine(result): - try: - task = asyncio.get_running_loop().create_task(result) - except RuntimeError: # no loop (garbage collection at shutdown): let it expire - result.close() - return - _pending_releases.add(task) # keep a reference until it finishes - task.add_done_callback(_pending_releases.discard) - - -class ConcurrencyLimiter(ABC): - @abstractmethod - async def acquire(self, key: str, limit: int) -> Lease | None: - """Take one of ``limit`` slots for ``key``; ``None`` when all are taken.""" - - def held(self, key: str) -> int | None: - """Slots held for ``key`` when cheaply known (in-process); otherwise None.""" - return None - - async def aclose(self) -> None: # noqa: B027 - optional hook - """Release resources (called on app shutdown).""" - - -class MemoryConcurrencyLimiter(ConcurrencyLimiter): - """Per-process slot counts. The default, and correct for a single worker. - - The event loop is single-threaded and there is no ``await`` between the check and the - increment, so acquiring needs no lock. - """ - - def __init__(self) -> None: - self._held: dict[str, int] = {} - - def held(self, key: str) -> int: - return self._held.get(key, 0) - - async def acquire(self, key: str, limit: int) -> Lease | None: - held = self._held.get(key, 0) - if held >= limit: - return None - self._held[key] = held + 1 - return Lease(lambda: self._release(key)) - - def _release(self, key: str) -> None: - held = self._held.get(key, 0) - if held <= 1: - self._held.pop(key, None) - else: - self._held[key] = held - 1 - - -class RedisConcurrencyLimiter(ConcurrencyLimiter): - """Slot counts shared through Redis, with expiring leases (see the module docstring).""" - - def __init__( - self, - client: Any, - *, - prefix: str | None = None, - lease_ttl: float = 60.0, - ) -> None: - self._client = client - self._prefix = prefix if prefix is not None else prefixed("10xgraph:concurrency") - self._lease_ttl = lease_ttl - self._refreshers: set[asyncio.Task[None]] = set() - # Counts used while Redis is unreachable, so one process still honours the cap. - self._local = MemoryConcurrencyLimiter() - - @property - def client(self) -> Any: - """The Redis client, so other shared state (idempotency records) uses the same one.""" - return self._client - - @classmethod - def from_url(cls, url: str, **kwargs: Any) -> RedisConcurrencyLimiter: - from tenxgraph_api.src.app.core.redis_client import create_redis - - return cls(create_redis(url), **kwargs) - - def _key(self, key: str) -> str: - return f"{self._prefix}:{key}" - - async def _now(self) -> float: - """The Redis server's clock. Lease expiry is compared across pods, so a pod's own - clock (skewed by more than the lease ttl) would trim another pod's live lease.""" - server_time = getattr(self._client, "time", None) - if not callable(server_time): - return time.time() - seconds, microseconds = await server_time() - return float(seconds) + float(microseconds) / 1_000_000 - - async def acquire(self, key: str, limit: int) -> Lease | None: - redis_key, member = self._key(key), uuid.uuid4().hex - try: - now = await self._now() - pipe = self._client.pipeline(transaction=True) - pipe.zremrangebyscore(redis_key, "-inf", now) - pipe.zadd(redis_key, {member: now + self._lease_ttl}) - pipe.zcard(redis_key) - pipe.expire(redis_key, int(self._lease_ttl) + 1) - results = await pipe.execute() - if int(results[2]) > limit: - await self._client.zrem(redis_key, member) - return None - except asyncio.CancelledError: - # The caller gave up (a client left while waiting). The transaction may already - # have added our member; take it out now rather than leave it for the lease ttl. - self._discard_later(redis_key, member) - raise - except Exception as exc: - logger.warning( - "Shared concurrency limit unavailable (%s); counting %s in this process", - exc, - log_safe(key), - ) - return await self._local.acquire(key, limit) - - lease = Lease(lambda: release()) # release is defined below, it needs the refresher - refresher = asyncio.create_task(self._refresh(redis_key, member, lease)) - self._refreshers.add(refresher) - refresher.add_done_callback(self._refreshers.discard) - - async def release() -> None: - refresher.cancel() - try: - await self._client.zrem(redis_key, member) - except Exception as exc: # the lease expires on its own - logger.debug("Lease release failed for %s: %s", log_safe(key), exc) - - return lease - - def _discard_later(self, redis_key: str, member: str) -> None: - """Remove ``member`` on a task of its own, for callers that are being cancelled.""" - - async def discard() -> None: - try: - await self._client.zrem(redis_key, member) - except Exception as exc: # the lease expires on its own - logger.debug("Discarding a lease failed for %s: %s", log_safe(redis_key), exc) - - task = asyncio.get_running_loop().create_task(discard()) - _pending_releases.add(task) - task.add_done_callback(_pending_releases.discard) - - async def _refresh(self, redis_key: str, member: str, lease: Lease) -> None: - """Keep a held lease alive; it expires ``lease_ttl`` after the last refresh.""" - interval = max(self._lease_ttl / 3, 0.01) - while True: - await asyncio.sleep(interval) - try: - expiry = await self._now() + self._lease_ttl - # ``xx`` never adds, and without ``ch`` Redis reports 0 for a refreshed member - # as well as for a missing one, so the reply could not tell them apart. - changed = await self._client.zadd(redis_key, {member: expiry}, xx=True, ch=True) - if not changed: - lease.lost = True - logger.warning( - "Lease %s expired or was evicted while held; taking it again", redis_key - ) - await self._client.zadd(redis_key, {member: expiry}) - await self._client.expire(redis_key, int(self._lease_ttl) + 1) - except Exception as exc: - logger.warning("Lease refresh failed for %s: %s", log_safe(redis_key), exc) - - async def aclose(self) -> None: - for task in list(self._refreshers): - task.cancel() - # Releases scheduled from sync code run on tasks of their own; the client has to stay - # open until they have, or the leases stay until their ttl (a 409 on the next pod). - while _pending_releases: - pending = set(_pending_releases) - _, still = await asyncio.wait(pending, timeout=SHUTDOWN_RELEASE_WAIT_SECONDS) - if still: - logger.warning("%d lease releases did not finish before shutdown", len(still)) - break - close = getattr(self._client, "aclose", None) - if callable(close): - with contextlib.suppress(Exception): - await close() - - -# Used when no shared limiter is bound: one per process. -_process_limiter = MemoryConcurrencyLimiter() - - -def shared_concurrency_limiter() -> ConcurrencyLimiter | None: - """The limiter bound in the app's container (Redis-backed when configured), or None.""" - try: - from tenxgraph_api.src.app.core.utils.container import current_container - - bound = current_container().try_get(ConcurrencyLimiter) - except Exception: - return None - return bound if isinstance(bound, ConcurrencyLimiter) else None - - -def concurrency_limiter() -> ConcurrencyLimiter: - """The shared limiter bound in the app's container, else this process's own.""" - return shared_concurrency_limiter() or _process_limiter diff --git a/tenxgraph_api/src/app/core/config/graph_config.py b/tenxgraph_api/src/app/core/config/graph_config.py index eba5ebd2..3d8fe9c4 100644 --- a/tenxgraph_api/src/app/core/config/graph_config.py +++ b/tenxgraph_api/src/app/core/config/graph_config.py @@ -139,62 +139,33 @@ def _parse_networks(raw: object) -> tuple[str, ...]: class RateLimitConfig: """Rate limit configuration parsed from 10xgraph.json. - Example (memory backend, default):: + Example:: "rate_limit": { "enabled": true, - "backend": "memory", "requests": 100, "window": 60, "by": "ip", "trusted_proxy_headers": false, - "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"] - } - - Example (Redis backend):: - - "rate_limit": { - "enabled": true, - "backend": "redis", - "requests": 100, - "window": 60, - "by": "ip", - "trusted_proxy_headers": true, - "exclude_paths": ["/ping"], - "redis": { - "url": "redis://localhost:6379/0", - "prefix": "10xgraph:rate-limit" - }, + "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"], "fail_open": true } - Example (custom backend):: - - "rate_limit": { - "enabled": true, - "backend": "custom", - "requests": 100, - "window": 60, - "by": "ip" - } - - For custom backends, bind a ``BaseRateLimitBackend`` instance in InjectQ. + The counters live in the API's cache store (``cache_store``, else ``redis`` / + ``REDIS_URL``, else in this process), like the run locks and caps. """ enabled: bool requests: int window: int by: str # "ip" | "user" | "global" - backend: str # "memory" | "redis" | "custom" - redis_url: str | None - redis_prefix: str exclude_paths: tuple[str, ...] trusted_proxy_headers: bool # honour X-Forwarded-For only when True # Number of proxies you control in front of the app. Only this many entries, # counted from the RIGHT of X-Forwarded-For, were appended by your own # infrastructure; anything further left came from the caller and is forgeable. trusted_proxy_hops: int = 1 - fail_open: bool = True # on backend error: True=allow, False=deny + fail_open: bool = True # on a cache store error: True=allow, False=deny with 503 # Networks your proxies connect from. X-Forwarded-For is honoured only for requests # whose peer address is in one of them, so a client that reaches the app directly # (bypassing the proxy) cannot pick its own address. Empty means @@ -205,12 +176,18 @@ class RateLimitConfig: def from_dict(cls, data: dict) -> "RateLimitConfig": if not isinstance(data, dict): raise ValueError("rate_limit must be an object") + for removed in ("backend", "redis"): + if removed in data: + raise ValueError( + f"rate_limit.{removed} was removed: rate limits are kept in the API's " + "cache store. Delete it and set `redis` in 10xgraph.json (or REDIS_URL) to " + "share the counters between workers, or `cache_store` for your own store." + ) enabled = _parse_bool(data.get("enabled", True), field="rate_limit.enabled") requests = int(data.get("requests", 100)) window = int(data.get("window", 60)) by = data.get("by", "ip") - backend = data.get("backend", "memory") trusted_proxy_headers = _parse_bool( data.get("trusted_proxy_headers", False), field="rate_limit.trusted_proxy_headers", @@ -221,18 +198,6 @@ def from_dict(cls, data: dict) -> "RateLimitConfig": exclude_paths = tuple(str(path) for path in exclude_paths_raw) fail_open = _parse_bool(data.get("fail_open", True), field="rate_limit.fail_open") - # Redis sub-object: {"url": "...", "prefix": "..."} - redis_obj = data.get("redis") or {} - if isinstance(redis_obj, str): - # Allow shorthand: "redis": "redis://..." - redis_url: str | None = _expand_env(redis_obj) - redis_prefix = "10xgraph:rate-limit" - elif isinstance(redis_obj, dict): - redis_url = _expand_env(redis_obj.get("url") or None) - redis_prefix = str(redis_obj.get("prefix", "10xgraph:rate-limit")) - else: - raise ValueError("rate_limit.redis must be an object or Redis URL string") - # How many proxies of YOUR OWN sit in front of the app and append to # X-Forwarded-For. Only that many entries, counted from the right, were # written by infrastructure you control; everything to the left of them came @@ -243,10 +208,6 @@ def from_dict(cls, data: dict) -> "RateLimitConfig": # Validation if by not in ("ip", "global", "user"): raise ValueError(f"rate_limit.by must be 'ip', 'user', or 'global', got '{by}'") - if backend not in ("memory", "redis", "custom"): - raise ValueError( - f"rate_limit.backend must be 'memory', 'redis', or 'custom', got '{backend}'" - ) if requests <= 0: raise ValueError("rate_limit.requests must be a positive integer") if window <= 0: @@ -261,24 +222,11 @@ def from_dict(cls, data: dict) -> "RateLimitConfig": "can reach the app except through it)." ) - # The memory backend counts per PROCESS. Under gunicorn with N workers the - # effective limit therefore becomes requests x N, and it resets whenever a - # worker restarts -- so it does not really limit anything in production. - if enabled and backend == "memory": - logger.warning( - "Rate limiting uses the in-memory backend. It counts per process, so " - "with N workers the real limit is requests x N and it resets on every " - "worker restart. Use the 'redis' backend for any multi-worker deployment." - ) - return cls( enabled=enabled, requests=requests, window=window, by=by, - backend=backend, - redis_url=redis_url, - redis_prefix=redis_prefix, exclude_paths=exclude_paths, trusted_proxy_headers=trusted_proxy_headers, trusted_proxy_hops=trusted_proxy_hops, @@ -287,6 +235,76 @@ def from_dict(cls, data: dict) -> "RateLimitConfig": ) +# A revoke_user entry is kept this long unless told otherwise. It has to outlive the longest +# token you issue, or an old token comes back to life when the entry goes. +DEFAULT_USER_REVOCATION_TTL = 30 * 24 * 3600 + + +def _positive_seconds(value: object, *, field: str) -> float: + if isinstance(value, bool) or not isinstance(value, int | float) or not value > 0: + raise ValueError(f"{field} must be a positive number of seconds, got {value!r}") + return float(value) + + +@dataclass(frozen=True) +class TokenRevocationConfig: + """``token_revocation`` in 10xgraph.json: ``true``, or an object with its settings. + + Example:: + + "token_revocation": {"enabled": true, "token_lifetime": 3600, "user_ttl": 2592000} + + Revocations live in the API's cache store (``cache_store``, else ``redis`` / + ``REDIS_URL``, else in this process). ``token_lifetime`` is the longest lifetime of the + tokens you issue, in seconds; it dates a token that has no ``iat`` from its ``exp``. + ``user_ttl`` is how long a ``revoke_user`` entry is kept by default (``null`` keeps it + until deleted). + """ + + token_lifetime: float | None = None + user_ttl: float | None = DEFAULT_USER_REVOCATION_TTL + + @classmethod + def parse(cls, value: object) -> "TokenRevocationConfig | None": + """The settings for ``value``; None when token revocation is off.""" + if value is None or value is False: + return None + if value is True: + return cls() + if isinstance(value, str): + raise ValueError( + f"token_revocation {value!r} is no longer supported: revocations are kept in " + 'the API\'s cache store. Use "token_revocation": true (or an object ' + '{"enabled": true, "token_lifetime": ..., "user_ttl": ...}) and set `redis` in ' + "10xgraph.json (or REDIS_URL) to share them between workers, or `cache_store` " + "for your own store." + ) + if not isinstance(value, dict): + raise ValueError("token_revocation must be true, false, null or an object") + unknown = set(value) - {"enabled", "token_lifetime", "user_ttl"} + if unknown: + raise ValueError( + f"token_revocation: unknown keys {sorted(unknown)}; use enabled, " + "token_lifetime and user_ttl" + ) + if not _parse_bool(value.get("enabled", True), field="token_revocation.enabled"): + return None + lifetime = value.get("token_lifetime") + user_ttl = value.get("user_ttl", DEFAULT_USER_REVOCATION_TTL) + return cls( + token_lifetime=( + None + if lifetime is None + else _positive_seconds(lifetime, field="token_revocation.token_lifetime") + ), + user_ttl=( + None + if user_ttl is None + else _positive_seconds(user_ttl, field="token_revocation.user_ttl") + ), + ) + + # Finite defaults so an unconfigured server cannot be held open by one client. DEFAULT_WS_MAX_CONNECTIONS = 1000 DEFAULT_WS_MAX_CONNECTIONS_PER_USER = 10 @@ -311,11 +329,12 @@ class WebSocketConfig: "realtime_client_overrides": ["voice", "modalities", "vad"] } - ``max_connections`` caps the concurrent WebSocket connections this server *process* - accepts (realtime ``/v1/graph/live`` + streaming ``/v1/graph/ws``). + ``max_connections`` caps the concurrent WebSocket connections the server accepts + (realtime ``/v1/graph/live`` + streaming ``/v1/graph/ws``). ``max_connections_per_user`` caps how many of those one verified user may hold, so a - single account cannot take every slot. Both are per process, like the in-memory - rate-limit backend; size them per worker. + single account cannot take every slot. Both are counted in the API's cache store: across + workers with a shared one (``redis``, ``REDIS_URL`` or ``cache_store``), per process with + the in-memory one (then size them per worker). A missing key gets a finite default (1000 and 10). Set a key to ``0`` or ``null`` to make it unlimited. WebSocket handshakes, and every graph run started over ``/v1/graph/ws``, also @@ -641,7 +660,30 @@ def store_path(self) -> str | None: @property def redis_url(self) -> str | None: - return self.data.get("redis", None) + """``redis`` key: the Redis URL of the API's cache store; ``$VAR`` / ``${VAR}`` expand. + + An unset variable is a config error, not a silent fall back to the in-memory store. + """ + value = self.data.get("redis", None) + if value is None or value == "": + return None + if not isinstance(value, str): + raise ValueError(f"redis must be a URL string, got {type(value).__name__}") + try: + return _expand_env(value) + except ValueError as exc: + raise ValueError(f"redis: {exc}") from None + + @property + def cache_store(self) -> str | None: + """``cache_store`` key: ``"module:attr"`` resolving to a ``BaseCacheStore`` (an instance + or a zero-argument factory), or None to pick one from ``redis`` / ``REDIS_URL``.""" + value = self.data.get("cache_store", None) + if value is None or value == "": + return None + if not isinstance(value, str) or ":" not in value: + raise ValueError(f"cache_store must be a 'module:attr' string, got {value!r}") + return value @property def thread_name_generator_path(self) -> str | None: @@ -793,12 +835,13 @@ def introspection_scoped(self) -> bool: return self.graph_introspection == "scoped" @property - def token_revocation(self) -> str | None: - """``token_revocation`` key: ``"redis"``, ``"memory"``, ``"module:attr"`` or None.""" - value = self.data.get("token_revocation", None) - if value is not None and not isinstance(value, str): - raise ValueError("token_revocation must be 'redis', 'memory', 'module:attr' or null") - return value or None + def token_revocation(self) -> TokenRevocationConfig | None: + """``token_revocation`` key: its settings when on (``true`` or an object), else None. + + See :class:`TokenRevocationConfig`. The old string values (``"redis"``, ``"memory"``, + ``"module:attr"``) raise ``ValueError`` naming the replacement. + """ + return TokenRevocationConfig.parse(self.data.get("token_revocation", None)) @property def runs(self) -> RunsConfig: diff --git a/tenxgraph_api/src/app/core/config/settings.py b/tenxgraph_api/src/app/core/config/settings.py index 31cd80ec..2010b1ee 100644 --- a/tenxgraph_api/src/app/core/config/settings.py +++ b/tenxgraph_api/src/app/core/config/settings.py @@ -143,9 +143,12 @@ class Settings(BaseSettings): ################################# ###### REDIS Config ########## ################################# + # The API's cache store (run locks and caps, idempotency, rate limit, token revocation, + # ownership cache) is a RedisCacheStore on this URL when 10xgraph.json sets neither + # `cache_store` nor `redis`. Unset (and neither key set): an in-memory store, per process. REDIS_URL: str | None = None - # Limits for every Redis client the API opens (run locks, caps, rate limit, token - # revocation, ownership cache, readiness probe). Without them a Redis that stops + # Limits for the Redis client of a RedisCacheStore built from `redis` or REDIS_URL (and + # of `10xgraph revoke`). Without them a Redis that stops # answering (not refusing) would hang each request that touches it. Seconds; 0 turns the # timeout off. REDIS_MAX_CONNECTIONS bounds each client's pool; a request waits up to # REDIS_POOL_TIMEOUT for a free connection. 0 means no bound. @@ -154,19 +157,21 @@ class Settings(BaseSettings): REDIS_MAX_CONNECTIONS: int = Field(default=50, ge=0) REDIS_POOL_TIMEOUT: float = Field(default=5.0, ge=0.0) REDIS_HEALTH_CHECK_INTERVAL: float = Field(default=30.0, ge=0.0) - # Put in front of every key the API writes to Redis (run locks and caps, ownership cache, - # token revocation), so two deployments that share one Redis cannot read each other's - # entries. Empty means no prefix. Give the core checkpointer the same value through its - # ``redis_key_prefix`` option so its cache keys are separated too. + # Put in front of every key a RedisCacheStore writes (run locks and caps, idempotency, + # rate limit, ownership cache, token revocation), so two deployments that share one Redis + # cannot read each other's entries. Empty means no prefix. Give the core checkpointer the + # same value through its ``redis_key_prefix`` option so its cache keys are separated too. REDIS_KEY_PREFIX: str = "" ################################# ###### Startup checks ########### ################################# - # At startup, before serving, ping every configured backend (the checkpointer's Postgres - # and Redis, the store, the API's Redis, the rate limiter's Redis) and refuse to start when - # one stays unreachable. Applies in development and production, so a wrong connection - # string fails the boot (and the deploy) instead of the first request. False skips it. + # At startup, before serving, call aping() on the checkpointer, the store and the cache + # store, and refuse to start when one stays unreachable. The cache store counts only when + # a fail-closed feature uses it (token_revocation, or rate_limit with fail_open false); + # otherwise an unreachable one is one warning. Applies in development and production, so + # a wrong connection string fails the boot (and the deploy) instead of the first request. + # False skips it. STARTUP_CHECKS: bool = True # How long, in seconds, startup keeps retrying a failing backend (with a short backoff) # before it gives up, so services started together (docker compose) can come up in any diff --git a/tenxgraph_api/src/app/core/config/setup_middleware.py b/tenxgraph_api/src/app/core/config/setup_middleware.py index 8290cef3..397b0042 100644 --- a/tenxgraph_api/src/app/core/config/setup_middleware.py +++ b/tenxgraph_api/src/app/core/config/setup_middleware.py @@ -14,7 +14,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from tenxgraph_api.src.app.core.exceptions.server_errors import UnhandledErrorMiddleware -from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware, build_backend +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware from tenxgraph_api.src.app.core.middleware.request_limits import RequestSizeLimitMiddleware from tenxgraph_api.src.app.core.middleware.security_headers import SecurityHeadersMiddleware @@ -481,23 +481,21 @@ def setup_middleware( if graph_config is not None: rate_limit_config = graph_config.rate_limit if rate_limit_config is not None: - backend = build_backend(rate_limit_config, container=container) - # Store on app.state so lifespan can close it cleanly - app.state.rate_limit_backend = backend + # The counters live in the cache store, bound in the container at startup. app.add_middleware( RateLimitMiddleware, config=rate_limit_config, - backend=backend, + container=container, ) logger.info( - "Rate limiting enabled: backend=%s, %d req/%ds, by=%s, " - "exclude_paths=%s, trusted_proxy_headers=%s", - rate_limit_config.backend, + "Rate limiting enabled: %d req/%ds, by=%s, exclude_paths=%s, " + "trusted_proxy_headers=%s, fail_open=%s", rate_limit_config.requests, rate_limit_config.window, rate_limit_config.by, rate_limit_config.exclude_paths or "(none)", "on" if rate_limit_config.trusted_proxy_headers else "off", + rate_limit_config.fail_open, ) app.add_middleware( diff --git a/tenxgraph_api/src/app/core/leases.py b/tenxgraph_api/src/app/core/leases.py new file mode 100644 index 00000000..a7bbc857 --- /dev/null +++ b/tenxgraph_api/src/app/core/leases.py @@ -0,0 +1,331 @@ +"""Leases: held slots on the cache store, renewed while held and released on exit. + +The per-thread run lock, the per-user run cap (``runs.max_concurrent_per_user``), the +WebSocket connection caps (``websocket.max_connections`` / ``max_connections_per_user``) and +the idempotency running state all count holders of a slot. :class:`LeaseManager` takes those +slots from the bound ``BaseCacheStore`` (``acquire`` / ``renew`` / ``release``), under keys +``concurrency:``, so they are shared between workers when the store is shared +(``RedisCacheStore``) and per process with ``InMemoryCacheStore``. + +A held lease expires ``LEASE_TTL_SECONDS`` after its last renewal and is renewed in the +background every third of that, so a worker that dies cannot hold a slot for longer than the +ttl. A renewal that finds the slot gone (expired during a stall, or evicted) sets +``Lease.lost`` and takes the slot again. + +A cache store error fails open to this process's own count: the error is logged and the slot +is taken from an in-process ``InMemoryCacheStore`` instead, because refusing every request +while the store is down would be an outage of its own. The cap then holds per process, not +across workers, until the store is back (so the per-thread run lock still stops a double run +in one worker). Leases held in either store count against the same limit in this process, and +a lease taken in the fallback stays there until it is released. + +Releases run on tasks of their own (a lease is often released from sync code, a ``finally`` +or a garbage collection callback), and the app's shutdown waits for them before it closes the +cache store (``wait_for_releases``). A caller cancelled while its slot is being taken has the +slot released on such a task too, so a client that left while waiting leaves no ghost slot. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections.abc import Callable, Coroutine +from typing import Any + +from tenxgraph_api.src.app.core.cache_store.base import BaseCacheStore, SlotResult +from tenxgraph_api.src.app.core.cache_store.memory_store import InMemoryCacheStore +from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe + + +logger = logging.getLogger("tenxgraph_api.concurrency") + +# A held lease expires this long after its last renewal. +LEASE_TTL_SECONDS = 60.0 + +# Longest the shutdown waits for pending releases before it closes the cache store anyway. +SHUTDOWN_RELEASE_WAIT_SECONDS = 5.0 + +# Feature segment of every lease key in the cache store. +LEASE_FEATURE = "concurrency" + +# Release coroutines scheduled from sync code, referenced until they finish. +_pending_releases: set[asyncio.Task[Any]] = set() + + +def _schedule(coro: Coroutine[Any, Any, Any]) -> asyncio.Task[Any] | None: + """Run ``coro`` on a task of its own, referenced until it finishes; None without a loop.""" + try: + task = asyncio.get_running_loop().create_task(coro) + except RuntimeError: # no loop (garbage collection at shutdown): let the slot expire + coro.close() + return None + _pending_releases.add(task) + task.add_done_callback(_pending_releases.discard) + return task + + +async def wait_for_releases(timeout: float = SHUTDOWN_RELEASE_WAIT_SECONDS) -> None: + """Wait (up to ``timeout`` in all) for releases scheduled on tasks of their own.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while _pending_releases: + remaining = deadline - loop.time() + if remaining <= 0: + logger.warning( + "%d lease releases did not finish before shutdown", len(_pending_releases) + ) + return + await asyncio.wait(set(_pending_releases), timeout=remaining) + + +class Lease: + """A held slot. ``release`` is idempotent and safe to call from sync code.""" + + def __init__(self, release: Callable[[], Any] | None = None) -> None: + self._release = release + self._released = release is None + # Set when a shared lease was found missing on renewal (expired or evicted while held). + self.lost = False + + @property + def released(self) -> bool: + return self._released + + def release(self) -> None: + if self._released: + return + self._released = True + result = self._release() # type: ignore[misc] + if asyncio.iscoroutine(result): + _schedule(result) + + +class _Held: + """Where a granted lease lives; ``slot_id`` changes when a lost slot is taken again.""" + + __slots__ = ("key", "limit", "renewer", "slot_id", "stop", "store") + + def __init__(self, store: BaseCacheStore, key: str, limit: int, slot_id: str) -> None: + self.store = store + self.key = key + self.limit = limit + self.slot_id: str | None = slot_id + self.stop = asyncio.Event() + self.renewer: asyncio.Task[None] | None = None + + +class LeaseManager: + """Leases on a ``BaseCacheStore``, failing open to an in-process store (module docstring).""" + + def __init__( + self, + cache: BaseCacheStore, + *, + lease_ttl: float = LEASE_TTL_SECONDS, + fallback: BaseCacheStore | None = None, + ) -> None: + self._cache = cache + self._lease_ttl = lease_ttl + # Slots taken while the cache store is unreachable, so one process still honours caps. + self._fallback = fallback if fallback is not None else InMemoryCacheStore() + # Leases this manager holds per lease key, and how many of them are in the fallback. + self._held: dict[str, int] = {} + self._in_fallback: dict[str, int] = {} + # Releases still running per lease key; a new acquire in this process waits for them. + self._releasing: dict[str, set[asyncio.Task[Any]]] = {} + self._renewers: set[asyncio.Task[None]] = set() + + @property + def cache(self) -> BaseCacheStore: + """The cache store the leases are shared through (also used for idempotency records).""" + return self._cache + + @property + def fallback(self) -> BaseCacheStore: + return self._fallback + + def held(self, key: str) -> int: + """Leases this process holds for ``key`` (not the count across workers).""" + return self._held.get(key, 0) + + async def acquire(self, key: str, limit: int) -> Lease | None: + """Take one of ``limit`` slots for ``key``; None when all are taken.""" + pending = self._releasing.get(key) + if pending: + # This process's own release of the same key goes first, so a slot given back + # just now is free for the next caller here (as it is with an in-process count). + await asyncio.wait(set(pending)) + store_key = f"{LEASE_FEATURE}:{key}" + local = self._in_fallback.get(key, 0) + try: + result = await self._take(self._cache, key, store_key, limit) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning( + "Shared concurrency limit unavailable (%s); counting %s in this process", + exc, + log_safe(key), + ) + # Leases this process holds in the shared store still count against the limit. + shared_here = self._held.get(key, 0) - self._in_fallback.get(key, 0) + result = await self._take(self._fallback, key, store_key, limit - shared_here) + if not result.granted or result.slot_id is None: + return None + return self._hold(self._fallback, key, store_key, limit, result.slot_id) + if not result.granted or result.slot_id is None: + return None + if local and result.count + local > limit: + # Leases taken in the fallback during an outage are invisible to the shared store. + self._release_later(key, self._release_slot(self._cache, store_key, result.slot_id)) + return None + return self._hold(self._cache, key, store_key, limit, result.slot_id) + + async def _take( + self, store: BaseCacheStore, key: str, store_key: str, limit: int + ) -> SlotResult: + """``store.acquire`` that gives the slot back if the caller is cancelled meanwhile.""" + task = asyncio.ensure_future(store.acquire(store_key, limit, self._lease_ttl)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + if not task.cancelled(): + # The store may grant the slot after the caller left; release it then. + self._release_later(key, self._discard_when_done(task, store, store_key)) + raise + + async def _discard_when_done( + self, task: asyncio.Future[SlotResult], store: BaseCacheStore, store_key: str + ) -> None: + try: + result = await task + except BaseException: # nothing was granted, or the error was already logged + return + if result.granted and result.slot_id is not None: + await self._release_slot(store, store_key, result.slot_id) + + def _hold( + self, store: BaseCacheStore, key: str, store_key: str, limit: int, slot_id: str + ) -> Lease: + held = _Held(store, store_key, limit, slot_id) + in_fallback = store is self._fallback + self._count(self._held, key, 1) + if in_fallback: + self._count(self._in_fallback, key, 1) + + def release() -> None: + self._count(self._held, key, -1) + if in_fallback: + self._count(self._in_fallback, key, -1) + with contextlib.suppress(RuntimeError): # the loop is already closed + held.stop.set() + self._release_later(key, self._finish(held)) + + lease = Lease(release) + renewer = asyncio.get_running_loop().create_task(self._renew(held, lease)) + held.renewer = renewer + self._renewers.add(renewer) + renewer.add_done_callback(self._renewers.discard) + return lease + + @staticmethod + def _count(table: dict[str, int], key: str, delta: int) -> None: + value = table.get(key, 0) + delta + if value > 0: + table[key] = value + else: + table.pop(key, None) + + def _release_later(self, key: str, coro: Coroutine[Any, Any, Any]) -> None: + task = _schedule(coro) + if task is None: + return + tasks = self._releasing.setdefault(key, set()) + tasks.add(task) + + def done(finished: asyncio.Task[Any]) -> None: + tasks.discard(finished) + if not tasks and self._releasing.get(key) is tasks: + del self._releasing[key] + + task.add_done_callback(done) + + async def _finish(self, held: _Held) -> None: + """Release a lease once its renewer has stopped (it may be taking the slot again).""" + renewer = held.renewer + if renewer is not None and not renewer.done(): + await asyncio.wait({renewer}) + slot_id, held.slot_id = held.slot_id, None + if slot_id is not None: + await self._release_slot(held.store, held.key, slot_id) + + @staticmethod + async def _release_slot(store: BaseCacheStore, store_key: str, slot_id: str) -> None: + try: + await store.release(store_key, slot_id) + except Exception as exc: # the slot expires on its own + logger.debug("Lease release failed for %s: %s", log_safe(store_key), exc) + + async def _renew(self, held: _Held, lease: Lease) -> None: + """Keep a held lease alive until it is released; take it again if it was lost.""" + interval = max(self._lease_ttl / 3, 0.01) + while True: + with contextlib.suppress(TimeoutError): + async with asyncio.timeout(interval): + await held.stop.wait() + if held.stop.is_set(): + return + try: + slot_id = held.slot_id + if slot_id is not None and await held.store.renew( + held.key, slot_id, self._lease_ttl + ): + continue + if held.stop.is_set(): + return + if slot_id is not None: + held.slot_id = None + lease.lost = True + logger.warning( + "Lease %s expired or was evicted while held; taking it again", + log_safe(held.key), + ) + result = await held.store.acquire(held.key, held.limit, self._lease_ttl) + if result.granted: + held.slot_id = result.slot_id + except Exception as exc: + logger.warning("Lease refresh failed for %s: %s", log_safe(held.key), exc) + + async def aclose(self) -> None: + """Wait for pending releases, then stop renewing leases that are still held.""" + await wait_for_releases() + renewers = list(self._renewers) + for task in renewers: + task.cancel() + await asyncio.gather(*renewers, return_exceptions=True) + with contextlib.suppress(Exception): + await self._fallback.aclose() + + +# Used when no LeaseManager is bound (no app container): one per process. +_process_manager: LeaseManager | None = None + + +def process_lease_manager() -> LeaseManager: + """This process's own lease manager, over an ``InMemoryCacheStore``.""" + global _process_manager # noqa: PLW0603 - one per process, made on first use + if _process_manager is None: + _process_manager = LeaseManager(InMemoryCacheStore()) + return _process_manager + + +def lease_manager() -> LeaseManager: + """The lease manager bound in the app's container, else this process's own.""" + try: + from tenxgraph_api.src.app.core.utils.container import current_container + + bound = current_container().try_get(LeaseManager) + except Exception: + bound = None + return bound if isinstance(bound, LeaseManager) else process_lease_manager() diff --git a/tenxgraph_api/src/app/core/middleware/__init__.py b/tenxgraph_api/src/app/core/middleware/__init__.py index 1fa92e61..8aa0463e 100644 --- a/tenxgraph_api/src/app/core/middleware/__init__.py +++ b/tenxgraph_api/src/app/core/middleware/__init__.py @@ -1,24 +1,13 @@ """Middleware modules for tenxgraph_api.""" -from .rate_limit import ( - BaseRateLimitBackend, - MemoryRateLimitBackend, - RateLimitDecision, - RateLimitMiddleware, - RedisRateLimitBackend, - build_backend, -) +from .rate_limit import RateLimitDecision, RateLimitMiddleware from .request_limits import RequestSizeLimitMiddleware from .security_headers import SecurityHeadersMiddleware, create_security_headers_middleware __all__ = [ - "BaseRateLimitBackend", - "RateLimitMiddleware", "RateLimitDecision", - "MemoryRateLimitBackend", - "RedisRateLimitBackend", - "build_backend", + "RateLimitMiddleware", "RequestSizeLimitMiddleware", "SecurityHeadersMiddleware", "create_security_headers_middleware", diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py b/tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py index 84c13c8f..efac6e4b 100644 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py +++ b/tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py @@ -1,15 +1,9 @@ -from .base import BaseRateLimitBackend, RateLimitDecision -from .factory import build_backend -from .memory import MemoryRateLimitBackend +from .limiter import RateLimitDecision, check from .middleware import RateLimitMiddleware -from .redis import RedisRateLimitBackend __all__ = [ - "BaseRateLimitBackend", - "MemoryRateLimitBackend", "RateLimitDecision", "RateLimitMiddleware", - "RedisRateLimitBackend", - "build_backend", + "check", ] diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/base.py b/tenxgraph_api/src/app/core/middleware/rate_limit/base.py deleted file mode 100644 index 0510a4d7..00000000 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/base.py +++ /dev/null @@ -1,46 +0,0 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass - - -# Retry-After (seconds) sent with the 503 answered while the backend cannot be reached. -UNAVAILABLE_RETRY_AFTER_SECONDS = 5 - - -@dataclass(frozen=True) -class RateLimitDecision: - """Result of a single rate-limit check. - - ``unavailable`` marks a denial caused by the backend (it could not be reached), not by the - limit: the middleware then answers 503 ``RATE_LIMIT_UNAVAILABLE`` instead of 429, with - ``reset_after`` as its ``Retry-After``. Backends that never set it keep working unchanged. - """ - - allowed: bool - remaining: int - reset_after: int - unavailable: bool = False - - @classmethod - def backend_unavailable( - cls, retry_after: int = UNAVAILABLE_RETRY_AFTER_SECONDS - ) -> "RateLimitDecision": - """A denial because the backend failed; answered 503 with ``Retry-After``.""" - return cls(allowed=False, remaining=0, reset_after=retry_after, unavailable=True) - - -class BaseRateLimitBackend(ABC): - """Abstract base class for rate-limit backends. - - Users can implement this class and bind an instance into InjectQ when they - need a backend other than the built-in memory or Redis implementations. A backend that - cannot reach its store and must not let the request through returns - :meth:`RateLimitDecision.backend_unavailable`. - """ - - @abstractmethod - async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: - """Atomically check and record a request for *key*.""" - - @abstractmethod - async def close(self) -> None: - """Release resources held by the backend, if any.""" diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/factory.py b/tenxgraph_api/src/app/core/middleware/rate_limit/factory.py deleted file mode 100644 index 0f09609c..00000000 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/factory.py +++ /dev/null @@ -1,101 +0,0 @@ -import logging -from typing import Any - -from injectq import InjectQ - -from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig - -from .base import BaseRateLimitBackend -from .memory import MemoryRateLimitBackend -from .redis import RedisRateLimitBackend - - -try: - # Needed at runtime: _bind_redis binds the client under its class too. - from redis.asyncio import Redis -except ImportError: # the redis extra is not installed - Redis = None # type: ignore[assignment,misc] - - -logger = logging.getLogger("tenxgraph_api.rate_limit") - - -def build_backend( - config: RateLimitConfig, - container: InjectQ | None = None, -) -> BaseRateLimitBackend: - """Build or resolve the configured rate-limit backend. - - Custom backends are provided by binding a ``BaseRateLimitBackend`` instance - into InjectQ, matching the style used by auth/authorization. - """ - container = container or InjectQ.get_instance() - - injected_backend = container.try_get(BaseRateLimitBackend) - if config.backend == "custom": - if not injected_backend: - raise ValueError( - "rate_limit.backend='custom' requires a BaseRateLimitBackend " - "instance bound in InjectQ" - ) - return injected_backend - - if config.backend == "redis": - return _build_redis_backend(config, container) - - logger.info("Rate-limit backend: memory (in-process, not shared across workers)") - return MemoryRateLimitBackend() - - -def _build_redis_backend( - config: RateLimitConfig, - container: InjectQ, -) -> RedisRateLimitBackend: - redis = _get_redis_from_container(container) - if redis is not None: - logger.info( - "Rate-limit backend: Redis from InjectQ (prefix=%s, fail_open=%s)", - config.redis_prefix, - config.fail_open, - ) - return RedisRateLimitBackend( - redis=redis, - prefix=config.redis_prefix, - fail_open=config.fail_open, - close_redis=False, - ) - - if not config.redis_url: - raise ValueError("rate_limit.redis.url is required when no Redis client is bound") - - backend = RedisRateLimitBackend.from_url( - redis_url=config.redis_url, - prefix=config.redis_prefix, - fail_open=config.fail_open, - ) - _bind_redis(container, backend._redis) - logger.info( - "Rate-limit backend: created Redis and bound it in InjectQ (prefix=%s, fail_open=%s)", - config.redis_prefix, - config.fail_open, - ) - return backend - - -def _get_redis_from_container(container: InjectQ) -> Any | None: - redis = container.try_get("redis") - if redis is not None: - return redis - - redis = container.try_get("redis_client") - if redis is not None: - return redis - - return container.try_get("Redis") - - -def _bind_redis(container: InjectQ, redis: "Redis") -> None: - if Redis is not None: - container.bind_instance(Redis, redis) - container.bind_instance("redis", redis) - container.bind_instance("redis_client", redis) diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/limiter.py b/tenxgraph_api/src/app/core/middleware/rate_limit/limiter.py new file mode 100644 index 00000000..fa946ca1 --- /dev/null +++ b/tenxgraph_api/src/app/core/middleware/rate_limit/limiter.py @@ -0,0 +1,134 @@ +"""Sliding-window rate limiting on the API's cache store. + +A request takes one slot of ``key`` that expires ``window`` seconds later and is never +released, so the live slots are exactly the requests of the last ``window`` seconds. A denied +request takes nothing (``BaseCacheStore.acquire`` adds no slot when it denies). With a shared +cache store (Redis) the window is shared by every worker; with the in-memory store each +process counts its own. +""" + +from __future__ import annotations + +import logging +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore + + +if TYPE_CHECKING: + from injectq import InjectQ + + +logger = logging.getLogger("tenxgraph_api.rate_limit") + +# Retry-After (seconds) sent with the 503 answered while the cache store cannot be reached. +UNAVAILABLE_RETRY_AFTER_SECONDS = 5 + + +@dataclass(frozen=True) +class RateLimitDecision: + """Result of a single rate-limit check. + + ``unavailable`` marks a denial caused by the cache store (it could not be reached), not by + the limit: the middleware then answers 503 ``RATE_LIMIT_UNAVAILABLE`` instead of 429, with + ``reset_after`` as its ``Retry-After``. + """ + + allowed: bool + remaining: int + reset_after: int + unavailable: bool = False + + @classmethod + def backend_unavailable( + cls, retry_after: int = UNAVAILABLE_RETRY_AFTER_SECONDS + ) -> RateLimitDecision: + """A denial because the cache store failed; answered 503 with ``Retry-After``.""" + return cls(allowed=False, remaining=0, reset_after=retry_after, unavailable=True) + + +# True while the cache store keeps failing: only the first error of an outage logs a traceback. +_failing = False + + +def store_key(key: str) -> str: + """The cache store key of a client's rate-limit window.""" + return f"ratelimit:{key}" + + +async def check( + cache: BaseCacheStore, key: str, limit: int, window: float, *, fail_open: bool = True +) -> RateLimitDecision: + """Count one request for ``key`` against ``limit`` per ``window`` seconds. + + On a cache store error the request is allowed (``fail_open``) or denied as unavailable + (answered 503, never the 429 of a real limit hit). + """ + global _failing # noqa: PLW0603 - one outage state per process + try: + result = await cache.acquire(store_key(key), limit, ttl=window) + if result.granted: + decision = RateLimitDecision( + allowed=True, + remaining=max(0, limit - result.count), + reset_after=math.ceil(window), + ) + else: + reset_after = math.ceil(window) + if result.oldest_expiry is not None: + reset_after = math.ceil(result.oldest_expiry - await cache.now()) + decision = RateLimitDecision( + allowed=False, remaining=0, reset_after=max(1, reset_after) + ) + except Exception as exc: + _log_error(exc, fail_open) + if fail_open: + return RateLimitDecision(allowed=True, remaining=0, reset_after=math.ceil(window)) + return RateLimitDecision.backend_unavailable() + if _failing: + _failing = False + logger.info("Rate-limit cache store recovered") + return decision + + +def _log_error(exc: Exception, fail_open: bool) -> None: + """Traceback for the first error of an outage; one line for each error after it.""" + global _failing # noqa: PLW0603 - one outage state per process + action = "allowing" if fail_open else "denying (503)" + if not _failing: + _failing = True + logger.exception("Rate-limit cache store error; %s requests until it recovers", action) + else: + logger.warning("Rate-limit cache store error (%s): %s: %s", action, type(exc).__name__, exc) + + +# Used when no cache store is bound and the caller has no fallback of its own: one per process. +_process_store: BaseCacheStore | None = None + + +def bound_cache_store( + container: InjectQ | None = None, fallback: BaseCacheStore | None = None +) -> BaseCacheStore: + """The cache store bound in ``container`` (default: the app's), else ``fallback``. + + Without a ``fallback`` an in-memory store of this process is used. The app binds its cache + store at startup, so the fallback only serves an app run without its lifespan. + """ + global _process_store # noqa: PLW0603 - one per process, made on first use + try: + if container is None: + from tenxgraph_api.src.app.core.utils.container import current_container + + container = current_container() + bound = container.try_get(BaseCacheStore) + except Exception: + bound = None + if isinstance(bound, BaseCacheStore): + return bound + if fallback is not None: + return fallback + if _process_store is None: + _process_store = InMemoryCacheStore() + return _process_store diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/memory.py b/tenxgraph_api/src/app/core/middleware/rate_limit/memory.py deleted file mode 100644 index c0d0f8c4..00000000 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/memory.py +++ /dev/null @@ -1,132 +0,0 @@ -import asyncio -import logging -import time -from collections import OrderedDict, deque - -from .base import BaseRateLimitBackend, RateLimitDecision - - -logger = logging.getLogger("tenxgraph_api.rate_limit") - -_NUM_STRIPES = 64 - - -class _Bucket: - """Sliding-window log for one key plus the time it was last used.""" - - __slots__ = ("last_seen", "times") - - def __init__(self, now: float) -> None: - self.times: deque[float] = deque() - self.last_seen = now - - -class MemoryRateLimitBackend(BaseRateLimitBackend): - """In-process sliding-window rate limiter. - - This is useful for development and single-process deployments. It is not - shared across workers or containers; use Redis for distributed enforcement. - - Keys are kept in least-recently-used order (oldest first). Every check moves its - key to the end, so evicting at the key cap and removing idle keys only touch the - front of the table and never scan it. - """ - - _SWEEP_INTERVAL = 2000 - _SWEEP_BATCH = 1000 - _CAP_WARNING_INTERVAL = 60.0 - - def __init__(self, max_unique_keys: int = 50_000) -> None: - self._buckets: OrderedDict[str, _Bucket] = OrderedDict() - self._locks = [asyncio.Lock() for _ in range(_NUM_STRIPES)] - self._sweep_lock = asyncio.Lock() - self._bg_tasks: set[asyncio.Task] = set() - self._check_count = 0 - self._max_unique_keys = max_unique_keys - self._last_cap_warning: float | None = None - - def _stripe_lock(self, key: str) -> asyncio.Lock: - return self._locks[hash(key) % _NUM_STRIPES] - - def _schedule_sweep(self, window: int) -> None: - task = asyncio.ensure_future(self._sweep(window)) - self._bg_tasks.add(task) - task.add_done_callback(self._bg_tasks.discard) - - def _pop_stale(self, cutoff: float, limit: int) -> int: - """Remove up to ``limit`` idle keys from the front; stop at the first fresh one.""" - removed = 0 - while self._buckets and removed < limit: - oldest = next(iter(self._buckets.values())) - if oldest.last_seen >= cutoff: - break - self._buckets.popitem(last=False) - removed += 1 - return removed - - def _warn_cap_reached(self, now: float) -> None: - last = self._last_cap_warning - if last is not None and now - last < self._CAP_WARNING_INTERVAL: - return - self._last_cap_warning = now - logger.warning( - "Rate-limit bucket cap (%d) reached; evicting least recently used keys", - self._max_unique_keys, - ) - - async def _sweep(self, window: int) -> None: - async with self._sweep_lock: - cutoff = time.monotonic() - window * 2 - total = 0 - while True: - # Each batch is synchronous, so the table is consistent at every await. - removed = self._pop_stale(cutoff, self._SWEEP_BATCH) - total += removed - if removed < self._SWEEP_BATCH: - break - await asyncio.sleep(0) - if total: - logger.debug("Rate-limit memory sweep removed %d stale buckets", total) - - async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: - self._check_count += 1 - if self._check_count % self._SWEEP_INTERVAL == 0: - self._schedule_sweep(window) - - async with self._stripe_lock(key): - # No await below: the table is only ever mutated in whole, atomic steps. - now = time.monotonic() - window_start = now - window - - entry = self._buckets.get(key) - if entry is None: - if len(self._buckets) >= self._max_unique_keys: - self._warn_cap_reached(now) - self._buckets.popitem(last=False) - entry = self._buckets[key] = _Bucket(now) - else: - self._buckets.move_to_end(key) - entry.last_seen = now - bucket = entry.times - - while bucket and bucket[0] < window_start: - bucket.popleft() - - count = len(bucket) - if count >= limit: - reset_after = max(1, int(bucket[0] + window - now) + 1) - return RateLimitDecision(allowed=False, remaining=0, reset_after=reset_after) - - bucket.append(now) - return RateLimitDecision( - allowed=True, - remaining=limit - len(bucket), - reset_after=window, - ) - - async def close(self) -> None: - for task in self._bg_tasks: - task.cancel() - if self._bg_tasks: - await asyncio.gather(*self._bg_tasks, return_exceptions=True) - self._bg_tasks.clear() diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py b/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py index f687e727..5a34b9b6 100644 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py +++ b/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py @@ -1,4 +1,5 @@ import time +from typing import TYPE_CHECKING from fastapi import status from fastapi.responses import JSONResponse @@ -7,37 +8,54 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from tenxgraph_api.src.app.core import logger +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe -from .base import BaseRateLimitBackend, RateLimitDecision from .keying import client_key_for +from .limiter import RateLimitDecision, bound_cache_store, check -# Liveness/readiness probes hit /ping. Limiting it (or failing closed on a backend outage) +if TYPE_CHECKING: + from injectq import InjectQ + + +# Liveness/readiness probes hit /ping. Limiting it (or failing closed on a cache store outage) # would make the orchestrator restart healthy pods. HEALTH_CHECK_PATHS = frozenset({"/ping", "/health/ready"}) class RateLimitMiddleware: - """Backend-agnostic rate-limit middleware. + """Rate-limit middleware over the API's cache store (see ``limiter.py``). - Pure ASGI: it answers 429 (limit hit) or 503 (backend unreachable) itself, or edits the - ``http.response.start`` message on the way out, so it adds no per-request task and never - buffers a streaming body. + Pure ASGI: it answers 429 (limit hit) or 503 (cache store unreachable, with ``fail_open`` + off) itself, or edits the ``http.response.start`` message on the way out, so it adds no + per-request task and never buffers a streaming body. + + ``cache`` pins the store; without it each request uses the ``BaseCacheStore`` bound in + ``container`` (bound at startup, after the middleware is added), or a store of this + middleware's own when none is bound. """ def __init__( self, app: ASGIApp, config: RateLimitConfig, - backend: BaseRateLimitBackend, + cache: BaseCacheStore | None = None, + container: "InjectQ | None" = None, ) -> None: self.app = app self.config = config - self.backend = backend + self.cache = cache + self.container = container + self._fallback: BaseCacheStore | None = None self._exclude = frozenset(config.exclude_paths) | HEALTH_CHECK_PATHS + def _bound_cache(self) -> BaseCacheStore: + if self._fallback is None: + self._fallback = InMemoryCacheStore() + return bound_cache_store(self.container, self._fallback) + def _client_key(self, request: Request) -> str: return client_key_for(request, self.config) @@ -48,16 +66,19 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: request = Request(scope) key = self._client_key(request) - decision = await self.backend.check( + cache = self.cache if self.cache is not None else self._bound_cache() + decision = await check( + cache, key, - limit=self.config.requests, - window=self.config.window, + self.config.requests, + self.config.window, + fail_open=self.config.fail_open, ) reset_at_epoch = int(time.time()) + decision.reset_after if not decision.allowed and decision.unavailable: - # The backend failed and fails closed: an outage, not the client's fault. The - # backend logs the error itself, so this does not log per request. + # The cache store failed and fail_open is off: an outage, not the client's fault. + # check() logs the error itself, so this does not log per request. await self._unavailable(decision, scope)(scope, receive, send) return diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py b/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py deleted file mode 100644 index 78a90b0a..00000000 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py +++ /dev/null @@ -1,146 +0,0 @@ -import logging -import time -import uuid -from typing import Any - -from .base import BaseRateLimitBackend, RateLimitDecision - - -try: - from redis.asyncio import Redis as AsyncRedis # type: ignore[import] - - _REDIS_AVAILABLE = True -except ImportError: - _REDIS_AVAILABLE = False - AsyncRedis = None # type: ignore[assignment,misc] - - -logger = logging.getLogger("tenxgraph_api.rate_limit") -_REDIS_EXTRA_INSTALL = 'pip install "10xgraph-api[redis]"' - -# Atomic sliding-window check using a Redis sorted set. -# -# The script: -# 1. Removes timestamps older than the current window. -# 2. Counts the remaining requests for the key. -# 3. If under the limit, adds the current request as a unique sorted-set member. -# 4. Sets an expiry so idle keys clean themselves up. -# 5. Returns whether the request is allowed, remaining quota, and reset time. -_SLIDING_WINDOW_LUA = """ -local key = KEYS[1] -local now_ms = tonumber(ARGV[1]) -local window_ms = tonumber(ARGV[2]) -local limit = tonumber(ARGV[3]) -local member = ARGV[4] - -local window_start = now_ms - window_ms - -redis.call('ZREMRANGEBYSCORE', key, '-inf', tostring(window_start)) - -local count = tonumber(redis.call('ZCARD', key)) - -if count < limit then - redis.call('ZADD', key, tostring(now_ms), member) - redis.call('EXPIRE', key, math.ceil(window_ms / 1000) + 1) - return {1, limit - count - 1, math.ceil(window_ms / 1000)} -else - local oldest = redis.call('ZRANGE', key, 0, 0, 'WITHSCORES') - local reset_after - if #oldest >= 2 then - local oldest_ms = tonumber(oldest[2]) - reset_after = math.ceil((oldest_ms + window_ms - now_ms) / 1000) + 1 - else - reset_after = math.ceil(window_ms / 1000) - end - return {0, 0, math.max(reset_after, 1)} -end -""" - - -class RedisRateLimitBackend(BaseRateLimitBackend): - """Distributed sliding-window rate limiter backed by Redis. - - On a Redis error it allows the request (``fail_open``) or denies it as unavailable, which - the middleware answers with 503, never with the 429 of a real limit hit. - """ - - # The URL the client was built from (``from_url``); None for a client passed in. The - # startup check pings it. - redis_url: str | None = None - # True while Redis keeps failing: only the first error of an outage logs a traceback. - _failing = False - - def __init__( - self, - redis: Any, - prefix: str, - fail_open: bool = True, - close_redis: bool = False, - ) -> None: - self._redis = redis - self._prefix = prefix - self._fail_open = fail_open - self._close_redis = close_redis - self._script = self._redis.register_script(_SLIDING_WINDOW_LUA) - - @classmethod - def from_url( - cls, - redis_url: str, - prefix: str, - fail_open: bool = True, - ) -> "RedisRateLimitBackend": - if not _REDIS_AVAILABLE or AsyncRedis is None: - raise ImportError( - "Redis backend requires the 'redis' package. " - f"Install the optional Redis extra with: {_REDIS_EXTRA_INSTALL}" - ) - from tenxgraph_api.src.app.core.redis_client import create_redis - - redis = create_redis(redis_url) - backend = cls(redis=redis, prefix=prefix, fail_open=fail_open, close_redis=True) - backend.redis_url = redis_url - return backend - - async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: - redis_key = f"{self._prefix}:{key}" - now_ms = int(time.time() * 1000) - window_ms = window * 1000 - member = f"{now_ms}:{uuid.uuid4().hex}" - - try: - result = await self._script( - keys=[redis_key], - args=[now_ms, window_ms, limit, member], - ) - decision = RateLimitDecision( - allowed=bool(result[0]), - remaining=int(result[1]), - reset_after=int(result[2]), - ) - except Exception as exc: - self._log_error(exc) - if self._fail_open: - return RateLimitDecision(allowed=True, remaining=0, reset_after=window) - return RateLimitDecision.backend_unavailable() - if self._failing: - self._failing = False - logger.info("Redis rate-limit backend recovered") - return decision - - def _log_error(self, exc: Exception) -> None: - """Traceback for the first error of an outage; one line for each error after it.""" - action = "allowing" if self._fail_open else "denying (503)" - if not self._failing: - self._failing = True - logger.exception( - "Redis rate-limit backend error; %s requests until it recovers", action - ) - else: - logger.warning( - "Redis rate-limit backend error (%s): %s: %s", action, type(exc).__name__, exc - ) - - async def close(self) -> None: - if self._close_redis: - await self._redis.aclose() diff --git a/tenxgraph_api/src/app/loader.py b/tenxgraph_api/src/app/loader.py index 744a93e1..6b1e0080 100644 --- a/tenxgraph_api/src/app/loader.py +++ b/tenxgraph_api/src/app/loader.py @@ -18,6 +18,8 @@ RoleBasedAuthorizationBackend, assert_ownership_supported, ) +from tenxgraph_api.src.app.core.cache_store.base import BaseCacheStore +from tenxgraph_api.src.app.core.cache_store.loader import load_and_bind_cache_store from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.utils.thread_name_generator import ThreadNameGenerator @@ -132,6 +134,37 @@ def load_store(path: str | None) -> BaseStore | None: return store +def load_cache_store(path: str) -> BaseCacheStore: + """Load the ``cache_store`` from ``"module:attr"``: an instance or a zero-arg factory.""" + if ":" not in path: + raise ValueError(f"Invalid cache_store path format '{path}'. Expected 'module:attribute'.") + + module_name_importable, function_name = path.split(":") + + try: + module = importlib.import_module(module_name_importable) + entry_point_obj = getattr(module, function_name) + cache_store = entry_point_obj + if not isinstance(cache_store, BaseCacheStore) and callable(cache_store): + cache_store = cache_store() + + if isinstance(cache_store, BaseCacheStore): + logger.info(f"Successfully loaded BaseCacheStore '{function_name}' from {path}.") + else: + raise TypeError("Loaded object is not a BaseCacheStore or a factory returning one.") + except ModuleNotFoundError as e: + logger.error(f"Module not found when loading BaseCacheStore from {path}: {e}") + raise ModuleNotFoundError(f"Module not found for cache_store path '{path}': {e}") + except AttributeError as e: + logger.error(f"Attribute not found when loading BaseCacheStore from {path}: {e}") + raise AttributeError(f"Attribute not found for cache_store path '{path}': {e}") + except Exception as e: + logger.error(f"Error loading BaseCacheStore from {path}: {e}") + raise Exception(f"Failed to load BaseCacheStore from {path}: {e}") + + return cache_store + + def load_container(path: str | None) -> InjectQ | None: if not path: return None @@ -316,36 +349,9 @@ def _load_custom_auth(path: str | None) -> BaseAuth: } -def _build_ownership_redis(redis_url: str | None) -> object | None: - """Build an async Redis client for the ownership L2 cache, or None. - - Returns None (L1-only) when no URL is configured or the ``redis`` package is absent; - the resolver degrades gracefully either way. - """ - if not redis_url: - return None - try: - import redis.asyncio # type: ignore[import] # noqa: F401 - except ImportError: - logger.warning( - "REDIS configured but the 'redis' package is not installed; ownership cache " - "runs in-process only (L1). Install the redis extra to share it across workers." - ) - return None - try: - from tenxgraph_api.src.app.core.redis_client import create_redis - - client = create_redis(redis_url) - logger.info("Ownership authorization L2 cache enabled (shared Redis).") - return client - except Exception as exc: # pragma: no cover - defensive - logger.warning("Failed to build ownership Redis client: %s; using L1 cache only.", exc) - return None - - def _resolve_authorization_backend( authorization: str | dict | None, - redis_url: str | None = None, + cache: BaseCacheStore | None = None, server_thread_ids: bool = False, auth_enabled: bool = False, ) -> AuthorizationBackend: @@ -367,7 +373,8 @@ def _resolve_authorization_backend( ``MODE=development`` and auth on ran ``allow_all``: the API stamped isolation scope ``none`` and every authenticated user could read every thread and memory. - ``redis_url`` (when set) backs the ownership cache's shared L2 tier. + ``cache`` (the API's cache store) backs the ownership cache's shared L2 tier; the + built-in backends use it and never close it. ``server_thread_ids`` (``runs.thread_ids == "server"``) makes the built-in ownership backends refuse client-named ids for threads that do not exist yet. """ @@ -380,7 +387,7 @@ def _resolve_authorization_backend( role_scopes=authorization.get("roles") or authorization.get("role_scopes") or {}, default_scopes=authorization.get("default_scopes") or (), isolation=authorization.get("isolation", "owner"), - redis=_build_ownership_redis(redis_url), + cache=cache, server_thread_ids=server_thread_ids, ) raise ValueError( @@ -410,9 +417,7 @@ def _resolve_authorization_backend( key, ) if builtin is OwnershipAuthorizationBackend: - return OwnershipAuthorizationBackend( - redis=_build_ownership_redis(redis_url), server_thread_ids=server_thread_ids - ) + return OwnershipAuthorizationBackend(cache=cache, server_thread_ids=server_thread_ids) return builtin() # Not configured: owner-only whenever there is an identity to own threads. @@ -422,9 +427,7 @@ def _resolve_authorization_backend( "(owner-only thread access). Set " '"authorization": "allow_all" to opt out.' ) - return OwnershipAuthorizationBackend( - redis=_build_ownership_redis(redis_url), server_thread_ids=server_thread_ids - ) + return OwnershipAuthorizationBackend(cache=cache, server_thread_ids=server_thread_ids) logger.info( "No authorization configured and auth is off; using DefaultAuthorizationBackend " @@ -442,12 +445,15 @@ def _is_production() -> bool: def load_and_bind_authorization( container: InjectQ, authorization_path: str | None, - redis_url: str | None = None, + cache: BaseCacheStore | None = None, server_thread_ids: bool = False, auth_enabled: bool = False, ) -> None: + """Bind the authorization backend; ``cache`` defaults to the bound ``BaseCacheStore``.""" + if cache is None: + cache = container.try_get(BaseCacheStore) backend = _resolve_authorization_backend( - authorization_path, redis_url, server_thread_ids, auth_enabled + authorization_path, cache, server_thread_ids, auth_enabled ) if server_thread_ids and not isinstance(backend, OwnershipAuthorizationBackend): logger.debug( @@ -458,54 +464,34 @@ def load_and_bind_authorization( container.bind_instance(AuthorizationBackend, backend) -def _bind_token_revocation( - container: InjectQ, config: GraphConfig, auth_enabled: bool, redis_url: str | None -) -> None: - """Bind the ``token_revocation`` backend from 10xgraph.json, if one is configured. +def _enable_token_revocation(container: InjectQ, config: GraphConfig, auth_enabled: bool) -> None: + """Bind the ``token_revocation`` settings from 10xgraph.json, if it is on. - A backend the developer already bound in their InjectQ container is left alone. + The checks read the cache store bound by ``load_and_bind_cache_store``, so call this after + it. With an in-memory cache store a revocation reaches only this process: warned about in + production or with more than one worker. """ - from tenxgraph_api.src.app.core.auth.revocation import ( - TokenRevocationBackend, - build_token_revocation, - ) + from tenxgraph_api.src.app.core.auth.revocation import TokenRevocationConfig + from tenxgraph_api.src.app.core.cache_store.loader import is_in_memory - spec = config.token_revocation - if not isinstance(spec, str) or not spec: + settings = config.token_revocation + if settings is None: return if not auth_enabled: - logger.warning("token_revocation is set but auth is off; there are no tokens to revoke.") - return - if container.try_get(TokenRevocationBackend) is not None: - logger.info("Using the TokenRevocationBackend bound in the InjectQ container.") - return - container.bind_instance(TokenRevocationBackend, build_token_revocation(spec, redis_url)) - logger.info("Token revocation enabled (%s).", spec) - - -def _bind_shared_concurrency(container: InjectQ, redis_url: str | None) -> None: - """Share the run and WebSocket connection caps across workers when Redis is configured. - - Without Redis each process counts its own slots, so a cap becomes ``limit x workers``. A - ``ConcurrencyLimiter`` the developer already bound is left alone. - """ - from tenxgraph_api.src.app.core.concurrency import ( - ConcurrencyLimiter, - RedisConcurrencyLimiter, - ) - - if not redis_url or container.try_get(ConcurrencyLimiter) is not None: + logger.warning("token_revocation is on but auth is off; there are no tokens to revoke.") return - try: - limiter = RedisConcurrencyLimiter.from_url(redis_url) - except ImportError: + container.bind_instance(TokenRevocationConfig, settings) + store = container.try_get(BaseCacheStore) + logger.info("Token revocation enabled (cache store: %s).", type(store).__name__) + workers = _web_concurrency() + if is_in_memory(store) and (_is_production() or workers > 1): logger.warning( - "REDIS configured but the 'redis' package is not installed; run and WebSocket " - "caps are counted per process." + "token_revocation is on with an in-memory cache store (workers=%d): a revocation " + "reaches only the process that recorded it, and `10xgraph revoke` cannot reach it " + "at all. Set `redis` in 10xgraph.json (or REDIS_URL, or a `cache_store`) to share " + "revocations between workers.", + workers, ) - return - container.bind_instance(ConcurrencyLimiter, limiter) - logger.info("Run and WebSocket connection caps are shared through Redis.") def _web_concurrency() -> int: @@ -516,11 +502,12 @@ def _web_concurrency() -> int: return 1 -def warn_per_process_state(container: InjectQ, redis_url: str | None) -> None: +def warn_per_process_state(container: InjectQ) -> None: """Warn when threads or caps live in one process and the deployment may run several. The in-memory and SQLite checkpointers keep threads in one process (or one file), and - without Redis the run locks, run caps and rate-limit counters are per process too. That is + with an in-memory cache store the run locks, run and WebSocket caps and idempotency + records are per process too. That is fine for one process; with more than one worker or replica every process has its own view. Warns in production or when ``WEB_CONCURRENCY`` is above 1 (the API cannot see replicas, so production always gets the line). The choice stays the developer's. @@ -528,6 +515,8 @@ def warn_per_process_state(container: InjectQ, redis_url: str | None) -> None: workers = _web_concurrency() if not (_is_production() or workers > 1): return + from tenxgraph_api.src.app.core.cache_store.loader import is_in_memory + checkpointer = container.try_get(BaseCheckpointer) kind = type(checkpointer).__name__ if kind in {"InMemoryCheckpointer", "SqliteCheckpointer"}: @@ -538,12 +527,13 @@ def warn_per_process_state(container: InjectQ, redis_url: str | None) -> None: kind, workers, ) - if not redis_url: + if is_in_memory(container.try_get(BaseCacheStore)): logger.warning( - "No Redis configured: the per-thread run lock, run and WebSocket caps and " - "in-memory rate limits are counted per process (workers=%d). With more than one " - "worker or replica each limit multiplies and two processes can run the same thread " - "at once. Set `redis` in 10xgraph.json for a shared view.", + "The cache store is in-memory: the per-thread run lock, run and WebSocket caps, " + "idempotency records and in-memory rate limits are counted per process " + "(workers=%d). With more than one worker or replica each limit multiplies and two " + "processes can run the same thread at once. Set `redis` in 10xgraph.json (or " + "REDIS_URL, or a `cache_store`) for a shared view.", workers, ) @@ -627,16 +617,17 @@ async def attach_all_modules( # bind None if not configured container.bind_instance(ThreadNameGenerator, None, allow_none=True) - # load authorization backend. The ownership cache's shared L2 tier reuses the - # configured Redis URL (config `redis`, else settings.REDIS_URL); absent -> L1 only. - from tenxgraph_api.src.app.core.config.settings import get_settings + # The cache store first: the ownership cache, token revocation and the rate limiter read + # it from the container. + load_and_bind_cache_store(container, config, get_settings()) + # load authorization backend. The ownership cache's shared L2 tier is the cache store + # (L1 only when it is in-memory). authorization_path = config.authorization_path - ownership_redis_url = config.redis_url or get_settings().REDIS_URL load_and_bind_authorization( container, authorization_path, - ownership_redis_url, + container.get(BaseCacheStore), server_thread_ids=config.runs.server_thread_ids, auth_enabled=bool(auth_config), ) @@ -647,9 +638,8 @@ async def attach_all_modules( container.get(AuthorizationBackend), container.try_get(BaseCheckpointer) ) - _bind_token_revocation(container, config, bool(auth_config), ownership_redis_url) - _bind_shared_concurrency(container, ownership_redis_url) - warn_per_process_state(container, ownership_redis_url) + _enable_token_revocation(container, config, bool(auth_config)) + warn_per_process_state(container) # --- Media service wiring --- from tenxgraph_api.src.app.core.config.media_settings import ( diff --git a/tenxgraph_api/src/app/main.py b/tenxgraph_api/src/app/main.py index f3b69c70..c8e183e8 100644 --- a/tenxgraph_api/src/app/main.py +++ b/tenxgraph_api/src/app/main.py @@ -106,10 +106,12 @@ async def _prepare_checkpointer() -> None: async def _check_backends(app: FastAPI) -> None: - """Refuse to start while a configured backend (Postgres, Redis, store) is unreachable. + """Refuse to start while the checkpointer, the store or a required cache store is down. - Retries for ``STARTUP_CHECK_TIMEOUT_SECONDS``, then raises ``StartupCheckError``, which - stops uvicorn and gunicorn with the failed components in the log. Off with + Calls ``aping()`` on each, retrying for ``STARTUP_CHECK_TIMEOUT_SECONDS``, then raises + ``StartupCheckError``, which stops uvicorn and gunicorn with the failed components in the + log. The cache store is required only by fail-closed features (token revocation, a rate + limiter with ``fail_open: false``); otherwise an outage is one warning. Off with ``STARTUP_CHECKS=false``. """ if not settings.STARTUP_CHECKS: @@ -120,11 +122,29 @@ async def _check_backends(app: FastAPI) -> None: return from tenxgraph_api.src.app.routers.ping.startup_checks import check_backends_at_startup - await check_backends_at_startup( - container, - timeout=settings.STARTUP_CHECK_TIMEOUT_SECONDS, - rate_limit_backend=getattr(app.state, "rate_limit_backend", None), - ) + await check_backends_at_startup(container, timeout=settings.STARTUP_CHECK_TIMEOUT_SECONDS) + + +async def _close_cache_store() -> None: + """Give back the leases whose releases are still running, then close the cache store. + + A release that can no longer reach the store leaves its slot (a run lock, a cap) held + until the lease ttl, so the next pod would answer 409 for that thread meanwhile. + """ + try: + from tenxgraph_api.src.app.core.cache_store import BaseCacheStore + from tenxgraph_api.src.app.core.leases import LeaseManager, wait_for_releases + + leases = container.try_get(LeaseManager) + if leases is not None: + await leases.aclose() + else: + await wait_for_releases() + cache_store = container.try_get(BaseCacheStore) + if cache_store is not None: + await cache_store.aclose() + except Exception as exc: + logger.debug("Cache store close failed: %s", exc) async def _cleanup_temp_media_cache() -> None: @@ -211,32 +231,11 @@ async def lifespan(app: FastAPI): except Exception as exc: logger.warning("Graph close did not finish cleanly: %s", exc) - # Close rate-limit backend (e.g. Redis connection pool) - backend = getattr(app.state, "rate_limit_backend", None) - if backend is not None: - await backend.close() - - # Close the shared concurrency limiter (its Redis client), if one is bound. - try: - from tenxgraph_api.src.app.core.concurrency import ConcurrencyLimiter - - limiter = container.try_get(ConcurrencyLimiter) - if limiter is not None: - await limiter.aclose() - except Exception as exc: - logger.debug("Concurrency limiter close failed: %s", exc) - - # Close the token revocation backend (its Redis client), if one is bound. - try: - from tenxgraph_api.src.app.core.auth.revocation import TokenRevocationBackend - - revocation = container.try_get(TokenRevocationBackend) - if revocation is not None: - await revocation.aclose() - except Exception as exc: - logger.debug("Token revocation backend close failed: %s", exc) + # The rate limiter, ownership cache and token revocation share the cache store: closed + # once here. + await _close_cache_store() - # Close the authorization backend's L2 Redis client, if it created one. + # A custom authorization backend may hold its own clients. try: from tenxgraph_api.src.app.core.auth.authorization import AuthorizationBackend diff --git a/tenxgraph_api/src/app/routers/graph/idempotency.py b/tenxgraph_api/src/app/routers/graph/idempotency.py index 0b797190..e13a7b7b 100644 --- a/tenxgraph_api/src/app/routers/graph/idempotency.py +++ b/tenxgraph_api/src/app/routers/graph/idempotency.py @@ -12,10 +12,11 @@ - the same key with a different request body: 422; - a run that failed stores nothing, so the same key may be retried. -No header, no change. Records live in Redis when it is configured (the client of the shared -concurrency limiter, keys under ``REDIS_KEY_PREFIX``), else in this process. The running state -is the same kind of lease the thread lock uses, so a crashed worker's key frees itself. A Redis -error is logged and the request runs without idempotency, as the concurrency limiter fails open. +No header, no change. Records live in the cache store (``idempotency::`` with a +ttl of ``IDEMPOTENCY_TTL_SECONDS``): shared between workers with Redis, else in this process. +The running state is the same kind of lease the thread lock uses (``runs:idem::``), +so a crashed worker's key frees itself. A cache store error is logged and the request runs +without idempotency, as the leases fail open. """ from __future__ import annotations @@ -23,7 +24,6 @@ import hashlib import json import re -import time import weakref from collections.abc import AsyncIterator from dataclasses import dataclass @@ -33,13 +33,9 @@ from tenxgraph_api.src.app.core import logger from tenxgraph_api.src.app.core.auth.request_config import ClientInputError -from tenxgraph_api.src.app.core.concurrency import ( - Lease, - RedisConcurrencyLimiter, - concurrency_limiter, -) from tenxgraph_api.src.app.core.config.settings import get_settings -from tenxgraph_api.src.app.core.redis_client import key_part, prefixed +from tenxgraph_api.src.app.core.leases import Lease, lease_manager +from tenxgraph_api.src.app.core.redis_client import key_part IDEMPOTENCY_HEADER = "Idempotency-Key" @@ -47,7 +43,6 @@ RETRY_AFTER_SECONDS = 5 # A result larger than this is not stored; the key then answers 409 instead of replaying. MAX_STORED_BYTES = 1024 * 1024 -MAX_MEMORY_RECORDS = 10_000 HTTP_CONFLICT = 409 HTTP_UNPROCESSABLE = 422 @@ -88,34 +83,12 @@ def __init__(self) -> None: ) -_memory: dict[str, tuple[float, str]] = {} - - -def _redis_client() -> Any | None: +async def _load(name: str) -> dict[str, Any] | None: try: - from tenxgraph_api.src.app.core.concurrency import shared_concurrency_limiter - - limiter = shared_concurrency_limiter() - except Exception: + raw = await lease_manager().cache.get(name) + except Exception as exc: + logger.warning("Idempotency lookup failed; running without it: %s", exc) return None - return limiter.client if isinstance(limiter, RedisConcurrencyLimiter) else None - - -async def _load(name: str) -> dict[str, Any] | None: - client = _redis_client() - raw: Any = None - if client is not None: - try: - raw = await client.get(name) - except Exception as exc: - logger.warning("Idempotency lookup failed; running without it: %s", exc) - return None - else: - entry = _memory.get(name) - if entry and entry[0] > time.monotonic(): - raw = entry[1] - elif entry: - _memory.pop(name, None) if raw is None: return None try: @@ -128,20 +101,10 @@ async def _load(name: str) -> dict[str, Any] | None: async def _save(name: str, record: dict[str, Any]) -> None: text = json.dumps(record, separators=(",", ":"), default=str) ttl = get_settings().IDEMPOTENCY_TTL_SECONDS - client = _redis_client() - if client is not None: - try: - await client.set(name, text, ex=ttl) - except Exception as exc: - logger.warning("Idempotency record could not be stored: %s", exc) - return - now = time.monotonic() - if len(_memory) >= MAX_MEMORY_RECORDS: - for stale in [k for k, (exp, _) in _memory.items() if exp <= now]: - _memory.pop(stale, None) - while len(_memory) >= MAX_MEMORY_RECORDS: # still full: drop the oldest - _memory.pop(next(iter(_memory))) - _memory[name] = (now + ttl, text) + try: + await lease_manager().cache.set(name, text.encode(), ttl=ttl) + except Exception as exc: + logger.warning("Idempotency record could not be stored: %s", exc) @dataclass @@ -181,7 +144,7 @@ async def check( raise ClientInputError(problem) user_id = user.get("user_id") if isinstance(user, dict) else None ident = f"{key_part(user_id or 'anonymous')}:{key_part(key)}" - name = f"{prefixed('10xgraph:idempotency')}:{ident}" + name = f"idempotency:{ident}" fp = fingerprint(kind, body) def _answer(record: dict[str, Any]) -> tuple[None, dict[str, Any]]: @@ -191,7 +154,7 @@ def _answer(record: dict[str, Any]) -> tuple[None, dict[str, Any]]: if (record := await _load(name)) is not None: return _answer(record) - lease = await concurrency_limiter().acquire(f"runs:idem:{ident}", 1) + lease = await lease_manager().acquire(f"runs:idem:{ident}", 1) if lease is None: raise IdempotencyConflict("a request with this Idempotency-Key is still running") if (record := await _load(name)) is not None: # finished while we took the lease diff --git a/tenxgraph_api/src/app/routers/graph/realtime_guard.py b/tenxgraph_api/src/app/routers/graph/realtime_guard.py index 179e07c4..be3c01e0 100644 --- a/tenxgraph_api/src/app/routers/graph/realtime_guard.py +++ b/tenxgraph_api/src/app/routers/graph/realtime_guard.py @@ -4,11 +4,12 @@ bypass them entirely. This module re-applies two protections at the handshake, as a FastAPI dependency: -1. The same global rate limit as REST (shared backend + bucket), so opening a socket counts - like any other request. -2. A per-process cap on concurrent WebSocket connections (``websocket.max_connections`` in +1. The same global rate limit as REST (same cache store and bucket), so opening a socket + counts like any other request. +2. A cap on concurrent WebSocket connections (``websocket.max_connections`` in 10xgraph.json), and a per-user cap (``websocket.max_connections_per_user``) so one - account cannot hold every slot. + account cannot hold every slot. Both are leases (``core.leases``): shared across workers + when the cache store is shared (Redis), per process otherwise. A socket lives much longer than its handshake, so two helpers apply for its whole lifetime: :func:`ws_run_allowed` counts every graph run against the same rate-limit bucket, and @@ -28,14 +29,11 @@ from injectq.integrations import InjectAPI from tenxgraph_api.src.app.core import logger -from tenxgraph_api.src.app.core.concurrency import ( - ConcurrencyLimiter, - Lease, - shared_concurrency_limiter, -) from tenxgraph_api.src.app.core.config.graph_config import GraphConfig +from tenxgraph_api.src.app.core.leases import Lease, LeaseManager, lease_manager from tenxgraph_api.src.app.core.middleware.rate_limit import keying from tenxgraph_api.src.app.core.middleware.rate_limit.keying import client_key_for +from tenxgraph_api.src.app.core.middleware.rate_limit.limiter import bound_cache_store, check from tenxgraph_api.src.app.core.redis_client import key_part from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe @@ -44,117 +42,40 @@ WS_TRY_AGAIN_LATER = 1013 -class _ConnectionRegistry: - """Per-process counter of active WebSocket connections. - - The event loop is single-threaded, so the check-and-increment in :meth:`try_acquire` is - atomic (no ``await`` between the test and the mutation) and needs no lock. This counter is - per process, matching the in-memory rate-limit backend's scope; for a multi-worker - deployment, set ``max_connections`` per worker accordingly. - """ - - def __init__(self) -> None: - self._active = 0 - self._per_user: dict[str, int] = {} - - @property - def active(self) -> int: - return self._active - - def try_acquire(self, max_connections: int | None) -> bool: - if max_connections is not None and self._active >= max_connections: - return False - self._active += 1 - return True - - def release(self) -> None: - if self._active > 0: - self._active -= 1 - - def try_acquire_user(self, user_key: str, max_per_user: int | None) -> bool: - held = self._per_user.get(user_key, 0) - if max_per_user is not None and held >= max_per_user: - return False - self._per_user[user_key] = held + 1 - return True - - def release_user(self, user_key: str) -> None: - held = self._per_user.get(user_key, 0) - if held <= 1: - self._per_user.pop(user_key, None) - else: - self._per_user[user_key] = held - 1 - - -# Module-level singleton (per process). -_registry = _ConnectionRegistry() - - async def realtime_connection_guard( websocket: WebSocket, config: GraphConfig = InjectAPI(GraphConfig), ) -> AsyncIterator[None]: """Gate a WebSocket handshake on the global rate limit and the concurrent-connection cap.""" - # 1) Shared global rate limit (same backend/bucket as the REST middleware). + # 1) Shared global rate limit (same cache store and bucket as the REST middleware). rl_config = config.rate_limit - backend = getattr(getattr(websocket, "app", None), "state", None) - backend = getattr(backend, "rate_limit_backend", None) - if rl_config is not None and backend is not None: + if rl_config is not None: + cache = bound_cache_store() # Kept on the connection so every run on this socket counts too (ws_run_allowed). - websocket.state.ws_rate_limit = (rl_config, backend) + websocket.state.ws_rate_limit = (rl_config, cache) key = client_key_for(websocket, rl_config) - decision = await backend.check(key, limit=rl_config.requests, window=rl_config.window) + decision = await check( + cache, key, rl_config.requests, rl_config.window, fail_open=rl_config.fail_open + ) if not decision.allowed: logger.warning("WebSocket rate limit exceeded for %s", log_safe(key)) raise WebSocketException(code=WS_TRY_AGAIN_LATER, reason="Rate limit exceeded") - # 2) Concurrent-connection cap, then the per-user share of it. Shared across workers when a - # ConcurrencyLimiter is bound (Redis configured); otherwise counted in this process. - ws_config = config.websocket - shared = shared_concurrency_limiter() - if shared is not None: - async with _shared_connection_slots(shared, websocket, ws_config): - yield - return - - max_conn = ws_config.max_connections - if not _registry.try_acquire(max_conn): - logger.warning( - "WebSocket connection limit reached (active=%d, max=%s)", - _registry.active, - max_conn, - ) - raise WebSocketException(code=WS_TRY_AGAIN_LATER, reason="Too many connections") - - user_key = keying.verified_user_id(websocket) - if user_key and not _registry.try_acquire_user(user_key, ws_config.max_connections_per_user): - _registry.release() - logger.warning( - "WebSocket per-user connection limit reached for %s (max=%s)", - log_safe(user_key), - ws_config.max_connections_per_user, - ) - raise WebSocketException( - code=WS_TRY_AGAIN_LATER, reason="Too many connections for this user" - ) - - try: + # 2) Concurrent-connection cap, then the per-user share of it. Leases on the cache store: + # shared across workers when it is shared (Redis), otherwise counted in this process. + async with _connection_slots(lease_manager(), websocket, config.websocket): yield - finally: - _registry.release() - if user_key: - _registry.release_user(user_key) @asynccontextmanager -async def _shared_connection_slots( - limiter: ConcurrencyLimiter, websocket: WebSocket, ws_config: Any +async def _connection_slots( + manager: LeaseManager, websocket: WebSocket, ws_config: Any ) -> AsyncIterator[None]: - """Hold the global and per-user connection slots from a shared limiter for the socket.""" + """Hold the global and per-user connection slots for the socket's lifetime.""" leases: list[Lease] = [] try: if ws_config.max_connections is not None: - lease = await limiter.acquire("ws:all", ws_config.max_connections) + lease = await manager.acquire("ws:all", ws_config.max_connections) if lease is None: logger.warning( "WebSocket connection limit reached (max=%s)", ws_config.max_connections @@ -163,7 +84,7 @@ async def _shared_connection_slots( leases.append(lease) user_key = keying.verified_user_id(websocket) if user_key and ws_config.max_connections_per_user is not None: - lease = await limiter.acquire( + lease = await manager.acquire( f"ws:user:{key_part(user_key)}", ws_config.max_connections_per_user ) if lease is None: @@ -191,9 +112,11 @@ async def ws_run_allowed(websocket: WebSocket) -> int | None: limiter = getattr(websocket.state, "ws_rate_limit", None) if not isinstance(limiter, tuple): # the guard stored none: rate limiting is off return None - rl_config, backend = limiter + rl_config, cache = limiter key = client_key_for(websocket, rl_config) - decision = await backend.check(key, limit=rl_config.requests, window=rl_config.window) + decision = await check( + cache, key, rl_config.requests, rl_config.window, fail_open=rl_config.fail_open + ) if decision.allowed: return None logger.warning("WebSocket run rate limit exceeded for %s", log_safe(key)) diff --git a/tenxgraph_api/src/app/routers/graph/router.py b/tenxgraph_api/src/app/routers/graph/router.py index 7757a115..bccb6ad2 100644 --- a/tenxgraph_api/src/app/routers/graph/router.py +++ b/tenxgraph_api/src/app/routers/graph/router.py @@ -29,7 +29,7 @@ TokenRevocationUnavailableError, token_revoked, ) -from tenxgraph_api.src.app.core.concurrency import Lease +from tenxgraph_api.src.app.core.leases import Lease from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe, summarize_validation_error from tenxgraph_api.src.app.routers.graph import idempotency from tenxgraph_api.src.app.routers.graph.realtime_guard import ( diff --git a/tenxgraph_api/src/app/routers/graph/run_guard.py b/tenxgraph_api/src/app/routers/graph/run_guard.py index 7245d467..3e96d1c1 100644 --- a/tenxgraph_api/src/app/routers/graph/run_guard.py +++ b/tenxgraph_api/src/app/routers/graph/run_guard.py @@ -12,8 +12,8 @@ from collections.abc import Awaitable, Callable from typing import Any -from tenxgraph_api.src.app.core.concurrency import Lease from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.leases import Lease from tenxgraph_api.src.app.routers.graph.run_limits import acquire_run_slot from tenxgraph_api.src.app.routers.graph.thread_lock import ( OnBusy, diff --git a/tenxgraph_api/src/app/routers/graph/run_limits.py b/tenxgraph_api/src/app/routers/graph/run_limits.py index 61ed878f..7054ed64 100644 --- a/tenxgraph_api/src/app/routers/graph/run_limits.py +++ b/tenxgraph_api/src/app/routers/graph/run_limits.py @@ -8,8 +8,9 @@ For a streaming response the slot is held until the stream ends, the client disconnects, or the stream object is discarded without ever being iterated. -Slots come from :func:`~tenxgraph_api.src.app.core.concurrency.concurrency_limiter`: shared -through Redis when it is configured, otherwise counted per process (size the cap per worker). +Slots are leases from :func:`~tenxgraph_api.src.app.core.leases.lease_manager`: shared between +workers when the cache store is (Redis), otherwise counted per process (size the cap per +worker). """ from __future__ import annotations @@ -22,8 +23,8 @@ from fastapi import HTTPException from tenxgraph_api.src.app.core import logger -from tenxgraph_api.src.app.core.concurrency import Lease, concurrency_limiter from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.leases import Lease, lease_manager from tenxgraph_api.src.app.core.redis_client import key_part from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe @@ -50,7 +51,7 @@ async def acquire_run_slot(user: dict[str, Any], runs: RunsConfig) -> Lease: limit = runs.max_concurrent_per_user if not user_id or limit is None: return Lease(None) - lease = await concurrency_limiter().acquire(run_slot_key(str(user_id)), limit) + lease = await lease_manager().acquire(run_slot_key(str(user_id)), limit) if lease is None: logger.warning( "Concurrent run limit reached for user %s (max=%s)", log_safe(user_id), limit diff --git a/tenxgraph_api/src/app/routers/graph/services/graph_service.py b/tenxgraph_api/src/app/routers/graph/services/graph_service.py index 537cd449..7bd8c1c9 100644 --- a/tenxgraph_api/src/app/routers/graph/services/graph_service.py +++ b/tenxgraph_api/src/app/routers/graph/services/graph_service.py @@ -33,8 +33,8 @@ tool_result_ids, unanswered_tool_calls, ) -from tenxgraph_api.src.app.core.concurrency import Lease from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, RunsConfig +from tenxgraph_api.src.app.core.leases import Lease from tenxgraph_api.src.app.core.shutdown import ( KEEPALIVE_SECONDS, cancel_and_wait, diff --git a/tenxgraph_api/src/app/routers/graph/thread_lock.py b/tenxgraph_api/src/app/routers/graph/thread_lock.py index fca78d8b..67d93ac3 100644 --- a/tenxgraph_api/src/app/routers/graph/thread_lock.py +++ b/tenxgraph_api/src/app/routers/graph/thread_lock.py @@ -5,13 +5,13 @@ WebSocket, AG-UI) takes this lock for the thread before the graph starts and holds it until the run is over. -The lock is a one-slot lease from ``core.concurrency.concurrency_limiter`` -(key ``runs:thread:``): shared through Redis when it is configured, otherwise -per process. A Redis lease is renewed in the background while the run is alive and expires on -its own if the worker dies, so a crashed worker cannot hold a thread forever. A Redis error -falls back to a per-process lock (the limiter fails open to its own count), so one worker still -runs one thing at a time on a thread. Run entry points take this lock together with the run -slot through ``run_guard.acquire_run_guard``. +The lock is a one-slot lease from ``core.leases.lease_manager`` (key +``runs:thread:``): shared through the cache store (Redis when it is configured, +otherwise per process). A lease is renewed in the background while the run is alive and +expires on its own if the worker dies, so a crashed worker cannot hold a thread forever. A +cache store error falls back to a per-process lock (the lease manager fails open to its own +count), so one worker still runs one thing at a time on a thread. Run entry points take this +lock together with the run slot through ``run_guard.acquire_run_guard``. A run that finds the thread busy waits up to ``THREAD_BUSY_WAIT_SECONDS`` and is then rejected with 409. With ``on_busy="cancel"`` it first asks the active run to stop (and @@ -29,7 +29,7 @@ from tenxgraph_api.src.app.core import logger from tenxgraph_api.src.app.core.auth.request_config import normalize_thread_id -from tenxgraph_api.src.app.core.concurrency import Lease, concurrency_limiter +from tenxgraph_api.src.app.core.leases import Lease, lease_manager from tenxgraph_api.src.app.core.redis_client import key_part from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe @@ -96,7 +96,7 @@ async def acquire_thread_lock( """ if wait_seconds is None: wait_seconds = THREAD_BUSY_WAIT_SECONDS - limiter = concurrency_limiter() + leases = lease_manager() key = thread_lock_key(thread_id) loop = asyncio.get_running_loop() deadline = loop.time() + wait_seconds @@ -104,7 +104,7 @@ async def acquire_thread_lock( next_stop = 0.0 waited = False while True: - lease = await limiter.acquire(key, 1) + lease = await leases.acquire(key, 1) if lease is not None: if waited and is_disconnected is not None and await is_disconnected(): lease.release() diff --git a/tenxgraph_api/src/app/routers/ping/readiness.py b/tenxgraph_api/src/app/routers/ping/readiness.py index bd32a2ea..e75de5b3 100644 --- a/tenxgraph_api/src/app/routers/ping/readiness.py +++ b/tenxgraph_api/src/app/routers/ping/readiness.py @@ -2,29 +2,27 @@ ``/ping`` is a liveness probe: it answers as long as the process can serve a request. A readiness probe must also say whether the server can do its job, so this checks the -dependencies a request needs: the checkpointer, the configured Redis, and the store. +dependencies a request needs: the checkpointer, the store and the API's cache store. Each is +asked with its own ``aping()`` (read-only; it never creates a schema or a key). Each check is time-boxed and reports only ``ok`` or ``fail``; components with nothing to check are left out, and the reason for a failure is logged, never returned, because the endpoint is public. Results are cached for ``CACHE_SECONDS`` with one check in flight at a time, so a flood -of probe requests cannot turn into a flood of database and Redis round trips. - -Redis is the one dependency the server survives losing: the run lock, run cap, WebSocket caps -and (by default) the rate limiter fail open, and ownership falls back to the database. A Redis -outage therefore must not take every pod out of rotation. Its result is still reported, but only -counts against readiness when a configured feature fails closed without it (see -:func:`redis_required`). While the app is shutting down readiness answers 503 (``draining``) so -the load balancer stops sending new requests. - -A component that knows best how to check itself can expose -``async def readiness_check() -> None`` (raise when not ready), which is used before the -built-in probes. +of probe requests cannot turn into a flood of database and cache round trips. + +The cache store is the one dependency the server survives losing: the run lock, run cap, +WebSocket caps and idempotency fall back to this process, the ownership cache falls back to +the database, and the rate limiter allows by default. A cache store outage therefore must not +take every pod out of rotation. Its result is still reported, but only counts against +readiness when a configured feature fails closed without it (see :func:`cache_required`). +While the app is shutting down readiness answers 503 (``draining``) so the load balancer stops +sending new requests. """ from __future__ import annotations import asyncio -import inspect +import contextlib import logging from collections.abc import Awaitable, Callable from typing import Any @@ -35,87 +33,17 @@ CHECK_TIMEOUT_SECONDS = 2.0 OK, FAIL, SKIPPED = "ok", "fail", "skipped" -# Thread id the owner-lookup probe asks about; no real thread uses it. -_PROBE_THREAD_ID = "__10xgraph_readiness_probe__" - class _Skip(Exception): - """The component has nothing this probe can check.""" - - -async def _maybe_await(value: Any) -> Any: - return await value if inspect.isawaitable(value) else value - - -async def _own_check(component: Any) -> bool: - """Run the component's ``readiness_check`` if it has one; True when it did.""" - check = getattr(component, "readiness_check", None) - if not callable(check): - return False - await _maybe_await(check()) - return True - - -async def _check_schema_version(checkpointer: Any) -> None: - """Fail while the database schema is older than the one this code expects. - - Reads the recorded version without creating anything (``aget_schema_version``), so a - missing or half-upgraded schema (boot setup failed, or the database was restored from an - older backup) keeps the pod out of rotation instead of failing every request. - """ - read_version = getattr(checkpointer, "aget_schema_version", None) - expected_fn = getattr(checkpointer, "_get_current_schema_version", None) - if not callable(read_version) or not callable(expected_fn): - return - found, expected = await read_version(), expected_fn() - if found < expected: - raise RuntimeError(f"checkpointer schema version {found} is below the expected {expected}") + """The component is not configured, or has no ``aping`` to call.""" -async def check_checkpointer(checkpointer: Any) -> None: - if checkpointer is None: +async def ping_component(component: Any) -> None: + """Call ``component.aping()``; skipped when there is no component or no ``aping``.""" + aping = getattr(component, "aping", None) if component is not None else None + if not callable(aping): raise _Skip - if await _own_check(checkpointer): - return - run_query = getattr(checkpointer, "_run_query", None) - if callable(run_query): # PgCheckpointer: one round trip on its pool - await run_query(lambda conn: conn.fetchval("SELECT 1")) - await _check_schema_version(checkpointer) - redis = getattr(checkpointer, "redis", None) - if redis is not None and callable(getattr(redis, "ping", None)): # its cache tier - await _maybe_await(redis.ping()) - if not callable(run_query): - try: # any other checkpointer: a cheap, read-only lookup - await checkpointer.aget_thread_owner(_PROBE_THREAD_ID) - except NotImplementedError: - raise _Skip from None - - -async def check_store(store: Any) -> None: - if store is None: - raise _Skip - if await _own_check(store): - return - for name in ("aping", "ping"): - ping = getattr(store, name, None) - if callable(ping): - await _maybe_await(ping()) - return - raise _Skip - - -async def check_redis(url: str | None) -> None: - if not url: - raise _Skip - from tenxgraph_api.src.app.core.redis_client import create_redis - - # A fresh connection per probe: probes are seconds apart, and a cached client would be - # tied to the event loop that created it. - client = create_redis(url) - try: - await client.ping() - finally: - await client.aclose() + await aping() async def check_outcome( @@ -144,61 +72,59 @@ async def run_check(name: str, check: Callable[[], Awaitable[None]]) -> str: return result -def readiness_targets(container: Any) -> tuple[Any, Any, str | None]: - """``(checkpointer, store, redis_url)`` the checks run against, from the app's container. - - Redis is the URL the API's own clients use (run locks and caps, ownership cache, token - revocation): ``redis`` in 10xgraph.json, else ``REDIS_URL``. None for what is not set. - """ +def readiness_targets(container: Any) -> tuple[Any, Any, Any]: + """``(checkpointer, store, cache store)`` bound in the app's container; None when unbound.""" from tenxgraph.storage.checkpointer import BaseCheckpointer from tenxgraph.storage.store import BaseStore - from tenxgraph_api.src.app.core.config.graph_config import GraphConfig - from tenxgraph_api.src.app.core.config.settings import get_settings + from tenxgraph_api.src.app.core.cache_store import BaseCacheStore - graph_config = container.try_get(GraphConfig) if GraphConfig in container else None - redis_url = getattr(graph_config, "redis_url", None) or get_settings().REDIS_URL - return ( - container.try_get(BaseCheckpointer) if BaseCheckpointer in container else None, - container.try_get(BaseStore) if BaseStore in container else None, - redis_url if isinstance(redis_url, str) else None, - ) + def bound(kind: Any) -> Any: + return container.try_get(kind) if kind in container else None + return bound(BaseCheckpointer), bound(BaseStore), bound(BaseCacheStore) -def redis_required(graph_config: Any) -> bool: - """True when a configured feature fails closed without Redis. - Token revocation on Redis answers 503 when Redis is down, and a Redis rate limiter with - ``fail_open`` off denies requests. Every other Redis use fails open. +def cache_required_by(graph_config: Any) -> list[str]: + """The configured features that fail closed without the cache store, by config key. + + Token revocation answers 503 while it cannot read revocations, and a rate limiter with + ``fail_open`` off denies requests. Every other use of the cache store fails open. """ - try: - if getattr(graph_config, "token_revocation", None) == "redis": - return True - limit = getattr(graph_config, "rate_limit", None) - return bool( - limit is not None - and getattr(limit, "backend", None) == "redis" - and getattr(limit, "fail_open", True) is False - ) - except Exception: - return False - - -def is_ready(checks: dict[str, str], *, redis_is_required: bool) -> bool: - """Whether ``checks`` allow traffic; a failed optional Redis does not count.""" + if graph_config is None: + return [] + found = [] + # A bad value is reported by the config check at boot, not here. + with contextlib.suppress(Exception): + if getattr(graph_config, "token_revocation", None) is not None: + found.append("token_revocation") + with contextlib.suppress(Exception): + limit = getattr(graph_config, "rate_limit", None) # None when absent or disabled + if limit is not None and getattr(limit, "fail_open", True) is False: + found.append("rate_limit.fail_open=false") + return found + + +def cache_required(graph_config: Any) -> bool: + """True when a configured feature fails closed without the cache store.""" + return bool(cache_required_by(graph_config)) + + +def is_ready(checks: dict[str, str], *, cache_is_required: bool) -> bool: + """Whether ``checks`` allow traffic; a failed cache store counts only when required.""" return all( - result != FAIL or (name == "redis" and not redis_is_required) + result != FAIL or (name == "cache" and not cache_is_required) for name, result in checks.items() ) -async def readiness(checkpointer: Any, store: Any, redis_url: str | None) -> dict[str, str]: +async def readiness(checkpointer: Any, store: Any, cache: Any) -> dict[str, str]: """Run every check concurrently; ``{component: ok | fail | skipped}``.""" - names = ("checkpointer", "redis", "store") + names = ("checkpointer", "store", "cache") results = await asyncio.gather( - run_check("checkpointer", lambda: check_checkpointer(checkpointer)), - run_check("redis", lambda: check_redis(redis_url)), - run_check("store", lambda: check_store(store)), + run_check("checkpointer", lambda: ping_component(checkpointer)), + run_check("store", lambda: ping_component(store)), + run_check("cache", lambda: ping_component(cache)), ) return dict(zip(names, results, strict=True)) @@ -208,7 +134,7 @@ async def readiness(checkpointer: Any, store: Any, redis_url: str | None) -> dic _cache_lock = asyncio.Lock() -async def cached_readiness(checkpointer: Any, store: Any, redis_url: str | None) -> dict[str, str]: +async def cached_readiness(checkpointer: Any, store: Any, cache: Any) -> dict[str, str]: """:func:`readiness`, reused for ``CACHE_SECONDS``; concurrent callers share one run. Skipped components are dropped from the result. @@ -216,7 +142,7 @@ async def cached_readiness(checkpointer: Any, store: Any, redis_url: str | None) loop = asyncio.get_running_loop() async with _cache_lock: if _cache["result"] is None or loop.time() - _cache["at"] >= CACHE_SECONDS: - checks = await readiness(checkpointer, store, redis_url) + checks = await readiness(checkpointer, store, cache) _cache["result"] = {k: v for k, v in checks.items() if v != SKIPPED} _cache["at"] = loop.time() return dict(_cache["result"]) diff --git a/tenxgraph_api/src/app/routers/ping/router.py b/tenxgraph_api/src/app/routers/ping/router.py index 35b6eb2b..a078c5c4 100644 --- a/tenxgraph_api/src/app/routers/ping/router.py +++ b/tenxgraph_api/src/app/routers/ping/router.py @@ -33,10 +33,10 @@ async def ping_server( "/health/ready", summary="Readiness probe", description=( - "Checks the dependencies a request needs (checkpointer, Redis, store). Answers 200 " - "when every configured one is reachable and 503 otherwise, and 503 while the server is " - "shutting down. A Redis outage alone stays 200 unless a configured feature (token " - "revocation, a fail-closed rate limiter) needs it. /ping stays the liveness " + "Checks the dependencies a request needs (checkpointer, store, cache store). Answers " + "200 when every configured one is reachable and 503 otherwise, and 503 while the server " + "is shutting down. A cache store outage alone stays 200 unless a configured feature " + "(token revocation, a fail-closed rate limiter) needs it. /ping stays the liveness " "probe. Public, like /ping: it reports only ok or fail per configured component, and the " "result is cached for a few seconds." ), @@ -47,7 +47,7 @@ async def readiness_probe(request: Request): from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.core.utils.container import current_container - from .readiness import cached_readiness, is_ready, readiness_targets, redis_required + from .readiness import cache_required, cached_readiness, is_ready, readiness_targets if getattr(request.app.state, "draining", False): # Shutting down: leave the load balancer's rotation before the sockets close. @@ -56,7 +56,7 @@ async def readiness_probe(request: Request): container: InjectQ = current_container() graph_config = container.try_get(GraphConfig) if GraphConfig in container else None checks = await cached_readiness(*readiness_targets(container)) - ready = is_ready(checks, redis_is_required=redis_required(graph_config)) + ready = is_ready(checks, cache_is_required=cache_required(graph_config)) return JSONResponse( status_code=200 if ready else 503, content={"status": "ready" if ready else "not_ready", "checks": checks}, diff --git a/tenxgraph_api/src/app/routers/ping/startup_checks.py b/tenxgraph_api/src/app/routers/ping/startup_checks.py index af068075..23c74904 100644 --- a/tenxgraph_api/src/app/routers/ping/startup_checks.py +++ b/tenxgraph_api/src/app/routers/ping/startup_checks.py @@ -1,14 +1,20 @@ -"""Startup gate: refuse to serve while a configured backend cannot be reached. +"""Startup gate: refuse to serve while a backend the server needs cannot be reached. -A server whose Redis, Postgres or vector store is down still answers ``/ping``, and before -this gate it served every other request with an error (a fail-closed rate limiter answered -429 to everything). The lifespan now runs the readiness probes (``readiness.py``) for each -configured backend before the app serves, retrying with a short backoff for +A server whose checkpointer or store is down still answers ``/ping``, and before this gate it +served every other request with an error. The lifespan now calls ``aping()`` on what is bound +(the checkpointer, the store and the API's cache store, as readiness does in +``readiness.py``) before the app serves, retrying with a short backoff for ``STARTUP_CHECK_TIMEOUT_SECONDS`` so services started together can come up in any order. When -one still fails, :class:`StartupCheckError` stops the boot and names each component, its -``host:port`` (never credentials) and the error. ``STARTUP_CHECKS=false`` turns the gate off. +one still fails, :class:`StartupCheckError` stops the boot and names each component, its class, +where its setting came from and its endpoint when the component describes one (never +credentials). ``STARTUP_CHECKS=false`` turns the gate off. -Unlike readiness, a Redis outage is not tolerated here: a configured backend that is wrong at +The cache store is fatal only when a configured feature fails closed without it +(``token_revocation``, or ``rate_limit`` with ``fail_open: false``; see +``readiness.cache_required_by``). Otherwise it is checked once and an unreachable one logs one +WARNING: the run locks and caps fall back to this process, and the server starts. + +Unlike readiness, a required backend's outage is not tolerated here: a backend that is wrong at boot is a deployment mistake, not a transient outage. """ @@ -19,18 +25,18 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from typing import Any -from urllib.parse import urlsplit +from tenxgraph_api.src.app.core.cache_store.loader import CacheStoreSource +from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.core.utils.log_sanitizer import redact_text from .readiness import ( CHECK_TIMEOUT_SECONDS, FAIL, OK, - check_checkpointer, + cache_required_by, check_outcome, - check_redis, - check_store, + ping_component, readiness_targets, ) @@ -39,107 +45,75 @@ FIRST_RETRY_DELAY_SECONDS = 0.5 MAX_RETRY_DELAY_SECONDS = 5.0 -HINT = "Start Redis/Postgres/Qdrant, or set STARTUP_CHECKS=false to skip this check." +HINT = "Start the backend, fix its URL, or set STARTUP_CHECKS=false to skip this check." class StartupCheckError(RuntimeError): - """A configured backend stayed unreachable for the whole startup window.""" + """A required backend stayed unreachable for the whole startup window.""" @dataclass class Target: - """One configured backend to check: a name, its probe and where it lives.""" + """One bound backend to check: a name, its probe and what to call it in messages.""" name: str check: Callable[[], Awaitable[None]] - endpoints: list[str] = field(default_factory=list) + details: list[str] = field(default_factory=list) + required: bool = True -def host_port(url: Any) -> str | None: - """``host:port`` of a URL or DSN, without user, password, path or query; None if unknown.""" - if not isinstance(url, str) or "://" not in url: - return None - try: - parts = urlsplit(url) - host = parts.hostname - port = parts.port - except ValueError: - return None - if not host: - return None - if ":" in host: - host = f"[{host}]" - return f"{host}:{port}" if port else host - - -def _redis_client_endpoint(client: Any) -> str | None: - """``host:port`` a redis-py client connects to, from its pool's connection settings.""" - kwargs = getattr(getattr(client, "connection_pool", None), "connection_kwargs", None) - if not isinstance(kwargs, dict) or not kwargs.get("host"): - return None - return f"{kwargs['host']}:{kwargs.get('port', 6379)}" - - -def _checkpointer_endpoints(checkpointer: Any) -> list[str]: - """Best effort: the Postgres and Redis a ``PgCheckpointer`` uses.""" - found = [] - pool_config = getattr(checkpointer, "_pg_pool_config", None) - dsn = pool_config.get("postgres_dsn") if isinstance(pool_config, dict) else None - if (where := host_port(dsn)) is not None: - found.append(f"postgres {where}") - if (where := _redis_client_endpoint(getattr(checkpointer, "redis", None))) is not None: - found.append(f"redis {where}") - return found - - -def _store_endpoints(store: Any) -> list[str]: - """Best effort: the server a ``QdrantStore`` talks to (its client's REST address).""" - remote = getattr(getattr(store, "client", None), "_client", None) - where = host_port(getattr(remote, "rest_uri", None)) - return [where] if where else [] - - -def startup_targets(container: Any, rate_limit_backend: Any = None) -> list[Target]: - """Every configured backend, as readiness sees it, plus the rate limiter's own Redis. - - Components that are not configured are left out. The rate limiter's Redis is added only - when it was built from its own URL and that URL differs from the API's Redis. - """ - checkpointer, store, redis_url = readiness_targets(container) +def _component_details(component: Any) -> list[str]: + """The class name and, when the component has a ``describe()``, its endpoint.""" + details = [type(component).__name__] + describe = getattr(component, "describe", None) + if callable(describe): + try: + where = describe() + except Exception: # a custom component's describe must not break boot + where = None + if isinstance(where, str) and where: + details.append(where) + return details + + +def _cache_source(container: Any) -> str | None: + found = container.try_get(CacheStoreSource) if CacheStoreSource in container else None + return getattr(found, "text", None) + + +def _graph_config(container: Any) -> Any: + return container.try_get(GraphConfig) if GraphConfig in container else None + + +def startup_targets(container: Any) -> list[Target]: + """The checkpointer, store and cache store bound in ``container``; unbound ones left out.""" + checkpointer, store, cache = readiness_targets(container) targets = [] if checkpointer is not None: targets.append( Target( "checkpointer", - lambda: check_checkpointer(checkpointer), - _checkpointer_endpoints(checkpointer), + lambda: ping_component(checkpointer), + _component_details(checkpointer), ) ) if store is not None: - targets.append(Target("store", lambda: check_store(store), _store_endpoints(store))) - if redis_url: + targets.append(Target("store", lambda: ping_component(store), _component_details(store))) + if cache is not None: + details = _component_details(cache) + if source := _cache_source(container): + details.insert(1, source) + required_by = cache_required_by(_graph_config(container)) + if required_by: + details.append(f"required by {' and '.join(required_by)}") targets.append( - Target("redis", lambda: check_redis(redis_url), [host_port(redis_url) or "?"]) - ) - limiter_url = getattr(rate_limit_backend, "redis_url", None) - if isinstance(limiter_url, str) and limiter_url and limiter_url != redis_url: - targets.append( - Target( - "rate_limit redis", - lambda: check_redis(limiter_url), - [host_port(limiter_url) or "?"], - ) - ) - elif hasattr(rate_limit_backend, "redis_url") and not limiter_url: - logger.info( - "Startup check: the rate limiter uses a Redis client bound in the container; " - "it is not checked at startup." + Target("cache", lambda: ping_component(cache), details, required=bool(required_by)) ) return targets def _describe(target: Target) -> str: - return f"{target.name} ({', '.join(target.endpoints)})" if target.endpoints else target.name + return f"{target.name} ({', '.join(target.details)})" if target.details else target.name def _error_text(error: BaseException | None, timeout: float) -> str: @@ -160,7 +134,7 @@ async def wait_for_backends( Each round checks the components still failing, concurrently, each bounded by ``attempt_timeout``. There is always at least one round. A component with nothing to check - (a store without ``aping``/``ping``) is skipped with an INFO line. + (no ``aping``) is skipped with an INFO line. """ loop = asyncio.get_running_loop() deadline = loop.time() + timeout @@ -205,8 +179,7 @@ async def wait_for_backends( raise StartupCheckError( "\n".join( [ - f"Refusing to start: {len(pending)} configured backend(s) unreachable after " - f"{timeout:g}s:", + f"Refusing to start: {len(pending)} backend(s) unreachable after {timeout:g}s:", *lines, HINT, ] @@ -214,18 +187,47 @@ async def wait_for_backends( ) +def _label(target: Target) -> str: + """``RedisCacheStore (from REDIS_URL, localhost:6379)``: the class, then the rest.""" + if not target.details: + return target.name + first, *rest = target.details + return f"{first} ({', '.join(rest)})" if rest else first + + +async def _check_optional(target: Target, attempt_timeout: float) -> bool: + """One attempt at a backend the server can run without; one WARNING when it fails.""" + result, error = await check_outcome(target.check, attempt_timeout) + if result != FAIL: + return True + logger.warning( + "Startup check: cache store %s unreachable: locks and caps fall back to per process " + "until it answers (%s). No configured feature needs it, so the server starts.", + _label(target), + _error_text(error, attempt_timeout), + ) + return False + + async def check_backends_at_startup( - container: Any, *, timeout: float, rate_limit_backend: Any = None + container: Any, *, timeout: float, attempt_timeout: float = CHECK_TIMEOUT_SECONDS ) -> None: """Run the startup gate for the app's container; log and raise when it fails.""" - targets = startup_targets(container, rate_limit_backend) + targets = startup_targets(container) if not targets: logger.debug("Startup check: no backends configured") return + required = [target for target in targets if target.required] + reachable = list(required) + for target in targets: + if not target.required and await _check_optional(target, attempt_timeout): + reachable.append(target) try: - await wait_for_backends(targets, timeout=timeout) + await wait_for_backends(required, timeout=timeout, attempt_timeout=attempt_timeout) except StartupCheckError as exc: # The text as the format string: an argument would have its line breaks escaped. logger.critical(str(exc)) raise - logger.info("Startup check passed: %s", ", ".join(target.name for target in targets)) + logger.info( + "Startup check passed: %s", ", ".join(target.name for target in reachable) or "nothing" + ) diff --git a/tenxgraph_api/testing/__init__.py b/tenxgraph_api/testing/__init__.py new file mode 100644 index 00000000..6f2f92f0 --- /dev/null +++ b/tenxgraph_api/testing/__init__.py @@ -0,0 +1 @@ +"""Test helpers for projects built on 10xgraph-api.""" diff --git a/tenxgraph_api/testing/cache_store_contract.py b/tenxgraph_api/testing/cache_store_contract.py new file mode 100644 index 00000000..bd9c7631 --- /dev/null +++ b/tenxgraph_api/testing/cache_store_contract.py @@ -0,0 +1,304 @@ +# ruff: noqa: PLR2004 - the checks compare against literal counts + +"""Contract checks for a ``BaseCacheStore`` implementation. + +Every check takes a factory (a zero-argument callable returning a store, or an awaitable of +one), opens its own store, works under keys of its own and closes the store, so the checks can +run against a shared server. Use them in two ways: + +As a pytest mixin (needs no pytest plugin; each test runs its check with ``asyncio.run``):: + + from tenxgraph_api.testing.cache_store_contract import CacheStoreContract + + + class TestMyStore(CacheStoreContract): + def make_store(self): + return MyStore(...) + +Or directly, from any async code:: + + await run_cache_store_contract(lambda: MyStore(...)) + +A failed check raises ``AssertionError``. The timing checks use TTLs of a few hundred +milliseconds, so the store must answer well within that. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import uuid +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import Any + +from tenxgraph_api.src.app.core.cache_store.base import BaseCacheStore + + +StoreFactory = Callable[[], BaseCacheStore | Awaitable[BaseCacheStore]] + +SHORT_TTL = 0.3 +LONG_TTL = 30.0 +# Allowed gap between expected and reported expiry times (round trips, clock resolution). +TOLERANCE = 0.25 + + +def _expect(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _some[T](value: T | None, message: str) -> T: + if value is None: + raise AssertionError(message) + return value + + +def _key(name: str) -> str: + """A key no other run uses, so checks can share a server.""" + return f"contract:{uuid.uuid4().hex}:{name}" + + +@contextlib.asynccontextmanager +async def _open(factory: StoreFactory) -> AsyncIterator[BaseCacheStore]: + made: Any = factory() + store = await made if inspect.isawaitable(made) else made + _expect(isinstance(store, BaseCacheStore), f"factory returned {type(store).__name__}") + try: + yield store + finally: + await store.aclose() + + +# 1 +async def check_aping(factory: StoreFactory) -> None: + async with _open(factory) as store: + await store.aping() + + +# 2 +async def check_acquire_limit(factory: StoreFactory) -> None: + async with _open(factory) as store: + key = _key("limit") + slots = set() + for taken in range(1, 4): + result = await store.acquire(key, 3, LONG_TTL) + _expect(result.granted, f"slot {taken} of 3 was denied") + _expect(result.count == taken, f"count {result.count} after {taken} grants") + _expect(result.slot_id is not None, "a granted slot has no slot_id") + slots.add(result.slot_id) + _expect(len(slots) == 3, "slot ids are not unique") + for _ in range(3): + denied = await store.acquire(key, 3, LONG_TTL) + _expect(not denied.granted, "acquire over the limit was granted") + _expect(denied.slot_id is None, "a denied acquire has a slot_id") + _expect(denied.count == 3, f"a denied acquire changed the count to {denied.count}") + _expect(not (await store.acquire(_key("zero"), 0, LONG_TTL)).granted, "limit 0 granted") + + +# 3 +async def check_concurrent_acquire(factory: StoreFactory) -> None: + async with _open(factory) as store: + key = _key("race") + results = await asyncio.gather(*(store.acquire(key, 5, LONG_TTL) for _ in range(50))) + granted = [r for r in results if r.granted] + _expect(len(granted) == 5, f"50 concurrent acquires granted {len(granted)}, not 5") + after = await store.acquire(key, 5, LONG_TTL) + _expect(not after.granted and after.count == 5, f"live count is {after.count}, not 5") + + +# 4 +async def check_release_and_renew(factory: StoreFactory) -> None: + async with _open(factory) as store: + key = _key("lease") + first = await store.acquire(key, 1, LONG_TTL) + _expect(first.granted, "the first slot was denied") + _expect(not (await store.acquire(key, 1, LONG_TTL)).granted, "a held lock was taken") + first_id = _some(first.slot_id, "slot_id missing") + await store.release(key, first_id) + await store.release(key, first_id) # releasing twice is not an error + _expect(not await store.renew(key, first_id, LONG_TTL), "renewed a released slot") + second = await store.acquire(key, 1, SHORT_TTL) + _expect(second.granted, "release did not free the slot") + _expect(second.count == 1, "renew of a released slot re-created it") + second_id = _some(second.slot_id, "slot_id missing") + + # renew extends: past the original ttl the slot is still held + await asyncio.sleep(SHORT_TTL / 2) + _expect(await store.renew(key, second_id, SHORT_TTL * 3), "renew of a held slot") + await asyncio.sleep(SHORT_TTL) + held = await store.acquire(key, 1, LONG_TTL) + _expect(not held.granted, "a renewed slot expired at its old ttl") + + # an expired slot cannot be renewed and is not re-created + other = _key("expired") + short = await store.acquire(other, 1, SHORT_TTL) + short_id = _some(short.slot_id, "slot_id missing") + await asyncio.sleep(SHORT_TTL + TOLERANCE) + _expect(not await store.renew(other, short_id, LONG_TTL), "renewed an expired slot") + retaken = await store.acquire(other, 1, LONG_TTL) + _expect(retaken.granted and retaken.count == 1, "renew re-created an expired slot") + await store.release(_key("missing"), "no-such-slot") + + +# 5 +async def check_slot_expiry(factory: StoreFactory) -> None: + async with _open(factory) as store: + key = _key("expiry") + start = await store.now() + first = await store.acquire(key, 5, SHORT_TTL) + first_oldest = _some(first.oldest_expiry, "oldest_expiry missing on a granted slot") + _expect( + abs(first_oldest - (start + SHORT_TTL)) <= TOLERANCE, + f"oldest_expiry {first_oldest} is not about now + ttl ({start + SHORT_TTL})", + ) + second = await store.acquire(key, 5, LONG_TTL) + _expect(second.count == 2, "second slot not counted") + _expect(second.oldest_expiry == first_oldest, "oldest_expiry is not the oldest") + await asyncio.sleep(SHORT_TTL + TOLERANCE) + later = await store.now() + third = await store.acquire(key, 5, LONG_TTL) + _expect(third.count == 2, f"expired slot still counted ({third.count} live)") + third_oldest = _some(third.oldest_expiry, "oldest_expiry missing") + _expect( + abs(third_oldest - (start + LONG_TTL)) <= TOLERANCE + (later - start), + "oldest_expiry did not move to the next live slot", + ) + denied = await store.acquire(key, 2, LONG_TTL) + _expect(not denied.granted, "acquire over the limit was granted") + _expect(denied.oldest_expiry == third_oldest, "denied result has no oldest_expiry") + + lone = _key("lone") + await store.acquire(lone, 1, SHORT_TTL) + await asyncio.sleep(SHORT_TTL + TOLERANCE) + fresh = await store.acquire(lone, 1, SHORT_TTL) + _expect(fresh.granted and fresh.count == 1, "the only slot did not expire") + + +# 6 +async def check_values(factory: StoreFactory) -> None: + async with _open(factory) as store: + key = _key("value") + _expect(await store.get(key) is None, "a missing key has a value") + _expect(await store.set(key, b"one", None) is True, "set(always) did not return True") + _expect(await store.get(key) == b"one", "get after set") + await store.set(key, b"two", LONG_TTL) + _expect(await store.get(key) == b"two", "set(always) did not overwrite") + await store.delete(key) + _expect(await store.get(key) is None, "delete left the value") + await store.delete(key) # deleting a missing key is not an error + + short = _key("ttl") + await store.set(short, b"x", SHORT_TTL) + _expect(await store.get(short) == b"x", "value with a ttl not readable") + await asyncio.sleep(SHORT_TTL + TOLERANCE) + _expect(await store.get(short) is None, "value outlived its ttl") + + absent = _key("absent") + _expect(await store.set(absent, b"a", LONG_TTL, "if_absent"), "if_absent on a new key") + _expect(not await store.set(absent, b"b", LONG_TTL, "if_absent"), "if_absent overwrote") + _expect(await store.get(absent) == b"a", "if_absent changed the value") + + greater = _key("greater") + _expect(await store.set(greater, b"10.5", LONG_TTL, "if_greater"), "if_greater on missing") + _expect(await store.set(greater, b"11", LONG_TTL, "if_greater"), "if_greater larger") + _expect(not await store.set(greater, b"3", LONG_TTL, "if_greater"), "if_greater smaller") + _expect(not await store.set(greater, b"11", LONG_TTL, "if_greater"), "if_greater equal") + _expect(float(await store.get(greater) or b"nan") == 11, "if_greater kept the wrong value") + for bad in (b"abc", b"", b"1e999"): + with contextlib.suppress(ValueError): + await store.set(greater, bad, LONG_TTL, "if_greater") + raise AssertionError(f"if_greater accepted {bad!r}") + + race = _key("greater-race") + values = [f"{1_700_000_000 + n * 0.25:.6f}".encode() for n in range(50)] + await asyncio.gather( + *(store.set(race, v, LONG_TTL, "if_greater") for v in reversed(values)) + ) + stored = await store.get(race) + _expect(stored is not None and float(stored) == float(values[-1]), "lost the maximum") + + +# 7 +async def check_clock(factory: StoreFactory) -> None: + async with _open(factory) as store: + previous = await store.now() + _expect(previous > 1_600_000_000, f"now() {previous} is not seconds since the epoch") + for _ in range(20): + current = await store.now() + _expect(current >= previous, f"now() went back from {previous} to {current}") + previous = current + + +# 8 +async def check_odd_keys(factory: StoreFactory) -> None: + async with _open(factory) as store: + base = _key("") + names = [f"{base}a:b:c", f"{base}with space", f"{base}ünicøde 漢字", f"{base}%3A"] + for index, name in enumerate(names): + await store.set(name, f"v{index}".encode(), LONG_TTL) + for index, name in enumerate(names): + _expect(await store.get(name) == f"v{index}".encode(), f"value under {name!r}") + for name in names: + slot = await store.acquire(name + ":slots", 1, LONG_TTL) + _expect(slot.granted, f"slot under {name!r}") + slot_id = _some(slot.slot_id, "slot_id missing") + _expect(await store.renew(name + ":slots", slot_id, LONG_TTL), "renew odd key") + await store.release(name + ":slots", slot_id) + await store.delete(name) + _expect(await store.get(name) is None, f"delete under {name!r}") + + +CONTRACT_CHECKS: tuple[Callable[[StoreFactory], Awaitable[None]], ...] = ( + check_aping, + check_acquire_limit, + check_concurrent_acquire, + check_release_and_renew, + check_slot_expiry, + check_values, + check_clock, + check_odd_keys, +) + + +async def run_cache_store_contract(factory: StoreFactory) -> None: + """Run every contract check against stores made by ``factory``.""" + for check in CONTRACT_CHECKS: + await check(factory) + + +class CacheStoreContract: + """Pytest mixin: subclass it as ``Test...`` and implement :meth:`make_store`.""" + + def make_store(self) -> BaseCacheStore | Awaitable[BaseCacheStore]: + raise NotImplementedError + + def _run(self, check: Callable[[StoreFactory], Awaitable[None]]) -> None: + async def main() -> None: + await check(self.make_store) + + asyncio.run(main()) + + def test_aping(self) -> None: + self._run(check_aping) + + def test_acquire_limit(self) -> None: + self._run(check_acquire_limit) + + def test_concurrent_acquire(self) -> None: + self._run(check_concurrent_acquire) + + def test_release_and_renew(self) -> None: + self._run(check_release_and_renew) + + def test_slot_expiry(self) -> None: + self._run(check_slot_expiry) + + def test_values(self) -> None: + self._run(check_values) + + def test_clock(self) -> None: + self._run(check_clock) + + def test_odd_keys(self) -> None: + self._run(check_odd_keys) diff --git a/tests/cli/test_group6_cli_templates.py b/tests/cli/test_group6_cli_templates.py index 20e066f8..9e591f78 100644 --- a/tests/cli/test_group6_cli_templates.py +++ b/tests/cli/test_group6_cli_templates.py @@ -36,8 +36,11 @@ def test_production_rate_limit_is_per_user_when_auth_is_on() -> None: def test_production_config_enables_token_revocation_with_redis() -> None: config = _cmd()._build_config(_context("jwt"), True) - assert config["token_revocation"] == "redis" - assert "token_revocation" not in _cmd()._build_config(_context("jwt", "memory"), True) + assert config["token_revocation"] is True + assert config["redis"] == "${REDIS_URL}" + memory = _cmd()._build_config(_context("jwt", "memory"), True) + assert "token_revocation" not in memory + assert "redis" not in memory assert "token_revocation" not in _cmd()._build_config(_context("none"), True) @@ -46,7 +49,10 @@ def test_static_production_template_matches() -> None: Path(__file__).parents[2] / "tenxgraph_api" / "cli" / "templates" / "prod" / "10xgraph.json" ) config = json.loads(path.read_text(encoding="utf-8")) - assert config["token_revocation"] == "redis" + assert config["token_revocation"] is True + assert config["redis"] == "${REDIS_URL}" + assert "backend" not in config["rate_limit"] + assert "redis" not in config["rate_limit"] assert "/redocs" in config["rate_limit"]["exclude_paths"] assert "/redoc" not in config["rate_limit"]["exclude_paths"] @@ -64,30 +70,99 @@ def test_k8s_env_does_not_shadow_the_secret() -> None: assert "ALLOWED_HOST" not in env -def test_revoke_command_writes_to_redis(monkeypatch) -> None: - from tenxgraph_api.cli.commands.revoke import RevokeCommand - from tenxgraph_api.src.app.core.auth import revocation +def _recording_cache(monkeypatch): + """Route ``RedisCacheStore.from_url`` to an in-memory store that records its URL.""" + from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, RedisCacheStore calls: list[tuple] = [] - class _Backend: - def __init__(self, url): - calls.append(("url", url)) + class _Store(RedisCacheStore): + """Stands in for a shared store; keeps its data in memory.""" + + def __init__(self): + self.inner = InMemoryCacheStore() - async def revoke_user(self, user, before=None): - calls.append(("user", user, before)) + async def now(self): + return await self.inner.now() - async def revoke_token(self, jti, expires_at=None): - calls.append(("jti", jti, expires_at)) + async def get(self, key): + return await self.inner.get(key) + + async def set(self, key, value, ttl, mode="always"): + return await self.inner.set(key, value, ttl, mode) async def aclose(self): calls.append(("closed",)) - monkeypatch.setattr(revocation, "RedisTokenRevocation", _Backend) + store = _Store() + + def from_url(url, **kwargs): + calls.append(("url", url)) + return store + + monkeypatch.setattr(RedisCacheStore, "from_url", staticmethod(from_url)) + return store, calls + + +def test_revoke_command_writes_to_the_cache_store(monkeypatch, tmp_path) -> None: + import asyncio + + from tenxgraph_api.cli.commands.revoke import RevokeCommand + + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("REDIS_URL", raising=False) + store, calls = _recording_cache(monkeypatch) cmd = RevokeCommand(output=OutputFormatter(stream=io.StringIO(), quiet=True)) - assert cmd.execute(user="u1", redis_url="redis://x") == 0 - assert cmd.execute(jti="j1", expires_at=5.0, redis_url="redis://x") == 0 - assert ("user", "u1", None) in calls and ("jti", "j1", 5.0) in calls + assert cmd.execute(user="u1", before=100.0, redis_url="redis://x") == 0 + assert cmd.execute(jti="j1", redis_url="redis://x") == 0 + assert calls.count(("url", "redis://x")) == 2 assert calls.count(("closed",)) == 2 + assert asyncio.run(store.get("revoked:user:u1")) == b"100.000000" + assert asyncio.run(store.get("revoked:jti:j1")) == b"1" assert cmd.execute(redis_url="redis://x") != 0 # neither --user nor --jti assert cmd.execute(user="u", jti="j", redis_url="redis://x") != 0 + + +def test_revoke_command_uses_the_server_selection(monkeypatch, tmp_path) -> None: + from tenxgraph_api.cli.commands.revoke import RevokeCommand + + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("REDIS_URL", raising=False) + _, calls = _recording_cache(monkeypatch) + (tmp_path / "10xgraph.json").write_text( + json.dumps({"agent": "g:app", "redis": "redis://config", "token_revocation": True}) + ) + cmd = RevokeCommand(output=OutputFormatter(stream=io.StringIO(), quiet=True)) + assert cmd.execute(user="u1") == 0 + assert ("url", "redis://config") in calls + + +def test_revoke_command_refuses_an_in_memory_cache_store(monkeypatch, tmp_path) -> None: + from tenxgraph_api.cli.commands.revoke import RevokeCommand + + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("REDIS_URL", raising=False) + (tmp_path / "10xgraph.json").write_text(json.dumps({"agent": "g:app"})) + stream = io.StringIO() + cmd = RevokeCommand(output=OutputFormatter(stream=stream)) + assert cmd.execute(user="u1") != 0 + import pytest + + from tenxgraph_api.cli.exceptions import ValidationError + + with pytest.raises(ValidationError, match="in-memory"): + cmd._cache_store(None, "10xgraph.json") + + +def test_revoke_command_rejects_a_removed_token_revocation_value(monkeypatch, tmp_path) -> None: + import pytest + + from tenxgraph_api.cli.commands.revoke import RevokeCommand + + monkeypatch.chdir(tmp_path) + (tmp_path / "10xgraph.json").write_text( + json.dumps({"agent": "g:app", "token_revocation": "redis"}) + ) + cmd = RevokeCommand(output=OutputFormatter(stream=io.StringIO(), quiet=True)) + with pytest.raises(ValueError, match="cache_store"): + cmd._cache_store("redis://x", "10xgraph.json") diff --git a/tests/cli/test_init_non_interactive.py b/tests/cli/test_init_non_interactive.py index 20115993..8adecb6a 100644 --- a/tests/cli/test_init_non_interactive.py +++ b/tests/cli/test_init_non_interactive.py @@ -2,11 +2,18 @@ from __future__ import annotations +import asyncio import io from pathlib import Path +from types import SimpleNamespace + +import pytest from tenxgraph_api.cli.commands.init import InitCommand from tenxgraph_api.cli.core.output import OutputFormatter +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, RedisCacheStore +from tenxgraph_api.src.app.core.cache_store.loader import build_cache_store +from tenxgraph_api.src.app.core.config.graph_config import GraphConfig def test_dry_run_does_not_create_target(tmp_path: Path) -> None: @@ -82,7 +89,9 @@ def test_production_dependencies_follow_the_answers(tmp_path: Path) -> None: import json config = json.loads((target / "10xgraph.json").read_text()) - assert config["rate_limit"]["redis"] == {"url": "${REDIS_URL}"} + assert config["redis"] == "${REDIS_URL}" + assert set(config["rate_limit"]) >= {"enabled", "requests", "window", "by"} + assert "redis" not in config["rate_limit"] agent_py = (target / "graph" / "agent.py").read_text() assert 'model="gpt-4o-mini"' in agent_py @@ -184,3 +193,53 @@ def test_init_adds_secret_patterns_to_an_existing_gitignore(tmp_path: Path) -> N content = (tmp_path / ".gitignore").read_text(encoding="utf-8") assert content.startswith("node_modules/\n") assert ".env\n" in content + + +@pytest.mark.parametrize( + ("template", "auth", "rate_limit"), + [ + ("quick-start", None, "none"), + ("production", "jwt", "none"), + ("production", "jwt", "memory"), + ("production", "jwt", "redis"), + ("production", "custom", "redis"), + ("production", "none", "redis"), + ], +) +def test_generated_config_loads_through_graph_config( + tmp_path: Path, monkeypatch, template: str, auth: str | None, rate_limit: str +) -> None: + """Every init recipe writes a 10xgraph.json the server parses without a config error.""" + target = tmp_path / "agent" + result = InitCommand(OutputFormatter(stream=io.StringIO())).execute( + path=str(target), + agent_name="Agent", + template=template, + auth=auth, + rate_limit=rate_limit, + non_interactive=True, + ) + assert result == 0 + monkeypatch.chdir(target) # the config names ".env": never load one from elsewhere + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + monkeypatch.setenv("JWT_SECRET_KEY", "x" * 32) + monkeypatch.setenv("JWT_ALGORITHM", "HS256") + + config = GraphConfig(str(target / "10xgraph.json")) + assert config.graph_path == "graph.agent:app" + config.auth_config() + config.websocket # noqa: B018 - parsing is the check + config.runs # noqa: B018 + assert config.cache_store is None + limit = config.rate_limit + assert (limit is not None) == (rate_limit != "none") + revocation = config.token_revocation + assert (revocation is not None) == (rate_limit == "redis" and auth in {"jwt", "custom"}) + if rate_limit == "redis": + assert config.redis_url == "redis://localhost:6379/0" + else: + assert config.redis_url is None + + store, _ = build_cache_store(config, SimpleNamespace(REDIS_URL=None)) + assert isinstance(store, RedisCacheStore if rate_limit == "redis" else InMemoryCacheStore) + asyncio.run(store.aclose()) diff --git a/tests/cli/test_init_prod.py b/tests/cli/test_init_prod.py index 374a335e..33e17226 100644 --- a/tests/cli/test_init_prod.py +++ b/tests/cli/test_init_prod.py @@ -93,7 +93,9 @@ def test_production_template_defaults_to_recommended_setup(tmp_path: Path) -> No assert config["auth"] == "jwt" assert config["authorization"] == "ownership" assert config["rate_limit"]["enabled"] is True - assert config["rate_limit"]["backend"] == "redis" + assert "backend" not in config["rate_limit"] + assert config["redis"] == "${REDIS_URL}" + assert config["token_revocation"] is True agent_py = (target / "graph" / "agent.py").read_text() assert "PgCheckpointer(" in agent_py diff --git a/tests/cli/test_production_readiness.py b/tests/cli/test_production_readiness.py index 8ae1d628..66170112 100644 --- a/tests/cli/test_production_readiness.py +++ b/tests/cli/test_production_readiness.py @@ -21,15 +21,14 @@ REDIS_RATE_LIMIT = { "enabled": True, - "backend": "redis", - "redis": {"url": "redis://x"}, "by": "user", } READY = { "agent": "graph.react:app", "auth": "jwt", + "redis": "redis://x", "rate_limit": REDIS_RATE_LIMIT, - "token_revocation": "redis", + "token_revocation": True, } AUTH_OFF_TEXT = "Auth is off" @@ -63,7 +62,37 @@ def test_safe_authorization_does_not_warn(value: str | None) -> None: def test_disabled_rate_limit_warns() -> None: config = {**READY, "rate_limit": {"enabled": False}} - assert _paths(production_readiness_warnings(config)) == ["rate_limit"] + issues = production_readiness_warnings(config) + assert _paths(issues) == ["rate_limit"] + assert "a shared cache store" in issues[0]["message"] + + +def test_token_revocation_without_a_shared_cache_store_warns(monkeypatch) -> None: + monkeypatch.delenv("REDIS_URL", raising=False) + config = {key: value for key, value in READY.items() if key != "redis"} + issues = validate_config(config) + assert _paths(issues) == ["token_revocation"] + assert "10xgraph revoke" in issues[0]["message"] + assert not _paths(validate_config({**config, "cache_store": "graph.cache:store"})) + + +@pytest.mark.parametrize("value", ["redis", "memory", "auth.revoke:Backend"]) +def test_old_token_revocation_strings_are_errors_naming_the_cache_store(value: str) -> None: + issues = validate_config({**READY, "token_revocation": value}) + errors = [i for i in issues if i["level"] == "error"] + assert [i["path"] for i in errors] == ["token_revocation"] + assert "cache_store" in errors[0]["message"] + + +def test_token_revocation_object_form_is_valid() -> None: + value = {"enabled": True, "token_lifetime": 3600, "user_ttl": None} + assert validate_config({**READY, "token_revocation": value}) == [] + + +@pytest.mark.parametrize("value", [False, {"enabled": False}]) +def test_token_revocation_turned_off_warns_with_auth(value: object) -> None: + issues = validate_config({**READY, "token_revocation": value}) + assert _paths(issues) == ["token_revocation"] def test_auth_without_token_revocation_warns() -> None: diff --git a/tests/integration_tests/test_idempotency_key.py b/tests/integration_tests/test_idempotency_key.py index a16e2b77..7ef8c565 100644 --- a/tests/integration_tests/test_idempotency_key.py +++ b/tests/integration_tests/test_idempotency_key.py @@ -1,21 +1,22 @@ """G-107: opt-in Idempotency-Key on invoke, stream and WebSocket runs.""" -# ruff: noqa: S101, PLR2004 - from __future__ import annotations -import asyncio import json +import logging from unittest.mock import AsyncMock, MagicMock import pytest from tenxgraph.core.state import StreamChunk, StreamEvent +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.graph import idempotency from tenxgraph_api.src.app.routers.graph.router import router as graph_router from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInvokeOutputSchema from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService +from tests.unit_tests.test_concurrency_limiter import RecordingStore from .conftest import build_app, make_client, user_headers @@ -26,10 +27,19 @@ @pytest.fixture(autouse=True) -def _clean(): - idempotency._memory.clear() - yield - idempotency._memory.clear() +def _app_logs_reach_caplog(monkeypatch): + """The app's log setup (run by any test that builds the app) stops the app loggers + from propagating; caplog listens on the root logger.""" + for name in ("tenxgraph_api", "10xgraph-api"): + monkeypatch.setattr(logging.getLogger(name), "propagate", True) + + +@pytest.fixture(autouse=True) +def _clean(monkeypatch): + """Fresh in-process records and leases for every test.""" + leases = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(idempotency, "lease_manager", lambda: leases) + return leases def _client(): @@ -144,24 +154,33 @@ async def test_oversized_result_is_marked_not_stored(): @pytest.mark.asyncio -async def test_redis_records_use_the_shared_client(monkeypatch): - store: dict[str, tuple[str, int]] = {} - client = MagicMock() - client.get = AsyncMock(side_effect=lambda name: store.get(name, (None,))[0]) - - async def _set(name, value, ex=None): - store[name] = (value, ex) - - client.set = _set - monkeypatch.setattr(idempotency, "_redis_client", lambda: client) +async def test_records_live_in_the_cache_store(monkeypatch): + store = RecordingStore() + monkeypatch.setattr(idempotency, "lease_manager", lambda: LeaseManager(store)) user = {"user_id": "alice"} run, _ = await idempotency.check(user, "r", "invoke", {}) await run.complete("{}") run.release() - (name, (_, ttl)) = next(iter(store.items())) - assert "idempotency" in name and ":alice:r" in name - assert ttl == 86400 - assert asyncio.iscoroutinefunction(client.get) or client.get.await_count >= 1 + sets = [call for call in store.calls if call[0] == "set"] + assert sets == [("set", "idempotency:alice:r", 86400)] + assert ("acquire", "concurrency:runs:idem:alice:r") in store.calls + assert json.loads(await store.get("idempotency:alice:r"))["fp"] == run.fp + + +@pytest.mark.asyncio +async def test_cache_store_errors_run_without_idempotency(monkeypatch, caplog): + store = RecordingStore() + store.fail = True + monkeypatch.setattr(idempotency, "lease_manager", lambda: LeaseManager(store)) + user = {"user_id": "alice"} + with caplog.at_level(logging.WARNING): + run, record = await idempotency.check(user, "down", "invoke", {}) + assert run is not None and record is None # the request runs + await run.complete("{}") # storing fails too: logged, not raised + run.release() + assert "Idempotency lookup failed; running without it" in caplog.text + assert "Idempotency record could not be stored" in caplog.text + assert "Shared concurrency limit unavailable" in caplog.text def test_ws_frame_carries_the_key(): diff --git a/tests/integration_tests/test_token_revocation_api.py b/tests/integration_tests/test_token_revocation_api.py index 79af1c40..b4acf15c 100644 --- a/tests/integration_tests/test_token_revocation_api.py +++ b/tests/integration_tests/test_token_revocation_api.py @@ -4,12 +4,11 @@ from __future__ import annotations -from typing import Any +import asyncio +from unittest.mock import AsyncMock, MagicMock -from tenxgraph_api.src.app.core.auth.revocation import ( - InMemoryTokenRevocation, - TokenRevocationBackend, -) +from tenxgraph_api.src.app.core.auth.revocation import TokenRevocationConfig, revoke_user +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore from tenxgraph_api.src.app.routers.checkpointer.router import router as checkpointer_router from .conftest import build_app, make_client, user_headers @@ -20,15 +19,18 @@ HTTP_UNAVAILABLE = 503 -def _client(backend: TokenRevocationBackend): - app = build_app(routers=[checkpointer_router], extra_bindings={TokenRevocationBackend: backend}) +def _client(cache: BaseCacheStore): + app = build_app( + routers=[checkpointer_router], + extra_bindings={BaseCacheStore: cache, TokenRevocationConfig: TokenRevocationConfig()}, + ) return make_client(app) def test_revoked_user_is_refused_and_others_are_not(): - backend = InMemoryTokenRevocation() - backend.revoke_user("mallory") # HeaderAuth identities carry no iat -> revoked - client = _client(backend) + cache = InMemoryCacheStore() + asyncio.run(revoke_user(cache, "mallory")) # HeaderAuth identities carry no iat -> revoked + client = _client(cache) assert client.get("/v1/threads", headers=user_headers("alice")).status_code == HTTP_OK resp = client.get("/v1/threads", headers=user_headers("mallory")) @@ -37,11 +39,17 @@ def test_revoked_user_is_refused_and_others_are_not(): def test_revocation_outage_fails_closed(): - class _Down(TokenRevocationBackend): - async def is_revoked(self, identity: dict[str, Any]) -> bool: - raise ConnectionError("redis down") - - client = _client(_Down()) + cache = MagicMock(spec=BaseCacheStore) + cache.get = AsyncMock(side_effect=ConnectionError("redis down")) + client = _client(cache) assert client.get("/v1/threads", headers=user_headers("alice")).status_code == ( HTTP_UNAVAILABLE ) + + +def test_revocation_off_ignores_the_cache_store(): + cache = MagicMock(spec=BaseCacheStore) + cache.get = AsyncMock(side_effect=ConnectionError("redis down")) + app = build_app(routers=[checkpointer_router], extra_bindings={BaseCacheStore: cache}) + client = make_client(app) + assert client.get("/v1/threads", headers=user_headers("alice")).status_code == HTTP_OK diff --git a/tests/unit_tests/auth/test_token_claims.py b/tests/unit_tests/auth/test_token_claims.py index 50f39645..d06067ae 100644 --- a/tests/unit_tests/auth/test_token_claims.py +++ b/tests/unit_tests/auth/test_token_claims.py @@ -18,7 +18,13 @@ ) from tenxgraph_api.src.app.core.auth.base_auth import BaseAuth from tenxgraph_api.src.app.core.auth.jwt_auth import JwtAuth -from tenxgraph_api.src.app.core.auth.revocation import InMemoryTokenRevocation, token_revoked +from tenxgraph_api.src.app.core.auth.revocation import ( + TokenRevocationConfig, + revoke_token, + revoke_user, + token_revoked, +) +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.settings import get_settings from tenxgraph_api.src.app.core.exceptions.user_exception import UserAccountError from tenxgraph_api.src.app.routers.checkpointer.router import router as checkpointer_router @@ -74,19 +80,19 @@ def test_string_user_id_is_unchanged(): @pytest.mark.asyncio async def test_jti_revocation_is_checked_with_an_odd_user_id(): - backend = InMemoryTokenRevocation() - backend.revoke_token("t-1") + store, on = InMemoryCacheStore(), TokenRevocationConfig() + await revoke_token(store, "t-1") for odd in (None, "", 0, ["x"], {"a": 1}): - assert await token_revoked({"user_id": odd, "jti": "t-1"}, backend) is True - assert await token_revoked({"user_id": "", "jti": "other"}, backend) is False + assert await token_revoked({"user_id": odd, "jti": "t-1"}, store, on) is True + assert await token_revoked({"user_id": "", "jti": "other"}, store, on) is False @pytest.mark.asyncio async def test_user_revocation_still_applies_to_a_valid_user_id(): - backend = InMemoryTokenRevocation() - backend.revoke_user("alice", before=100) - assert await token_revoked({"user_id": "alice", "iat": 50}, backend) is True - assert await token_revoked({"user_id": "bob", "iat": 50}, backend) is False + store, on = InMemoryCacheStore(), TokenRevocationConfig() + await revoke_user(store, "alice", before=100) + assert await token_revoked({"user_id": "alice", "iat": 50}, store, on) is True + assert await token_revoked({"user_id": "bob", "iat": 50}, store, on) is False # --------------------------------------------------------------------------- diff --git a/tests/unit_tests/cache_store/__init__.py b/tests/unit_tests/cache_store/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit_tests/cache_store/test_cache_store_contract.py b/tests/unit_tests/cache_store/test_cache_store_contract.py new file mode 100644 index 00000000..954f6864 --- /dev/null +++ b/tests/unit_tests/cache_store/test_cache_store_contract.py @@ -0,0 +1,23 @@ +"""The public cache store contract, run against both built-in stores. + +``RedisCacheStore`` runs only when ``TEST_REDIS_URL`` points at a Redis to use (a spare DB +number; the checks write keys under ``contract:`` and let them expire). +""" + +import os + +import pytest + +from tenxgraph_api import InMemoryCacheStore, RedisCacheStore +from tenxgraph_api.testing.cache_store_contract import CacheStoreContract + + +class TestInMemoryCacheStoreContract(CacheStoreContract): + def make_store(self): + return InMemoryCacheStore() + + +@pytest.mark.skipif(not os.environ.get("TEST_REDIS_URL"), reason="TEST_REDIS_URL is not set") +class TestRedisCacheStoreContract(CacheStoreContract): + def make_store(self): + return RedisCacheStore.from_url(os.environ["TEST_REDIS_URL"]) diff --git a/tests/unit_tests/cache_store/test_memory_cache_store.py b/tests/unit_tests/cache_store/test_memory_cache_store.py new file mode 100644 index 00000000..bd98c0ce --- /dev/null +++ b/tests/unit_tests/cache_store/test_memory_cache_store.py @@ -0,0 +1,219 @@ +"""InMemoryCacheStore: caps, eviction, lease protection and the sweep.""" + +import asyncio +import logging + +import pytest + +from tenxgraph_api.src.app.core.cache_store import memory_store +from tenxgraph_api.src.app.core.cache_store.memory_store import InMemoryCacheStore + + +@pytest.fixture(autouse=True) +def _app_logs_reach_caplog(monkeypatch): + """The app's log setup (run by any test that builds the app) stops ``tenxgraph_api`` + from propagating; caplog listens on the root logger.""" + monkeypatch.setattr(logging.getLogger("tenxgraph_api"), "propagate", True) + + +@pytest.mark.asyncio +async def test_slot_cap_evicts_least_recently_used_unprotected_key(caplog): + store = InMemoryCacheStore(max_slot_keys=3) + for name in ("a", "b", "c"): + assert (await store.acquire(f"ratelimit:{name}", 5, 30)).granted + await store.acquire("ratelimit:a", 5, 30) # a is now the most recently used + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.cache_store"): + await store.acquire("ratelimit:d", 5, 30) + assert set(store._slots) == {"ratelimit:a", "ratelimit:c", "ratelimit:d"} + assert "slot cap (3) reached" in caplog.text + + +@pytest.mark.asyncio +async def test_live_leases_are_never_evicted(caplog): + store = InMemoryCacheStore(max_slot_keys=2) + lock = await store.acquire("concurrency:runs:thread:1", 1, 30) + cap = await store.acquire("concurrency:runs:user:u", 10, 30) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.cache_store"): + await store.acquire("ratelimit:x", 5, 30) + # Both leases survive; the table grows past the cap instead. + assert len(store._slots) == 3 + assert "only live leases are left" in caplog.text + assert not (await store.acquire("concurrency:runs:thread:1", 1, 30)).granted + assert await store.renew("concurrency:runs:thread:1", lock.slot_id, 30) + assert await store.renew("concurrency:runs:user:u", cap.slot_id, 30) + # The next new key evicts the unprotected rate-limit entry, not a lease. + await store.acquire("ratelimit:y", 5, 30) + assert "ratelimit:x" not in store._slots + assert "concurrency:runs:thread:1" in store._slots + + +@pytest.mark.asyncio +async def test_expired_lease_is_evicted_first(): + store = InMemoryCacheStore(max_slot_keys=2) + await store.acquire("concurrency:old", 1, 0.05) + await store.acquire("ratelimit:r", 5, 30) + await asyncio.sleep(0.1) + await store.acquire("ratelimit:new", 5, 30) + assert set(store._slots) == {"ratelimit:r", "ratelimit:new"} + + +@pytest.mark.asyncio +async def test_value_cap_evicts_lru_but_keeps_revocations(caplog): + store = InMemoryCacheStore(max_values=3) + await store.set("revoked:jti:t1", b"1", 30) + await store.set("idempotency:a", b"{}", 30) + await store.set("idempotency:b", b"{}", 30) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.cache_store"): + await store.set("idempotency:c", b"{}", 30) + assert await store.get("revoked:jti:t1") == b"1" + assert await store.get("idempotency:a") is None + assert await store.get("idempotency:c") == b"{}" + assert "key cap (3) reached" in caplog.text + + +@pytest.mark.asyncio +async def test_value_cap_grows_when_only_protected_keys_are_left(caplog): + store = InMemoryCacheStore(max_values=2) + await store.set("revoked:jti:1", b"1", 30) + await store.set("revoked:jti:2", b"1", 30) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.cache_store"): + await store.set("revoked:jti:3", b"1", 30) + assert len(store._values) == 3 + assert "only protected keys are left" in caplog.text + + +@pytest.mark.asyncio +async def test_overwriting_a_key_at_the_cap_evicts_nothing(): + store = InMemoryCacheStore(max_values=2) + await store.set("idempotency:a", b"1", 30) + await store.set("idempotency:b", b"1", 30) + await store.set("idempotency:a", b"2", 30) + assert await store.get("idempotency:b") == b"1" + + +@pytest.mark.asyncio +async def test_cap_warning_is_rate_limited(caplog): + store = InMemoryCacheStore(max_values=1) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.cache_store"): + for n in range(5): + await store.set(f"idempotency:{n}", b"1", 30) + assert caplog.text.count("key cap (1) reached") == 1 + + +@pytest.mark.asyncio +async def test_denied_acquire_on_a_missing_key_creates_nothing(): + store = InMemoryCacheStore() + result = await store.acquire("ratelimit:none", 0, 30) + assert not result.granted and result.count == 0 and result.oldest_expiry is None + assert "ratelimit:none" not in store._slots + + +@pytest.mark.asyncio +async def test_release_of_last_slot_drops_the_key(): + store = InMemoryCacheStore() + slot = await store.acquire("concurrency:k", 1, 30) + await store.release("concurrency:k", slot.slot_id) + assert "concurrency:k" not in store._slots + + +@pytest.mark.asyncio +async def test_sweep_removes_expired_entries(monkeypatch): + monkeypatch.setattr(InMemoryCacheStore, "SWEEP_BATCH", 2) + store = InMemoryCacheStore() + for n in range(5): + await store.acquire(f"ratelimit:{n}", 5, 0.05) + await store.set(f"idempotency:{n}", b"1", 0.05) + live = await store.acquire("concurrency:live", 1, 30) + await store.set("idempotency:live", b"1", None) + await asyncio.sleep(0.1) + await store._sweep() + assert set(store._slots) == {"concurrency:live"} + assert set(store._values) == {"idempotency:live"} + assert live.granted + + +@pytest.mark.asyncio +async def test_sweep_is_scheduled_by_operation_count(monkeypatch): + monkeypatch.setattr(InMemoryCacheStore, "SWEEP_EVERY_OPS", 3) + store = InMemoryCacheStore() + await store.set("idempotency:x", b"1", 0.01) + await asyncio.sleep(0.05) + for _ in range(3): + await store.get("idempotency:other") + task = store._sweep_task + assert task is not None + await task + assert "idempotency:x" not in store._values + await store.aclose() + assert store._sweep_task is None + + +@pytest.mark.asyncio +async def test_aclose_cancels_a_running_sweep(): + store = InMemoryCacheStore() + gate = asyncio.Event() + + async def slow_sweep(): + await gate.wait() + + store._sweep_task = asyncio.get_running_loop().create_task(slow_sweep()) + await asyncio.sleep(0) + await store.aclose() + assert store._sweep_task is None + + +@pytest.mark.asyncio +async def test_if_greater_overwrites_a_stored_non_number(): + store = InMemoryCacheStore() + await store.set("revoked:user:u", b"junk", 30) + assert await store.set("revoked:user:u", b"5", 30, "if_greater") + assert await store.get("revoked:user:u") == b"5" + + +@pytest.mark.asyncio +async def test_if_greater_writes_normalised_text(): + store = InMemoryCacheStore() + assert await store.set("revoked:user:u", b" 7.5 ", None, "if_greater") + assert await store.get("revoked:user:u") == b"7.5" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ttl", [0, -1, float("inf"), float("nan")]) +async def test_bad_ttl_is_rejected(ttl): + store = InMemoryCacheStore() + with pytest.raises(ValueError): + await store.acquire("ratelimit:k", 1, ttl) + with pytest.raises(ValueError): + await store.renew("ratelimit:k", "id", ttl) + with pytest.raises(ValueError): + await store.set("k", b"1", ttl) + + +@pytest.mark.asyncio +async def test_unknown_set_mode_is_rejected(): + store = InMemoryCacheStore() + with pytest.raises(ValueError): + await store.set("k", b"1", None, "sometimes") # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_delete_removes_a_slot_set_too(): + store = InMemoryCacheStore() + await store.acquire("concurrency:k", 1, 30) + await store.delete("concurrency:k") + assert (await store.acquire("concurrency:k", 1, 30)).granted + + +@pytest.mark.asyncio +async def test_many_renewals_keep_the_heap_small(): + store = InMemoryCacheStore() + slot = await store.acquire("concurrency:k", 1, 30) + for _ in range(200): + assert await store.renew("concurrency:k", slot.slot_id, 30) + assert len(store._slots["concurrency:k"].heap) < 40 + + +def test_defaults_match_the_old_caps(): + assert memory_store.DEFAULT_MAX_SLOT_KEYS == 50_000 + assert memory_store.DEFAULT_MAX_VALUES == 10_000 # the old in-memory idempotency cap + assert InMemoryCacheStore().describe() is None diff --git a/tests/unit_tests/cache_store/test_redis_cache_store.py b/tests/unit_tests/cache_store/test_redis_cache_store.py new file mode 100644 index 00000000..98ece8f5 --- /dev/null +++ b/tests/unit_tests/cache_store/test_redis_cache_store.py @@ -0,0 +1,273 @@ +"""RedisCacheStore without a server: key layout, commands sent, replies, ownership. + +The Lua scripts themselves run only against a real Redis (the contract suite with +``TEST_REDIS_URL``). +""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from tenxgraph_api.src.app.core.auth import revocation +from tenxgraph_api.src.app.core.auth.ownership_resolver import ThreadOwnershipResolver +from tenxgraph_api.src.app.core.cache_store import redis_store +from tenxgraph_api.src.app.core.cache_store.redis_store import RedisCacheStore +from tenxgraph_api.src.app.core.config import settings as settings_module +from tenxgraph_api.src.app.core.redis_client import key_part, prefixed + + +class NoScriptError(Exception): + pass + + +def _client(): + client = MagicMock() + for name in ("ping", "time", "evalsha", "eval", "zrem", "get", "set", "delete", "aclose"): + setattr(client, name, AsyncMock()) + return client + + +def _store(client=None, prefix=None): + return RedisCacheStore(client=client or _client(), prefix=prefix) + + +@pytest.fixture +def deploy_prefix(monkeypatch): + real = settings_module.get_settings() + fake = SimpleNamespace(**{**real.model_dump(), "REDIS_KEY_PREFIX": "dep"}) + monkeypatch.setattr(settings_module, "get_settings", lambda: fake) + return "dep" + + +# Key layout + + +@pytest.mark.parametrize("prefix", [None, "dep"]) +def test_keys_match_the_layout_used_before_the_cache_store(request, prefix): + """Old and new pods must share locks, records and revocations during a rolling deploy.""" + if prefix: + request.getfixturevalue("deploy_prefix") + store = RedisCacheStore(client=_client()) + + # RedisConcurrencyLimiter: f"{prefixed('10xgraph:concurrency')}:{lease key}" + for lease in ("runs:thread:42", "runs:user:u%3A1", "runs:idem:u:k", "ws:all", "ws:user:u"): + assert ( + store.redis_key(f"concurrency:{lease}") == f"{prefixed('10xgraph:concurrency')}:{lease}" + ) + + # The ownership resolver used f"{prefixed('af:authz:owner')}:id:" and ":tomb:". + thread = "th:1" + owner_ns = prefixed("af:authz:owner") + owner_key = ThreadOwnershipResolver._owner_key(thread) + assert store.redis_key(owner_key) == f"{owner_ns}:id:{key_part(thread)}" + tomb_key = ThreadOwnershipResolver._tombstone_key(thread) + assert store.redis_key(tomb_key) == f"{owner_ns}:tomb:{key_part(thread)}" + + # Token revocation used f"{prefixed('10xgraph:revoked')}:jti:" and ":user:". + revoked_ns = prefixed("10xgraph:revoked") + assert store.redis_key(revocation.jti_key("j:1")) == f"{revoked_ns}:jti:{key_part('j:1')}" + assert store.redis_key(revocation.user_key("u:1")) == f"{revoked_ns}:user:{key_part('u:1')}" + + # routers/graph/idempotency.py: f"{prefixed('10xgraph:idempotency')}:{ident}" + ident = f"{key_part('u1')}:{key_part('key:1')}" + assert store.redis_key(f"idempotency:{ident}") == f"{prefixed('10xgraph:idempotency')}:{ident}" + + lead = f"{prefix}:" if prefix else "" + assert store.redis_key("ratelimit:ip:1.2.3.4") == f"{lead}10xgraph:ratelimit2:ip:1.2.3.4" + assert store.redis_key("other:x") == f"{lead}10xgraph:cache:other:x" + + +def test_explicit_prefix_overrides_the_setting(deploy_prefix): + assert _store(prefix="").redis_key("concurrency:a") == "10xgraph:concurrency:a" + assert _store(prefix="p").redis_key("owner:id:a") == "p:af:authz:owner:id:a" + + +def test_unknown_and_bare_keys_cannot_collide_with_feature_keys(): + store = _store(prefix="") + assert store.redis_key("plain") == "10xgraph:cache:plain" + assert store.redis_key("concurrency") == "10xgraph:cache:concurrency" + assert store.redis_key("cache:concurrency:a") == "10xgraph:cache:cache:concurrency:a" + assert store.redis_key("concurrency:a") != store.redis_key("cache:concurrency:a") + + +# Construction, describe, aclose + + +def test_needs_exactly_one_of_url_or_client(): + with pytest.raises(ValueError): + RedisCacheStore() + with pytest.raises(ValueError): + RedisCacheStore("redis://localhost", client=_client()) + + +def test_describe_never_shows_credentials(): + store = RedisCacheStore.from_url("redis://user:s3cret@cache.internal:6380/2") + assert store.describe() == "cache.internal:6380" + assert "s3cret" not in (store.describe() or "") + assert RedisCacheStore("rediss://:pw@[::1]:6379/0").describe() == "[::1]:6379" + assert RedisCacheStore("unix:///tmp/redis.sock").describe() is None + + +def test_describe_for_a_passed_client_uses_its_pool(): + client = _client() + client.connection_pool.connection_kwargs = {"host": "h", "port": 7000, "password": "pw"} + assert _store(client).describe() == "h:7000" + client.connection_pool.connection_kwargs = {} + assert _store(client).describe() is None + + +@pytest.mark.asyncio +async def test_aclose_closes_only_a_client_it_created(monkeypatch): + created = _client() + monkeypatch.setattr(redis_store, "create_redis", lambda url: created) + await RedisCacheStore("redis://localhost:6379").aclose() + created.aclose.assert_awaited_once() + + passed = _client() + await RedisCacheStore(client=passed).aclose() + passed.aclose.assert_not_awaited() + + +# Commands + + +@pytest.mark.asyncio +async def test_now_uses_the_server_clock(): + client = _client() + client.time.return_value = (1_700_000_000, 250_000) + assert await _store(client).now() == 1_700_000_000.25 + + +@pytest.mark.asyncio +async def test_aping_sends_ping(): + client = _client() + await _store(client).aping() + client.ping.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_acquire_decodes_a_grant(): + client = _client() + client.evalsha.return_value = [1, 2, b"1700000000.500000"] + result = await _store(client, prefix="").acquire("concurrency:runs:thread:1", 3, 60) + assert result.granted and result.count == 2 + assert result.oldest_expiry == 1_700_000_000.5 + assert result.slot_id is not None + sha, numkeys, key, limit, ttl, slot_id, margin = client.evalsha.await_args.args + assert sha == redis_store._ACQUIRE.sha and numkeys == 1 + assert key == "10xgraph:concurrency:runs:thread:1" + assert (limit, ttl, slot_id, margin) == (3, "60.0", result.slot_id, 1000) + + +@pytest.mark.asyncio +async def test_acquire_decodes_a_denial_on_an_empty_set(): + client = _client() + client.evalsha.return_value = [0, 0, b""] + result = await _store(client).acquire("ratelimit:k", 0, 60) + assert not result.granted and result.count == 0 + assert result.oldest_expiry is None and result.slot_id is None + + +@pytest.mark.asyncio +async def test_script_is_loaded_when_the_server_does_not_have_it(): + client = _client() + client.evalsha.side_effect = NoScriptError("NOSCRIPT No matching script") + client.eval.return_value = 1 + assert await _store(client).renew("concurrency:k", "slot", 60) + assert client.eval.await_args.args[0] == redis_store._RENEW_LUA + + +@pytest.mark.asyncio +async def test_other_script_errors_are_raised(): + client = _client() + client.evalsha.side_effect = ConnectionError("down") + with pytest.raises(ConnectionError): + await _store(client).acquire("concurrency:k", 1, 60) + client.eval.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_renew_reports_a_missing_slot(): + client = _client() + client.evalsha.return_value = 0 + assert not await _store(client).renew("concurrency:k", "gone", 60) + + +@pytest.mark.asyncio +async def test_release_is_zrem(): + client = _client() + await _store(client, prefix="").release("concurrency:k", "slot") + client.zrem.assert_awaited_once_with("10xgraph:concurrency:k", "slot") + + +@pytest.mark.asyncio +async def test_set_modes_send_the_right_commands(): + client = _client() + store = _store(client, prefix="") + assert await store.set("idempotency:a", b"v", 2.5) + client.set.assert_awaited_with("10xgraph:idempotency:a", b"v", px=2500) + assert await store.set("idempotency:a", b"v", None) + client.set.assert_awaited_with("10xgraph:idempotency:a", b"v", px=None) + + client.set.return_value = None + assert not await store.set("idempotency:a", b"v", 1, "if_absent") + client.set.assert_awaited_with("10xgraph:idempotency:a", b"v", px=1000, nx=True) + client.set.return_value = True + assert await store.set("idempotency:a", b"v", 1, "if_absent") + + client.evalsha.return_value = 1 + assert await store.set("revoked:user:u", b"1700000000.000001", 30, "if_greater") + args = client.evalsha.await_args.args + assert args[0] == redis_store._IF_GREATER.sha + assert args[2:] == ("10xgraph:revoked:user:u", "1700000000.000001", "30000") + client.evalsha.return_value = 0 + assert not await store.set("revoked:user:u", b"1", None, "if_greater") + assert client.evalsha.await_args.args[4] == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bad", [b"abc", b"", b"0x10", b"1_000", b"inf", b"nan", b"1e999"]) +async def test_if_greater_rejects_non_decimal_values(bad): + client = _client() + with pytest.raises(ValueError): + await _store(client).set("revoked:user:u", bad, None, "if_greater") + client.evalsha.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ttl_and_mode_are_validated_before_any_command(): + client = _client() + store = _store(client) + with pytest.raises(ValueError): + await store.acquire("concurrency:k", 1, 0) + with pytest.raises(ValueError): + await store.set("k", b"v", -1) + with pytest.raises(ValueError): + await store.set("k", b"v", 1, "sometimes") # type: ignore[arg-type] + client.evalsha.assert_not_awaited() + client.set.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_get_returns_bytes_from_a_decoding_client(): + client = _client() + client.get.return_value = "text" + assert await _store(client).get("idempotency:a") == b"text" + client.get.return_value = None + assert await _store(client).get("idempotency:a") is None + + +@pytest.mark.asyncio +async def test_delete_is_del(): + client = _client() + await _store(client, prefix="").delete("owner:id:t") + client.delete.assert_awaited_once_with("af:authz:owner:id:t") + + +def test_scripts_touch_one_key_and_use_the_server_clock(): + for script in (redis_store._ACQUIRE_LUA, redis_store._RENEW_LUA): + assert "redis.call('TIME')" in script + assert "KEYS[2]" not in script + assert "'XX'" in redis_store._RENEW_LUA + assert "KEYS[2]" not in redis_store._IF_GREATER_LUA diff --git a/tests/unit_tests/test_closing_and_cancel.py b/tests/unit_tests/test_closing_and_cancel.py index a5b2b130..0c7386ba 100644 --- a/tests/unit_tests/test_closing_and_cancel.py +++ b/tests/unit_tests/test_closing_and_cancel.py @@ -1,7 +1,5 @@ """A run refused while the graph closes (BF3), client disconnect (BF6), cancelled run (BF7).""" -# ruff: noqa: S101 - import asyncio import json from types import SimpleNamespace @@ -14,8 +12,9 @@ from tenxgraph.storage.checkpointer import InMemoryCheckpointer from tenxgraph.utils.constants import END -from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.graph import run_limits, thread_lock from tenxgraph_api.src.app.routers.graph.router import ( WS_SERVICE_RESTART, @@ -33,9 +32,9 @@ @pytest.fixture(autouse=True) def limiters(monkeypatch): - locks, slots = MemoryConcurrencyLimiter(), MemoryConcurrencyLimiter() - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: locks) - monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: slots) + locks, slots = LeaseManager(InMemoryCacheStore()), LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: locks) + monkeypatch.setattr(run_limits, "lease_manager", lambda: slots) monkeypatch.setattr(thread_lock, "THREAD_BUSY_WAIT_SECONDS", 0.3) return SimpleNamespace(locks=locks, slots=slots) diff --git a/tests/unit_tests/test_concurrency_limiter.py b/tests/unit_tests/test_concurrency_limiter.py index 3b6a61e6..432f64ae 100644 --- a/tests/unit_tests/test_concurrency_limiter.py +++ b/tests/unit_tests/test_concurrency_limiter.py @@ -1,112 +1,118 @@ -"""Shared run / WebSocket caps across workers.""" - -# ruff: noqa: S101 +"""Leases on the cache store: shared run / WebSocket caps, fail open, wiring and selection.""" import asyncio -import time +import logging +import sys +import textwrap from types import SimpleNamespace -from unittest.mock import patch import pytest from fastapi import WebSocketException from injectq import InjectQ -from tenxgraph_api.src.app.core import concurrency -from tenxgraph_api.src.app.core.concurrency import ( - ConcurrencyLimiter, - Lease, - MemoryConcurrencyLimiter, - RedisConcurrencyLimiter, +from tenxgraph_api.src.app.core import leases +from tenxgraph_api.src.app.core.cache_store import ( + BaseCacheStore, + InMemoryCacheStore, + RedisCacheStore, ) +from tenxgraph_api.src.app.core.cache_store import loader as cache_loader +from tenxgraph_api.src.app.core.leases import Lease, LeaseManager + + +@pytest.fixture(autouse=True) +def _app_logs_reach_caplog(monkeypatch): + """The app's log setup (run by any test that builds the app) stops ``tenxgraph_api`` + from propagating; caplog listens on the root logger.""" + monkeypatch.setattr(logging.getLogger("tenxgraph_api"), "propagate", True) -class _FakeRedis: - """Enough of redis.asyncio for sorted-set leases; one instance = one shared server.""" +class RecordingStore(BaseCacheStore): + """An ``InMemoryCacheStore`` that records every call and can be switched to failing. - def __init__(self): - self.zsets: dict[str, dict[str, float]] = {} + One instance stands for one shared server: give it to two ``LeaseManager`` objects to + model two workers. + """ + + def __init__(self) -> None: + self.inner = InMemoryCacheStore() + self.calls: list[tuple] = [] self.fail = False self.closed = False - def _check(self): + def _record(self, *call) -> None: + self.calls.append(call) if self.fail: - raise ConnectionError("redis down") - - async def zrem(self, key, member): - self._check() - self.zsets.get(key, {}).pop(member, None) - - async def zadd(self, key, mapping, xx=False, ch=False): - """Like redis-py: the count of new members (plus changed scores with ``ch``).""" - self._check() - zset = self.zsets.setdefault(key, {}) - count = 0 - for member, score in mapping.items(): - if xx and member not in zset: - continue - if member not in zset or ch and zset[member] != score: - count += 1 - zset[member] = score - return count - - async def expire(self, key, seconds): - self._check() - - async def aclose(self): - self.closed = True + raise ConnectionError("cache store down") + + def ops(self, name: str) -> list[tuple]: + return [call for call in self.calls if call[0] == name] - def pipeline(self, transaction=True): - redis, ops = self, [] + async def aping(self) -> None: + self._record("aping") - class _Pipe: - def zremrangebyscore(self, key, low, high): - ops.append(("zrem_range", key, high)) + async def now(self) -> float: + self._record("now") + return await self.inner.now() - def zadd(self, key, mapping): - ops.append(("zadd", key, mapping)) + async def acquire(self, key, limit, ttl): + self._record("acquire", key) + return await self.inner.acquire(key, limit, ttl) - def zcard(self, key): - ops.append(("zcard", key)) + async def renew(self, key, slot_id, ttl): + self._record("renew", key) + return await self.inner.renew(key, slot_id, ttl) - def expire(self, key, seconds): - ops.append(("expire", key)) + async def release(self, key, slot_id): + self._record("release", key) + await self.inner.release(key, slot_id) + + async def get(self, key): + self._record("get", key) + return await self.inner.get(key) + + async def set(self, key, value, ttl, mode="always"): + self._record("set", key, ttl) + return await self.inner.set(key, value, ttl, mode) + + async def delete(self, key): + self._record("delete", key) + await self.inner.delete(key) + + def slots(self, key: str) -> int: + """Live slots held under the store key ``key``.""" + slot_set = self.inner._slots.get(key) + return len(slot_set.expiries) if slot_set is not None else 0 + + async def aclose(self) -> None: + self.closed = True - async def execute(self): - redis._check() - results = [] - for op in ops: - zset = redis.zsets.setdefault(op[1], {}) - if op[0] == "zrem_range": - for member in [m for m, score in zset.items() if score <= op[2]]: - del zset[member] - results.append(None) - elif op[0] == "zadd": - zset.update(op[2]) - results.append(1) - elif op[0] == "zcard": - results.append(len(zset)) - else: - results.append(True) - return results - return _Pipe() +async def _settle(): + """Let scheduled releases run.""" + for _ in range(5): + await asyncio.sleep(0) # --------------------------------------------------------------------------- -# in-process limiter and leases +# leases in one process # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_memory_limiter_counts_per_key(): - limiter = MemoryConcurrencyLimiter() - first = await limiter.acquire("k", 2) - assert await limiter.acquire("k", 2) is not None - assert await limiter.acquire("k", 2) is None - assert await limiter.acquire("other", 2) is not None +async def test_lease_manager_counts_per_key(): + manager = LeaseManager(InMemoryCacheStore()) + first = await manager.acquire("k", 2) + assert await manager.acquire("k", 2) is not None + assert await manager.acquire("k", 2) is None + assert await manager.acquire("other", 2) is not None first.release() first.release() # idempotent - assert limiter.held("k") == 1 + assert manager.held("k") == 1 + # The slot given back is free at once for the next caller in this process. + assert await manager.acquire("k", 2) is not None + assert await manager.acquire("k", 2) is None + await manager.aclose() @pytest.mark.asyncio @@ -129,16 +135,31 @@ async def release(): # pragma: no cover - must not run Lease(release).release() # no running loop: closed, not awaited, no warning +@pytest.mark.asyncio +async def test_scheduled_releases_are_referenced_until_done(): + done = asyncio.Event() + + async def release(): + await asyncio.sleep(0) + done.set() + + Lease(release).release() + assert len(leases._pending_releases) == 1 + await asyncio.wait_for(done.wait(), 1) + await asyncio.sleep(0) + assert len(leases._pending_releases) == 0 + + # --------------------------------------------------------------------------- -# Redis limiter: one cap across workers +# one cap across workers # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_two_workers_share_one_cap(): - redis = _FakeRedis() - worker_a = RedisConcurrencyLimiter(redis, lease_ttl=60) - worker_b = RedisConcurrencyLimiter(redis, lease_ttl=60) + store = RecordingStore() + worker_a = LeaseManager(store, lease_ttl=60) + worker_b = LeaseManager(store, lease_ttl=60) lease_a = await worker_a.acquire("runs:user:alice", 2) lease_b = await worker_b.acquire("runs:user:alice", 2) @@ -146,10 +167,10 @@ async def test_two_workers_share_one_cap(): # A per-process count would allow a third (one each) -- the shared one does not. assert await worker_a.acquire("runs:user:alice", 2) is None assert await worker_b.acquire("runs:user:alice", 2) is None - assert len(redis.zsets["10xgraph:concurrency:runs:user:alice"]) == 2 # backed out + assert store.slots("concurrency:runs:user:alice") == 2 # a denial adds nothing lease_a.release() - await asyncio.sleep(0) # let the async release run + await _settle() assert await worker_b.acquire("runs:user:alice", 2) is not None await worker_a.aclose() await worker_b.aclose() @@ -157,51 +178,99 @@ async def test_two_workers_share_one_cap(): @pytest.mark.asyncio async def test_a_dead_workers_lease_expires(): - redis = _FakeRedis() - limiter = RedisConcurrencyLimiter(redis, lease_ttl=60) - clock = [time.time()] - with patch.object(concurrency.time, "time", lambda: clock[0]): - lease = await limiter.acquire("k", 1) - assert lease is not None - await limiter.aclose() # the worker dies: no refresh, no release - assert await limiter.acquire("k", 1) is None - clock[0] += 61 - assert await limiter.acquire("k", 1) is not None + store = InMemoryCacheStore() + dead = LeaseManager(store, lease_ttl=0.2) + alive = LeaseManager(store, lease_ttl=0.2) + assert await dead.acquire("k", 1) is not None + for task in list(dead._renewers): # the worker dies: no renewal, no release + task.cancel() + assert await alive.acquire("k", 1) is None + await asyncio.sleep(0.25) + assert await alive.acquire("k", 1) is not None + await alive.aclose() @pytest.mark.asyncio -async def test_a_held_lease_is_refreshed(): - redis = _FakeRedis() - limiter = RedisConcurrencyLimiter(redis, lease_ttl=0.06) - lease = await limiter.acquire("k", 1) - first_expiry = next(iter(redis.zsets["10xgraph:concurrency:k"].values())) - await asyncio.sleep(0.05) - assert next(iter(redis.zsets["10xgraph:concurrency:k"].values())) > first_expiry +async def test_a_held_lease_is_renewed_and_released(): + store = RecordingStore() + manager = LeaseManager(store, lease_ttl=0.06) + lease = await manager.acquire("k", 1) + await asyncio.sleep(0.15) # well past the ttl + assert store.ops("renew") + assert store.slots("concurrency:k") == 1 + assert not lease.lost lease.release() - await asyncio.sleep(0.01) - assert redis.zsets["10xgraph:concurrency:k"] == {} - await limiter.aclose() + await _settle() + assert store.slots("concurrency:k") == 0 + renewals = len(store.ops("renew")) + await asyncio.sleep(0.05) + assert len(store.ops("renew")) == renewals # renewal stopped with the release + await manager.aclose() + + +# --------------------------------------------------------------------------- +# fail open +# --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_redis_outage_fails_open(): - redis = _FakeRedis() - redis.fail = True - limiter = RedisConcurrencyLimiter(redis) - lease = await limiter.acquire("k", 1) +async def test_cache_store_outage_fails_open(caplog): + store = RecordingStore() + store.fail = True + manager = LeaseManager(store) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.concurrency"): + lease = await manager.acquire("k", 1) assert lease is not None # allowed, counted in this process only - assert await limiter.acquire("k", 1) is None # still one holder at a time per process + assert "Shared concurrency limit unavailable" in caplog.text + assert await manager.acquire("k", 1) is None # still one holder at a time per process + lease.release() + assert await manager.acquire("k", 1) is not None + await manager.aclose() + + +@pytest.mark.asyncio +async def test_a_fallback_lease_is_released_in_the_fallback(): + store = RecordingStore() + manager = LeaseManager(store) + store.fail = True + lease = await manager.acquire("k", 1) + store.fail = False # the store is back while the lease is held + # This process still holds the slot, so the shared store must not grant a second one. + assert await manager.acquire("k", 1) is None + await _settle() + assert store.slots("concurrency:k") == 0 # the shared slot taken meanwhile was given back lease.release() - assert await limiter.acquire("k", 1) is not None + await _settle() + assert manager.fallback._slots == {} + assert await manager.acquire("k", 1) is not None + await manager.aclose() @pytest.mark.asyncio -async def test_aclose_closes_the_client(): - redis = _FakeRedis() - limiter = RedisConcurrencyLimiter(redis) - await limiter.acquire("k", 5) - await limiter.aclose() - assert redis.closed +async def test_a_shared_lease_counts_against_the_fallback(): + store = RecordingStore() + manager = LeaseManager(store) + lease = await manager.acquire("runs:thread:t1", 1) + store.fail = True # the store goes down while the run holds the lock + assert await manager.acquire("runs:thread:t1", 1) is None # no second run in this process + store.fail = False + lease.release() + await manager.aclose() + + +@pytest.mark.asyncio +async def test_store_failing_mid_run_keeps_the_lease(caplog): + store = RecordingStore() + manager = LeaseManager(store, lease_ttl=0.06) + lease = await manager.acquire("runs:thread:t1", 1) + store.fail = True + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.concurrency"): + await asyncio.sleep(0.05) + assert "Lease refresh failed" in caplog.text + assert not lease.released # the run goes on + lease.release() # the release fails quietly; the slot expires on its own + await _settle() + await manager.aclose() # --------------------------------------------------------------------------- @@ -209,82 +278,244 @@ async def test_aclose_closes_the_client(): # --------------------------------------------------------------------------- -def test_limiter_lookup_prefers_the_bound_one(monkeypatch): +def test_lease_manager_lookup_prefers_the_bound_one(monkeypatch): container = InjectQ() monkeypatch.setattr(InjectQ, "get_instance", classmethod(lambda cls: container)) - assert concurrency.shared_concurrency_limiter() is None - assert concurrency.concurrency_limiter() is concurrency._process_limiter - shared = RedisConcurrencyLimiter(_FakeRedis()) - container.bind_instance(ConcurrencyLimiter, shared) - assert concurrency.concurrency_limiter() is shared - - -def test_loader_binds_a_redis_limiter_when_redis_is_configured(): - from tenxgraph_api.src.app.loader import _bind_shared_concurrency - - container = InjectQ() - _bind_shared_concurrency(container, None) - assert container.try_get(ConcurrencyLimiter) is None - _bind_shared_concurrency(container, "redis://localhost:6379/0") - assert isinstance(container.try_get(ConcurrencyLimiter), RedisConcurrencyLimiter) - - mine = MemoryConcurrencyLimiter() - other = InjectQ() - other.bind_instance(ConcurrencyLimiter, mine) - _bind_shared_concurrency(other, "redis://localhost:6379/0") - assert other.try_get(ConcurrencyLimiter) is mine + assert leases.lease_manager() is leases.process_lease_manager() + store = RecordingStore() + cache_loader.bind_cache_store(container, store) + bound = leases.lease_manager() + assert bound is container.try_get(LeaseManager) + assert bound.cache is store + assert container.try_get(BaseCacheStore) is store @pytest.mark.asyncio -async def test_run_slots_use_the_shared_limiter(monkeypatch): +async def test_run_slots_use_the_lease_manager(monkeypatch): from tenxgraph_api.src.app.core.config.graph_config import RunsConfig from tenxgraph_api.src.app.routers.graph import run_limits - redis = _FakeRedis() - shared = RedisConcurrencyLimiter(redis) - monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: shared) + store = RecordingStore() + manager = LeaseManager(store) + monkeypatch.setattr(run_limits, "lease_manager", lambda: manager) await run_limits.acquire_run_slot({"user_id": "alice"}, RunsConfig(max_concurrent_per_user=1)) - assert len(redis.zsets["10xgraph:concurrency:runs:user:alice"]) == 1 - await shared.aclose() + assert store.slots("concurrency:runs:user:alice") == 1 + await manager.aclose() @pytest.mark.asyncio -async def test_websocket_caps_use_the_shared_limiter(monkeypatch): +async def test_websocket_caps_use_the_lease_manager(monkeypatch): from tenxgraph_api.src.app.core.config.graph_config import WebSocketConfig from tenxgraph_api.src.app.routers.graph import realtime_guard - redis = _FakeRedis() - shared = RedisConcurrencyLimiter(redis) + store = RecordingStore() + manager = LeaseManager(store) monkeypatch.setattr(realtime_guard.keying, "verified_user_id", lambda ws: "alice") ws_config = WebSocketConfig(max_connections=10, max_connections_per_user=1) websocket = SimpleNamespace() - async with realtime_guard._shared_connection_slots(shared, websocket, ws_config): - assert set(redis.zsets) == { - "10xgraph:concurrency:ws:all", - "10xgraph:concurrency:ws:user:alice", - } + async with realtime_guard._connection_slots(manager, websocket, ws_config): + assert set(store.inner._slots) == {"concurrency:ws:all", "concurrency:ws:user:alice"} with pytest.raises(WebSocketException): - async with realtime_guard._shared_connection_slots(shared, websocket, ws_config): + async with realtime_guard._connection_slots(manager, websocket, ws_config): pass - # The refused attempt gave its global slot back (released on the next loop turn). - await asyncio.sleep(0) - assert len(redis.zsets["10xgraph:concurrency:ws:all"]) == 1 - await asyncio.sleep(0) - assert all(not members for members in redis.zsets.values()) - await shared.aclose() + # The refused attempt gave its global slot back. + await _settle() + assert store.slots("concurrency:ws:all") == 1 + await _settle() + assert store.inner._slots == {} + await manager.aclose() @pytest.mark.asyncio -async def test_scheduled_releases_are_referenced_until_done(): - done = asyncio.Event() +async def test_websocket_caps_hold_per_process_without_a_shared_store(monkeypatch): + from tenxgraph_api.src.app.core.config.graph_config import WebSocketConfig + from tenxgraph_api.src.app.routers.graph import realtime_guard - async def release(): - await asyncio.sleep(0) - done.set() + manager = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(realtime_guard.keying, "verified_user_id", lambda ws: None) + ws_config = WebSocketConfig(max_connections=1, max_connections_per_user=1) + async with realtime_guard._connection_slots(manager, SimpleNamespace(), ws_config): + assert manager.held("ws:all") == 1 + with pytest.raises(WebSocketException): + async with realtime_guard._connection_slots(manager, SimpleNamespace(), ws_config): + pass + assert manager._held == {} + await manager.aclose() - Lease(release).release() - assert len(concurrency._pending_releases) == 1 - await asyncio.wait_for(done.wait(), 1) - await asyncio.sleep(0) - assert len(concurrency._pending_releases) == 0 + +# --------------------------------------------------------------------------- +# selection (Section 5.0) +# --------------------------------------------------------------------------- + + +class _Config: + def __init__(self, cache_store=None, redis_url=None): + self.cache_store = cache_store + self.redis_url = redis_url + + +@pytest.fixture +def cache_module(tmp_path, monkeypatch): + """A project module with a ``cache_store`` instance and a factory.""" + (tmp_path / "my_cache.py").write_text( + textwrap.dedent( + """ + from tests.unit_tests.test_concurrency_limiter import RecordingStore + + instance = RecordingStore() + + def factory(): + return RecordingStore() + + not_a_store = object() + """ + ) + ) + monkeypatch.syspath_prepend(str(tmp_path)) + yield "my_cache" + sys.modules.pop("my_cache", None) + + +def test_selection_order(cache_module): + redis_url = "redis://:hunter2@config-host:6380/0" + env = SimpleNamespace(REDIS_URL="redis://:hunter2@env-host:6379/1") + store, source = cache_loader.build_cache_store( + _Config(cache_store=f"{cache_module}:instance", redis_url=redis_url), env + ) + assert isinstance(store, RecordingStore) + assert source == f"from cache_store {cache_module}:instance" + + store, source = cache_loader.build_cache_store(_Config(redis_url=redis_url), env) + assert isinstance(store, RedisCacheStore) and source == "from redis in 10xgraph.json" + assert store.describe() == "config-host:6380" + + store, source = cache_loader.build_cache_store(_Config(), env) + assert isinstance(store, RedisCacheStore) and source == "from REDIS_URL" + assert store.describe() == "env-host:6379" + + store, source = cache_loader.build_cache_store(_Config(), SimpleNamespace(REDIS_URL=None)) + assert isinstance(store, InMemoryCacheStore) + assert source == "no cache_store, redis or REDIS_URL set" + assert cache_loader.build_cache_store(None, SimpleNamespace(REDIS_URL=None))[1] == source + + +def test_cache_store_factory_and_bad_paths(cache_module): + env = SimpleNamespace(REDIS_URL=None) + store, _ = cache_loader.build_cache_store(_Config(cache_store=f"{cache_module}:factory"), env) + assert isinstance(store, RecordingStore) + with pytest.raises(Exception, match="not a BaseCacheStore"): + cache_loader.build_cache_store(_Config(cache_store=f"{cache_module}:not_a_store"), env) + with pytest.raises(Exception, match="missing"): + cache_loader.build_cache_store(_Config(cache_store=f"{cache_module}:missing"), env) + + +def test_graph_config_cache_store_key(tmp_path): + from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + + path = tmp_path / "10xgraph.json" + path.write_text('{"agent": "a:b", "cache_store": "my_cache:instance"}') + assert GraphConfig(str(path)).cache_store == "my_cache:instance" + path.write_text('{"agent": "a:b"}') + assert GraphConfig(str(path)).cache_store is None + path.write_text('{"agent": "a:b", "cache_store": "redis"}') + with pytest.raises(ValueError, match="module:attr"): + _ = GraphConfig(str(path)).cache_store + + +def test_boot_logs_one_info_line_without_credentials(caplog): + container = InjectQ() + env = SimpleNamespace(REDIS_URL="redis://user:hunter2@cache.internal:6379/0") + with caplog.at_level(logging.INFO, logger="tenxgraph_api.cache_store"): + store = cache_loader.load_and_bind_cache_store(container, _Config(), env) + lines = [r.getMessage() for r in caplog.records if r.name == "tenxgraph_api.cache_store"] + assert lines == ["Cache store: RedisCacheStore (from REDIS_URL, cache.internal:6379)"] + assert "hunter2" not in caplog.text and "user" not in lines[0] + assert container.try_get(BaseCacheStore) is store + + caplog.clear() + with caplog.at_level(logging.INFO, logger="tenxgraph_api.cache_store"): + cache_loader.load_and_bind_cache_store(InjectQ(), _Config(), SimpleNamespace()) + assert caplog.records[-1].getMessage() == ( + "Cache store: InMemoryCacheStore (no cache_store, redis or REDIS_URL set)" + ) + + +@pytest.mark.asyncio +async def test_custom_cache_store_backs_locks_caps_and_idempotency(cache_module, monkeypatch): + """A ``cache_store: "module:attr"`` store is what every lease and record goes through.""" + from tenxgraph_api.src.app.core.config.graph_config import RunsConfig + from tenxgraph_api.src.app.routers.graph import idempotency + from tenxgraph_api.src.app.routers.graph.run_guard import acquire_run_guard + + container = InjectQ() + monkeypatch.setattr(InjectQ, "get_instance", classmethod(lambda cls: container)) + store = cache_loader.load_and_bind_cache_store( + container, _Config(cache_store=f"{cache_module}:factory"), SimpleNamespace() + ) + assert isinstance(store, RecordingStore) + + guard = await acquire_run_guard( + user={"user_id": "alice"}, runs=RunsConfig(max_concurrent_per_user=2), thread_id="t1" + ) + run, _ = await idempotency.check({"user_id": "alice"}, "k1", "invoke", {}) + await run.complete("{}") + run.release() + guard.release() + await _settle() + + acquired = {call[1] for call in store.ops("acquire")} + assert acquired == { + "concurrency:runs:thread:t1", + "concurrency:runs:user:alice", + "concurrency:runs:idem:alice:k1", + } + assert store.ops("set") == [("set", "idempotency:alice:k1", 86400)] + assert {call[1] for call in store.ops("release")} == acquired + await container.get(LeaseManager).aclose() + + +@pytest.mark.asyncio +async def test_custom_cache_store_backs_ownership_revocation_and_rate_limits(cache_module): + """The same custom store serves the ownership cache, token revocation and rate limits.""" + from unittest.mock import AsyncMock, MagicMock + + from tenxgraph_api.src.app import loader + from tenxgraph_api.src.app.core.auth import revocation + from tenxgraph_api.src.app.core.auth.authorization import AuthorizationBackend + from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + from tenxgraph_api.src.app.core.middleware.rate_limit import limiter + + container = InjectQ() + store = cache_loader.load_and_bind_cache_store( + container, _Config(cache_store=f"{cache_module}:factory"), SimpleNamespace() + ) + assert isinstance(store, RecordingStore) + + checkpointer = MagicMock() + checkpointer.aget_thread_owner = AsyncMock(return_value="alice") + loader.load_and_bind_authorization(container, "ownership") + authz = container.get(AuthorizationBackend) + authz._checkpointer = checkpointer + assert await authz.authorize({"user_id": "alice"}, "graph", "invoke", resource_id="t1") + await authz.evict("t1", "alice") + + config = MagicMock(spec=GraphConfig) + config.token_revocation = revocation.TokenRevocationConfig() + loader._enable_token_revocation(container, config, True) + settings = container.get(revocation.TokenRevocationConfig) + await revocation.revoke_user(store, "mallory", before=100.0) + bound = limiter.bound_cache_store(container) + assert bound is store + assert await revocation.token_revoked({"user_id": "mallory", "iat": 50}, bound, settings) + + assert (await limiter.check(bound, "ip:1.2.3.4", 5, 60)).allowed + + keys = {call[1] for call in store.calls if len(call) > 1} + assert { + "owner:id:t1", + "owner:tomb:t1", + "revoked:user:mallory", + "ratelimit:ip:1.2.3.4", + } <= keys + assert ("delete", "owner:id:t1") in store.calls diff --git a/tests/unit_tests/test_config_editor.py b/tests/unit_tests/test_config_editor.py index 76d0e9b4..5572de57 100644 --- a/tests/unit_tests/test_config_editor.py +++ b/tests/unit_tests/test_config_editor.py @@ -21,6 +21,7 @@ ) from tenxgraph_api.cli.config_editor.schema import AUTHORIZATION_SCOPES, KNOWN_TOP_LEVEL_KEYS from tenxgraph_api.cli.config_editor.server import TOKEN_HEADER +from tenxgraph_api.src.app.core.config.graph_config import DEFAULT_USER_REVOCATION_TTL def _levels(issues: list[dict], path: str) -> list[str]: @@ -142,6 +143,54 @@ def test_rate_limit_seed_does_not_claim_a_server_default_for_excluded_paths() -> assert _fields()["rate_limit.exclude_paths"]["label"] == "Extra paths to skip" +def test_rate_limit_has_no_backend_fields_and_the_cache_store_is_wired() -> None: + fields = _fields() + assert not [ + path for path in fields if path.startswith(("rate_limit.backend", "rate_limit.redis")) + ] + rate_limit = next(s for s in build_schema()["sections"] if s["id"] == "rate_limit") + assert "backend" not in rate_limit["default"] + assert fields["cache_store"]["group"] == fields["redis"]["group"] == "Cache store" + assert "cache_store" in KNOWN_TOP_LEVEL_KEYS + + +def test_token_revocation_section_writes_true_and_takes_its_settings() -> None: + section = next(s for s in build_schema()["sections"] if s["id"] == "token_revocation") + assert section["key"] == "token_revocation" + assert section["default"] is True + paths = [".".join(f["path"]) for f in section["fields"]] + assert paths == [ + "token_revocation.enabled", + "token_revocation.token_lifetime", + "token_revocation.user_ttl", + ] + assert _fields()["token_revocation.user_ttl"]["default"] == DEFAULT_USER_REVOCATION_TTL + config = {"agent": "a:b", "auth": "jwt", "redis": "redis://x"} + for value in (True, {"token_lifetime": 3600}, {"enabled": True, "user_ttl": None}): + issues = validate_config({**config, "token_revocation": value}) + assert not [i for i in issues if i["path"].startswith("token_revocation")] + + +def test_redis_key_expands_env_vars_and_warns_when_unset(monkeypatch) -> None: + monkeypatch.delenv("AF_TEST_REDIS_URL", raising=False) + issues = validate_config({"agent": "a:b", "redis": "${AF_TEST_REDIS_URL}"}) + assert _levels(issues, "redis") == ["warning"] + monkeypatch.setenv("AF_TEST_REDIS_URL", "redis://x") + assert ( + _levels(validate_config({"agent": "a:b", "redis": "${AF_TEST_REDIS_URL}"}), "redis") == [] + ) + + +def test_cache_store_must_be_an_import_path() -> None: + assert _levels(validate_config({"agent": "a:b", "cache_store": "nope"}), "cache_store") == [ + "error" + ] + assert ( + _levels(validate_config({"agent": "a:b", "cache_store": "pkg.mod:store"}), "cache_store") + == [] + ) + + def test_unlimited_values_are_accepted_by_the_server() -> None: for value in (0, None): config = { @@ -235,28 +284,16 @@ def test_authorization_variants(authorization: object, path: str, expected: list def test_rate_limit_uses_server_parser() -> None: - issues = validate_config( - {"agent": "graph.react:app", "rate_limit": {"by": "country", "backend": "redis"}} - ) + issues = validate_config({"agent": "graph.react:app", "rate_limit": {"by": "country"}}) assert _levels(issues, "rate_limit") == ["error"] assert "rate_limit.by" in next(i["message"] for i in issues if i["path"] == "rate_limit") -def test_rate_limit_unset_env_var_is_only_a_warning(monkeypatch) -> None: - monkeypatch.delenv("AF_TEST_REDIS_URL", raising=False) - issues = validate_config( - { - "agent": "graph.react:app", - "rate_limit": {"backend": "redis", "redis": {"url": "${AF_TEST_REDIS_URL}"}}, - } - ) - assert _levels(issues, "rate_limit") == [] - assert _levels(issues, "rate_limit.redis.url") == ["warning"] - - -def test_rate_limit_memory_backend_warns() -> None: - issues = validate_config({"agent": "graph.react:app", "rate_limit": {"backend": "memory"}}) - assert _levels(issues, "rate_limit.backend") == ["warning"] +@pytest.mark.parametrize("removed", [{"backend": "memory"}, {"redis": {"url": "redis://x"}}]) +def test_removed_rate_limit_keys_are_errors_naming_the_cache_store(removed) -> None: + issues = validate_config({"agent": "graph.react:app", "rate_limit": removed}) + assert _levels(issues, "rate_limit") == ["error"] + assert "cache_store" in next(i["message"] for i in issues if i["path"] == "rate_limit") def test_websocket_rejects_negative_limit() -> None: diff --git a/tests/unit_tests/test_graph_config.py b/tests/unit_tests/test_graph_config.py index fc5c0a21..edf571f5 100644 --- a/tests/unit_tests/test_graph_config.py +++ b/tests/unit_tests/test_graph_config.py @@ -102,3 +102,17 @@ def test_default_config_path_prefers_10xgraph_json(tmp_path: Path, monkeypatch): (tmp_path / "10xgraph.json").write_text(json.dumps({"agent": "new:app"})) assert default_config_path() == "10xgraph.json" assert GraphConfig().graph_path == "new:app" + + +def test_redis_key_expands_env_vars(tmp_path: Path, monkeypatch): + cfg_path = tmp_path / "cfg.json" + cfg_path.write_text(json.dumps({"agent": "mod:func", "redis": "${TG_TEST_REDIS}"})) + monkeypatch.setenv("TG_TEST_REDIS", "redis://cache:6379/0") + assert GraphConfig(str(cfg_path)).redis_url == "redis://cache:6379/0" + + monkeypatch.delenv("TG_TEST_REDIS") + with pytest.raises(ValueError, match="redis: Unresolved environment variable"): + _ = GraphConfig(str(cfg_path)).redis_url + + cfg_path.write_text(json.dumps({"agent": "mod:func", "redis": ""})) + assert GraphConfig(str(cfg_path)).redis_url is None diff --git a/tests/unit_tests/test_group3a_storage.py b/tests/unit_tests/test_group3a_storage.py index 21137395..77b58002 100644 --- a/tests/unit_tests/test_group3a_storage.py +++ b/tests/unit_tests/test_group3a_storage.py @@ -1,7 +1,5 @@ """Group 3a (storage) API fixes: G-97 write guard, G-123 partial delete, G-40 keys, G-42 boot.""" -# ruff: noqa: S101 - from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -11,9 +9,10 @@ from tenxgraph_api.src.app import main as app_main from tenxgraph_api.src.app.core import redis_client +from tenxgraph_api.src.app.core.auth import revocation from tenxgraph_api.src.app.core.auth.ownership_resolver import ThreadOwnershipResolver -from tenxgraph_api.src.app.core.auth.revocation import RedisTokenRevocation -from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter, RedisConcurrencyLimiter +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, RedisCacheStore +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.checkpointer import router as cp_router from tenxgraph_api.src.app.routers.checkpointer.services.checkpointer_service import ( CheckpointerService, @@ -23,16 +22,14 @@ from tenxgraph_api.src.app.routers.graph.run_limits import run_slot_key from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import FixGraphRequestSchema -from .test_concurrency_limiter import _FakeRedis - USER = {"user_id": "alice"} @pytest.fixture(autouse=True) def fresh_locks(monkeypatch): - limiter = MemoryConcurrencyLimiter() - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: limiter) + limiter = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: limiter) return limiter @@ -149,21 +146,22 @@ def key_prefix(monkeypatch): def test_prefix_is_applied_to_every_api_redis_key(key_prefix): - resolver = ThreadOwnershipResolver(AsyncMock(), redis=MagicMock()) - assert resolver._redis_key("t:1").startswith("prod-eu:af:authz:owner:id:") - assert ":" not in resolver._redis_key("t:1").split(":id:", 1)[1] - assert resolver._tombstone_key("t1").startswith("prod-eu:af:authz:owner:tomb:") - revocation = RedisTokenRevocation(client=MagicMock()) - assert revocation._jti_key("j").startswith("prod-eu:10xgraph:revoked:jti:") - assert revocation._user_key("u:1") == "prod-eu:10xgraph:revoked:user:u%3A1" - limiter = RedisConcurrencyLimiter(_FakeRedis()) - assert limiter._key("runs:thread:t1").startswith("prod-eu:10xgraph:concurrency:") + store = RedisCacheStore(client=MagicMock()) + owner_key = store.redis_key(ThreadOwnershipResolver._owner_key("t:1")) + assert owner_key.startswith("prod-eu:af:authz:owner:id:") + assert ":" not in owner_key.split(":id:", 1)[1] + tomb_key = store.redis_key(ThreadOwnershipResolver._tombstone_key("t1")) + assert tomb_key.startswith("prod-eu:af:authz:owner:tomb:") + assert store.redis_key(revocation.jti_key("j")).startswith("prod-eu:10xgraph:revoked:jti:") + assert store.redis_key(revocation.user_key("u:1")) == "prod-eu:10xgraph:revoked:user:u%3A1" + lease_key = "concurrency:" + thread_lock.thread_lock_key("t1") + assert store.redis_key(lease_key).startswith("prod-eu:10xgraph:concurrency:") def test_no_prefix_keeps_the_existing_keys(): - resolver = ThreadOwnershipResolver(AsyncMock(), redis=MagicMock()) - assert resolver._redis_key("t1") == "af:authz:owner:id:t1" - assert RedisTokenRevocation(client=MagicMock())._jti_key("j") == "10xgraph:revoked:jti:j" + store = RedisCacheStore(client=MagicMock()) + assert store.redis_key(ThreadOwnershipResolver._owner_key("t1")) == "af:authz:owner:id:t1" + assert store.redis_key(revocation.jti_key("j")) == "10xgraph:revoked:jti:j" # --------------------------------------------------------------------------- diff --git a/tests/unit_tests/test_health_check_probes.py b/tests/unit_tests/test_health_check_probes.py index 5b9c4aed..05a91952 100644 --- a/tests/unit_tests/test_health_check_probes.py +++ b/tests/unit_tests/test_health_check_probes.py @@ -5,14 +5,12 @@ from fastapi import FastAPI from fastapi.testclient import TestClient +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, SlotResult from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig from tenxgraph_api.src.app.core.config.setup_middleware import ( HealthCheckAwareTrustedHostMiddleware, ) -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - RateLimitDecision, - RateLimitMiddleware, -) +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware def _app() -> FastAPI: @@ -45,20 +43,19 @@ def test_allowed_host_still_reaches_every_path(): assert client.get("/v1/data").status_code == 200 -class _DenyAll: - """A backend that is down with fail_open=false, or simply exhausted.""" - - async def check(self, key, *, limit, window): - return RateLimitDecision(allowed=False, remaining=0, reset_after=30) +class _DenyAll(InMemoryCacheStore): + """A cache store whose every window is exhausted.""" - async def close(self): - pass + async def acquire(self, key, limit, ttl): + return SlotResult( + granted=False, count=limit, oldest_expiry=await self.now() + 30, slot_id=None + ) def test_probe_is_never_rate_limited(): app = _app() - config = RateLimitConfig.from_dict({"enabled": True, "backend": "memory", "requests": 1}) - app.add_middleware(RateLimitMiddleware, config=config, backend=_DenyAll()) + config = RateLimitConfig.from_dict({"enabled": True, "requests": 1}) + app.add_middleware(RateLimitMiddleware, config=config, cache=_DenyAll()) client = TestClient(app) assert client.get("/ping").status_code == 200 diff --git a/tests/unit_tests/test_identity_cache.py b/tests/unit_tests/test_identity_cache.py index b03cd795..b0a6ea6f 100644 --- a/tests/unit_tests/test_identity_cache.py +++ b/tests/unit_tests/test_identity_cache.py @@ -20,11 +20,9 @@ from tenxgraph_api.src.app.core.auth.identity_cache import recall_identity, remember_identity from tenxgraph_api.src.app.core.auth.jwt_auth import JwtAuth from tenxgraph_api.src.app.core.auth.permissions import RequirePermission +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, RateLimitConfig -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - MemoryRateLimitBackend, - RateLimitMiddleware, -) +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware class _CountingAuthBase(BaseAuth): @@ -128,9 +126,9 @@ def _client(auth: _CountingAuthBase) -> TestClient: app = FastAPI() config = RateLimitConfig.from_dict( - {"enabled": True, "backend": "memory", "requests": 100, "window": 60, "by": "user"} + {"enabled": True, "requests": 100, "window": 60, "by": "user"} ) - app.add_middleware(RateLimitMiddleware, config=config, backend=MemoryRateLimitBackend()) + app.add_middleware(RateLimitMiddleware, config=config, cache=InMemoryCacheStore()) setup_fastapi(container, app) @app.get("/v1/threads") diff --git a/tests/unit_tests/test_loader.py b/tests/unit_tests/test_loader.py index 1bab730b..440efa6a 100644 --- a/tests/unit_tests/test_loader.py +++ b/tests/unit_tests/test_loader.py @@ -365,6 +365,8 @@ async def test_attach_all_modules(): config.thread_name_generator_path = "mod:generator" config.authorization_path = "mod:authorization" config.store_path = None + config.cache_store = None + config.redis_url = None config.remote_tools = [ RemoteToolConfig( node="tools", @@ -438,6 +440,7 @@ class _Bare(BaseCheckpointer): # type: ignore[misc] config.authorization_path = "ownership" config.store_path = None config.redis_url = None + config.cache_store = None config.remote_tools = [] container = InjectQ() @@ -462,11 +465,15 @@ class _Bare(BaseCheckpointer): # type: ignore[misc] # --------------------------------------------------------------------------- -def _warn_container(checkpointer): +def _warn_container(checkpointer, cache_store=None): from unittest.mock import MagicMock + from tenxgraph_api.src.app.core.cache_store import BaseCacheStore + container = MagicMock() - container.try_get.return_value = checkpointer + container.try_get.side_effect = lambda key: ( + cache_store if key is BaseCacheStore else checkpointer + ) return container @@ -479,10 +486,10 @@ def test_warns_for_in_memory_checkpointer_and_missing_redis_in_production(monkey monkeypatch.setattr(loader, "_is_production", lambda: True) with caplog.at_level(logging.WARNING): - loader.warn_per_process_state(_warn_container(InMemoryCheckpointer()), None) + loader.warn_per_process_state(_warn_container(InMemoryCheckpointer())) text = " ".join(r.getMessage() for r in caplog.records) assert "InMemoryCheckpointer" in text - assert "No Redis configured" in text + assert "The cache store is in-memory" in text def test_no_warning_in_single_process_development(monkeypatch, caplog): @@ -495,19 +502,22 @@ def test_no_warning_in_single_process_development(monkeypatch, caplog): monkeypatch.setattr(loader, "_is_production", lambda: False) monkeypatch.delenv("WEB_CONCURRENCY", raising=False) with caplog.at_level(logging.WARNING): - loader.warn_per_process_state(_warn_container(InMemoryCheckpointer()), None) + loader.warn_per_process_state(_warn_container(InMemoryCheckpointer())) assert not caplog.records def test_web_concurrency_above_one_warns_even_in_development(monkeypatch, caplog): import logging + from unittest.mock import MagicMock from tenxgraph_api.src.app import loader + from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, RedisCacheStore monkeypatch.setattr(loader, "_is_production", lambda: False) monkeypatch.setenv("WEB_CONCURRENCY", "4") with caplog.at_level(logging.WARNING): - loader.warn_per_process_state(_warn_container(None), "redis://x") + shared = RedisCacheStore(client=MagicMock()) + loader.warn_per_process_state(_warn_container(None, shared)) assert not caplog.records - loader.warn_per_process_state(_warn_container(None), None) - assert "No Redis configured" in caplog.text + loader.warn_per_process_state(_warn_container(None, InMemoryCacheStore())) + assert "The cache store is in-memory" in caplog.text diff --git a/tests/unit_tests/test_lock_lease_integrity.py b/tests/unit_tests/test_lock_lease_integrity.py index 1feea77f..5af7e548 100644 --- a/tests/unit_tests/test_lock_lease_integrity.py +++ b/tests/unit_tests/test_lock_lease_integrity.py @@ -1,96 +1,120 @@ -"""Lease integrity of the shared limiter behind the thread lock (G-91, G-94).""" - -# ruff: noqa: S101 +"""Lease integrity behind the thread lock (G-91, G-94).""" import asyncio import pytest -from tenxgraph_api.src.app.core import concurrency -from tenxgraph_api.src.app.core.concurrency import RedisConcurrencyLimiter - -from .test_concurrency_limiter import _FakeRedis +from tenxgraph_api.src.app.core import leases +from tenxgraph_api.src.app.core.leases import LeaseManager +from .test_concurrency_limiter import RecordingStore -KEY = "10xgraph:concurrency:runs:thread:t1" +KEY = "concurrency:runs:thread:t1" -class _SlowPipeRedis(_FakeRedis): - """A Redis whose transaction takes a while, so a caller can be cancelled inside it.""" - def pipeline(self, transaction=True): - pipe = super().pipeline(transaction) - execute = pipe.execute +class _SlowAcquireStore(RecordingStore): + """A store that grants the slot, then takes a while to answer, so a caller can be + cancelled after the slot was taken.""" - async def slow_execute(): - results = await execute() # the server applied it ... - await asyncio.sleep(10) # ... but the reply is slow - return results + async def acquire(self, key, limit, ttl): + result = await super().acquire(key, limit, ttl) # the server applied it ... + await asyncio.sleep(0.2) # ... but the reply is slow + return result - pipe.execute = slow_execute - return pipe +class _StrictCloseStore(RecordingStore): + """Refuses calls once closed, like a real client.""" -class _StrictCloseRedis(_FakeRedis): - """Refuses commands once closed, like a real client.""" - - async def zrem(self, key, member): + async def release(self, key, slot_id): await asyncio.sleep(0.01) if self.closed: raise ConnectionError("client closed") - await super().zrem(key, member) + await super().release(key, slot_id) @pytest.mark.asyncio -async def test_redis_outage_keeps_one_run_per_thread_in_this_process(): - redis = _FakeRedis() - redis.fail = True - limiter = RedisConcurrencyLimiter(redis) - first = await limiter.acquire("runs:thread:t1", 1) +async def test_cache_store_outage_keeps_one_run_per_thread_in_this_process(): + store = RecordingStore() + store.fail = True + manager = LeaseManager(store) + first = await manager.acquire("runs:thread:t1", 1) assert first is not None - assert await limiter.acquire("runs:thread:t1", 1) is None + assert await manager.acquire("runs:thread:t1", 1) is None first.release() - assert await limiter.acquire("runs:thread:t1", 1) is not None - await limiter.aclose() + assert await manager.acquire("runs:thread:t1", 1) is not None + await manager.aclose() @pytest.mark.asyncio async def test_lost_lease_is_detected_and_restored(): - redis = _FakeRedis() - limiter = RedisConcurrencyLimiter(redis, lease_ttl=0.06) - lease = await limiter.acquire("runs:thread:t1", 1) + store = RecordingStore() + manager = LeaseManager(store, lease_ttl=0.06) + lease = await manager.acquire("runs:thread:t1", 1) assert lease is not None and not lease.lost - redis.zsets[KEY].clear() # evicted, or expired during a stall + await store.inner.delete(KEY) # evicted, or expired during a stall await asyncio.sleep(0.05) assert lease.lost - assert len(redis.zsets[KEY]) == 1 # held again, so a second run is still refused - assert await limiter.acquire("runs:thread:t1", 1) is None + assert store.slots(KEY) == 1 # held again, so a second run is still refused + other = LeaseManager(store, lease_ttl=0.06) + assert await other.acquire("runs:thread:t1", 1) is None lease.release() - await limiter.aclose() + await asyncio.sleep(0.01) + assert store.slots(KEY) == 0 # the slot taken again is the one released + await manager.aclose() @pytest.mark.asyncio async def test_a_cancelled_waiter_leaves_no_ghost_lease(): - redis = _SlowPipeRedis() - limiter = RedisConcurrencyLimiter(redis, lease_ttl=60) - waiter = asyncio.create_task(limiter.acquire("runs:thread:t1", 1)) + store = _SlowAcquireStore() + manager = LeaseManager(store, lease_ttl=60) + waiter = asyncio.create_task(manager.acquire("runs:thread:t1", 1)) await asyncio.sleep(0.05) + assert store.slots(KEY) == 1 # granted on the server, the reply still on its way waiter.cancel() with pytest.raises(asyncio.CancelledError): await waiter + await asyncio.sleep(0.3) + assert store.slots(KEY) == 0 + assert manager._held == {} + await manager.aclose() + + +@pytest.mark.asyncio +async def test_shutdown_waits_for_a_cancelled_waiters_release(): + store = _SlowAcquireStore() + manager = LeaseManager(store, lease_ttl=60) + waiter = asyncio.create_task(manager.acquire("runs:thread:t1", 1)) await asyncio.sleep(0.05) - assert redis.zsets.get(KEY, {}) == {} - await limiter.aclose() + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + await manager.aclose() # the release of the late grant runs before the store closes + assert store.slots(KEY) == 0 + assert not leases._pending_releases @pytest.mark.asyncio -async def test_aclose_waits_for_pending_releases_before_closing_the_client(): - redis = _StrictCloseRedis() - limiter = RedisConcurrencyLimiter(redis, lease_ttl=60) - lease = await limiter.acquire("runs:thread:t1", 1) +async def test_aclose_waits_for_pending_releases_before_closing_the_store(): + store = _StrictCloseStore() + manager = LeaseManager(store, lease_ttl=60) + lease = await manager.acquire("runs:thread:t1", 1) assert lease is not None lease.release() # scheduled, not yet run - await limiter.aclose() - assert redis.zsets[KEY] == {} - assert redis.closed - assert not concurrency._pending_releases + await manager.aclose() + await store.aclose() # what the lifespan does next + assert store.slots(KEY) == 0 + assert store.closed + assert not leases._pending_releases + assert not manager._renewers + + +@pytest.mark.asyncio +async def test_aclose_stops_renewing_leases_still_held(): + store = RecordingStore() + manager = LeaseManager(store, lease_ttl=0.06) + assert await manager.acquire("runs:thread:t1", 1) is not None + await manager.aclose() + renewals = len(store.ops("renew")) + await asyncio.sleep(0.05) + assert len(store.ops("renew")) == renewals diff --git a/tests/unit_tests/test_middleware_order.py b/tests/unit_tests/test_middleware_order.py index 4cedcba0..370ba639 100644 --- a/tests/unit_tests/test_middleware_order.py +++ b/tests/unit_tests/test_middleware_order.py @@ -26,7 +26,7 @@ def _app(allowed_host: str = "testserver", max_size: int = 10 * 1024 * 1024) -> _env_file=None, ) graph_config = SimpleNamespace( - rate_limit=RateLimitConfig.from_dict({"backend": "memory", "requests": 1, "window": 60}), + rate_limit=RateLimitConfig.from_dict({"requests": 1, "window": 60}), observability=None, ) app = FastAPI() diff --git a/tests/unit_tests/test_ownership_authorization.py b/tests/unit_tests/test_ownership_authorization.py index a34424b4..1e75a313 100644 --- a/tests/unit_tests/test_ownership_authorization.py +++ b/tests/unit_tests/test_ownership_authorization.py @@ -312,6 +312,7 @@ async def test_attach_binds_ownership_when_auth_is_on_in_development(monkeypatch config.authorization_path = None config.store_path = None config.redis_url = None + config.cache_store = None config.remote_tools = [] config.runs = RunsConfig() diff --git a/tests/unit_tests/test_ownership_resolver.py b/tests/unit_tests/test_ownership_resolver.py index a6f0ffb4..c086ad6a 100644 --- a/tests/unit_tests/test_ownership_resolver.py +++ b/tests/unit_tests/test_ownership_resolver.py @@ -1,4 +1,4 @@ -"""Unit tests for ThreadOwnershipResolver (L1 LRU + optional Redis L2).""" +"""Unit tests for ThreadOwnershipResolver (L1 LRU + optional cache store L2).""" from __future__ import annotations @@ -20,20 +20,23 @@ async def __call__(self, thread_id: str): class _FakeRedis: + """A shared cache store (only the key methods the resolver uses).""" + def __init__(self): self.store: dict[str, bytes] = {} self.gets = 0 self.sets = 0 - self.ttls: dict[str, int | None] = {} + self.ttls: dict[str, float | None] = {} async def get(self, key): self.gets += 1 return self.store.get(key) - async def set(self, key, value, ex=None): + async def set(self, key, value, ttl=None, mode="always"): self.sets += 1 - self.ttls[key] = ex - self.store[key] = value.encode() if isinstance(value, str) else value + self.ttls[key] = ttl + self.store[key] = value + return True async def delete(self, key): self.store.pop(key, None) @@ -66,14 +69,14 @@ async def test_negative_result_is_never_cached(): async def test_redis_l2_is_populated_and_promotes_to_l1(): lookup = _CountingLookup({"t1": "alice"}) redis = _FakeRedis() - r = ThreadOwnershipResolver(lookup, redis=redis) + r = ThreadOwnershipResolver(lookup, cache=redis) assert await r.owner_of("t1") == "alice" # DB -> fills L2 + L1 assert redis.sets == 1 # A fresh resolver sharing the same Redis resolves without touching the DB. lookup2 = _CountingLookup({"t1": "alice"}) - r2 = ThreadOwnershipResolver(lookup2, redis=redis) + r2 = ThreadOwnershipResolver(lookup2, cache=redis) assert await r2.owner_of("t1") == "alice" assert lookup2.calls == 0 # served from L2 # And it is now in r2's L1 too (no further Redis reads). @@ -86,11 +89,11 @@ async def test_redis_l2_is_populated_and_promotes_to_l1(): async def test_evict_clears_l1_and_l2(): lookup = _CountingLookup({"t1": "alice"}) redis = _FakeRedis() - r = ThreadOwnershipResolver(lookup, redis=redis) + r = ThreadOwnershipResolver(lookup, cache=redis) await r.owner_of("t1") await r.evict("t1") - assert list(redis.store) == ["af:authz:owner:tomb:t1"] # owner cleared, tombstone left + assert list(redis.store) == ["owner:tomb:t1"] # owner cleared, tombstone left await r.owner_of("t1") assert lookup.calls == 2 # had to re-resolve after eviction @@ -118,14 +121,14 @@ class _BrokenRedis: async def get(self, key): raise RuntimeError("redis down") - async def set(self, key, value, ex=None): + async def set(self, key, value, ttl=None, mode="always"): raise RuntimeError("redis down") async def delete(self, key): raise RuntimeError("redis down") lookup = _CountingLookup({"t1": "alice"}) - r = ThreadOwnershipResolver(lookup, redis=_BrokenRedis()) + r = ThreadOwnershipResolver(lookup, cache=_BrokenRedis()) # Redis failing must not fail the request; falls through to the DB lookup. assert await r.owner_of("t1") == "alice" @@ -166,7 +169,7 @@ async def test_l1_entry_expires_after_its_ttl(monkeypatch): @pytest.mark.asyncio async def test_l2_entries_are_written_with_a_ttl(): redis = _FakeRedis() - r = ThreadOwnershipResolver(_CountingLookup({"t1": "alice"}), redis=redis, l2_ttl=600) + r = ThreadOwnershipResolver(_CountingLookup({"t1": "alice"}), cache=redis, l2_ttl=600) await r.owner_of("t1") assert list(redis.ttls.values()) == [600] @@ -200,11 +203,11 @@ async def test_delete_leaves_a_tombstone_for_one_l1_ttl(): @pytest.mark.asyncio async def test_tombstone_reaches_other_workers_through_redis(): redis = _FakeRedis() - deleting = ThreadOwnershipResolver(_CountingLookup({}), redis=redis, l1_ttl=30.0) - other = ThreadOwnershipResolver(_CountingLookup({}), redis=redis, l1_ttl=30.0) + deleting = ThreadOwnershipResolver(_CountingLookup({}), cache=redis, l1_ttl=30.0) + other = ThreadOwnershipResolver(_CountingLookup({}), cache=redis, l1_ttl=30.0) await deleting.evict("t1") assert await other.recently_deleted("t1") is True - assert redis.ttls["af:authz:owner:tomb:t1"] == 30 + assert redis.ttls["owner:tomb:t1"] == 30 @pytest.mark.asyncio @@ -221,7 +224,7 @@ class _Broken(_FakeRedis): async def get(self, key): raise ConnectionError("down") - r = ThreadOwnershipResolver(_CountingLookup({}), redis=_Broken()) + r = ThreadOwnershipResolver(_CountingLookup({}), cache=_Broken()) assert await r.recently_deleted("t1") is False @@ -259,8 +262,8 @@ async def test_deleting_owner_falls_back_to_the_cached_owner(): @pytest.mark.asyncio async def test_shared_tombstone_carries_the_deleting_owner(): redis = _FakeRedis() - deleting = ThreadOwnershipResolver(_CountingLookup({}), redis=redis) - other = ThreadOwnershipResolver(_CountingLookup({}), redis=redis) + deleting = ThreadOwnershipResolver(_CountingLookup({}), cache=redis) + other = ThreadOwnershipResolver(_CountingLookup({}), cache=redis) await deleting.evict("t1", owner="alice") assert await other.recently_deleted("t1", "alice") is False assert await other.recently_deleted("t1", "bob") is True @@ -273,9 +276,9 @@ async def test_thread_id_cannot_collide_with_a_delete_marker(): """B31: a thread named "deleted:V" or "tomb:V" must not read or write V's delete marker.""" redis = _FakeRedis() r = ThreadOwnershipResolver( - _CountingLookup({"deleted:v": "mallory", "tomb:v": "mallory"}), redis=redis + _CountingLookup({"deleted:v": "mallory", "tomb:v": "mallory"}), cache=redis ) - other = ThreadOwnershipResolver(_CountingLookup({}), redis=redis) + other = ThreadOwnershipResolver(_CountingLookup({}), cache=redis) await r.owner_of("deleted:v") await r.owner_of("tomb:v") @@ -283,3 +286,29 @@ async def test_thread_id_cannot_collide_with_a_delete_marker(): await r.evict("v", "alice") assert await other.owner_of("tomb:v") == "mallory" + + +@pytest.mark.asyncio +async def test_in_memory_cache_store_is_not_a_second_l1(): + from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore + + r = ThreadOwnershipResolver(_CountingLookup({"t1": "alice"}), cache=InMemoryCacheStore()) + assert r._cache is None + assert await r.owner_of("t1") == "alice" + + +@pytest.mark.asyncio +async def test_l2_keys_through_the_redis_cache_store_keep_the_old_layout(): + from unittest.mock import AsyncMock, MagicMock + + from tenxgraph_api.src.app.core.cache_store import RedisCacheStore + + client = MagicMock() + client.get = AsyncMock(return_value=None) + client.set = AsyncMock() + r = ThreadOwnershipResolver( + _CountingLookup({"t:1": "alice"}), cache=RedisCacheStore(client=client), l2_ttl=600 + ) + await r.owner_of("t:1") + client.get.assert_awaited_with("af:authz:owner:id:t%3A1") + assert client.set.await_args.args[0] == "af:authz:owner:id:t%3A1" diff --git a/tests/unit_tests/test_pure_asgi_middleware.py b/tests/unit_tests/test_pure_asgi_middleware.py index 5096ba6d..60f69b7c 100644 --- a/tests/unit_tests/test_pure_asgi_middleware.py +++ b/tests/unit_tests/test_pure_asgi_middleware.py @@ -10,17 +10,15 @@ from fastapi.testclient import TestClient from starlette.middleware.base import BaseHTTPMiddleware +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig from tenxgraph_api.src.app.core.config.setup_middleware import RequestIDMiddleware -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - MemoryRateLimitBackend, - RateLimitMiddleware, -) +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware from tenxgraph_api.src.app.core.middleware.security_headers import SecurityHeadersMiddleware def _rate_limit_config(**overrides) -> RateLimitConfig: - data = {"enabled": True, "backend": "memory", "requests": 5, "window": 60} + data = {"enabled": True, "requests": 5, "window": 60} data.update(overrides) return RateLimitConfig.from_dict(data) @@ -48,9 +46,7 @@ async def body(): return StreamingResponse(body(), media_type="text/plain") - app.add_middleware( - RateLimitMiddleware, config=_rate_limit_config(), backend=MemoryRateLimitBackend() - ) + app.add_middleware(RateLimitMiddleware, config=_rate_limit_config(), cache=InMemoryCacheStore()) app.add_middleware(SecurityHeadersMiddleware) app.add_middleware(RequestIDMiddleware) return app @@ -85,7 +81,7 @@ async def root(): app.add_middleware( RateLimitMiddleware, config=_rate_limit_config(requests=1), - backend=MemoryRateLimitBackend(), + cache=InMemoryCacheStore(), ) app.add_middleware(SecurityHeadersMiddleware) app.add_middleware(RequestIDMiddleware) @@ -108,7 +104,7 @@ async def ws(websocket: WebSocket): await websocket.close() app.add_middleware( - RateLimitMiddleware, config=_rate_limit_config(requests=1), backend=MemoryRateLimitBackend() + RateLimitMiddleware, config=_rate_limit_config(requests=1), cache=InMemoryCacheStore() ) app.add_middleware(SecurityHeadersMiddleware) app.add_middleware(RequestIDMiddleware) diff --git a/tests/unit_tests/test_rate_limit.py b/tests/unit_tests/test_rate_limit.py index 9aa42024..c3402886 100644 --- a/tests/unit_tests/test_rate_limit.py +++ b/tests/unit_tests/test_rate_limit.py @@ -3,23 +3,15 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -from injectq import InjectQ +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, SlotResult from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - BaseRateLimitBackend, - MemoryRateLimitBackend, - RateLimitDecision, - RateLimitMiddleware, - RedisRateLimitBackend, - build_backend, -) +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware, check def _config(**overrides) -> RateLimitConfig: data = { "enabled": True, - "backend": "memory", "requests": 2, "window": 60, "by": "ip", @@ -29,82 +21,87 @@ def _config(**overrides) -> RateLimitConfig: @pytest.mark.asyncio -async def test_memory_backend_enforces_limit(): - backend = MemoryRateLimitBackend() +async def test_check_enforces_limit(): + cache = InMemoryCacheStore() - first = await backend.check("client", limit=2, window=60) - second = await backend.check("client", limit=2, window=60) - third = await backend.check("client", limit=2, window=60) + first = await check(cache, "client", 2, 60) + second = await check(cache, "client", 2, 60) + third = await check(cache, "client", 2, 60) - assert first.allowed is True - assert second.allowed is True + assert (first.allowed, first.remaining, first.reset_after) == (True, 1, 60) + assert (second.allowed, second.remaining) == (True, 0) assert third.allowed is False assert third.remaining == 0 + assert 1 <= third.reset_after <= 60 @pytest.mark.asyncio -async def test_memory_backend_unique_key_cap_evicts(): - backend = MemoryRateLimitBackend(max_unique_keys=2) - - await backend.check("a", limit=10, window=60) - await backend.check("b", limit=10, window=60) - await backend.check("c", limit=10, window=60) - - assert len(backend._buckets) <= 2 - assert "c" in backend._buckets +async def test_check_takes_a_slot_that_is_never_released(): + cache = InMemoryCacheStore() + await check(cache, "client", 2, 60) + result = await cache.acquire("ratelimit:client", 10, ttl=60) + assert result.count == 2 # ours plus the request's, still live @pytest.mark.asyncio -async def test_redis_backend_uses_unique_members_for_same_millisecond(monkeypatch): - calls = [] +async def test_denied_request_waits_for_the_oldest_slot(): + class Full(InMemoryCacheStore): + async def now(self) -> float: + return 1000.0 - async def fake_script(*, keys, args): - calls.append((keys, args)) - return [1, 1, 60] + async def acquire(self, key, limit, ttl): + return SlotResult(granted=False, count=limit, oldest_expiry=1012.2, slot_id=None) - backend = object.__new__(RedisRateLimitBackend) - backend._prefix = "10xgraph:test" - backend._fail_open = True - backend._script = fake_script + decision = await check(Full(), "client", 2, 60) + assert (decision.allowed, decision.remaining, decision.reset_after) == (False, 0, 13) - monkeypatch.setattr("time.time", lambda: 123.456) - await backend.check("client", limit=2, window=60) - await backend.check("client", limit=2, window=60) - - first_member = calls[0][1][3] - second_member = calls[1][1][3] - assert first_member.startswith("123456:") - assert second_member.startswith("123456:") - assert first_member != second_member +@pytest.mark.asyncio +async def test_window_expiry_frees_the_client(): + cache = InMemoryCacheStore() + assert (await check(cache, "client", 1, 0.2)).allowed is True + assert (await check(cache, "client", 1, 0.2)).allowed is False + import asyncio + await asyncio.sleep(0.3) + assert (await check(cache, "client", 1, 0.2)).allowed is True -def test_rate_limit_config_parses_boolean_strings_and_expands_redis_url(monkeypatch): - monkeypatch.setenv("RATE_LIMIT_REDIS_URL", "redis://localhost:6379/4") +def test_rate_limit_config_parses_boolean_strings(): config = RateLimitConfig.from_dict( { "enabled": "true", - "backend": "redis", "requests": 5, "window": 10, "by": "global", "trusted_proxy_headers": "false", "fail_open": "no", - "redis": {"url": "${RATE_LIMIT_REDIS_URL}", "prefix": "10xgraph:test"}, } ) assert config.enabled is True assert config.trusted_proxy_headers is False assert config.fail_open is False - assert config.redis_url == "redis://localhost:6379/4" -def test_rate_limit_config_allows_custom_backend_without_path(): - config = RateLimitConfig.from_dict({"backend": "custom"}) +@pytest.mark.parametrize( + "removed", + [ + {"backend": "memory"}, + {"backend": "redis"}, + {"backend": "custom"}, + {"redis": {"url": "redis://localhost:6379/0"}}, + {"redis": "redis://localhost:6379/0"}, + ], +) +def test_removed_rate_limit_keys_name_the_cache_store(removed): + with pytest.raises(ValueError, match="cache_store") as exc: + RateLimitConfig.from_dict(removed) + assert "`redis`" in str(exc.value) + - assert config.backend == "custom" +def test_rate_limit_fail_open_defaults_to_true(): + assert RateLimitConfig.from_dict({}).fail_open is True def test_rate_limit_config_rejects_invalid_boolean_string(): @@ -114,8 +111,7 @@ def test_rate_limit_config_rejects_invalid_boolean_string(): def test_rate_limit_middleware_ignores_forwarded_for_by_default(): app = FastAPI() - backend = MemoryRateLimitBackend() - app.add_middleware(RateLimitMiddleware, config=_config(), backend=backend) + app.add_middleware(RateLimitMiddleware, config=_config(), cache=InMemoryCacheStore()) @app.get("/") def root(): @@ -129,11 +125,10 @@ def root(): def test_rate_limit_middleware_uses_forwarded_for_when_trusted(): app = FastAPI() - backend = MemoryRateLimitBackend() app.add_middleware( RateLimitMiddleware, config=_config(trusted_proxy_headers=True), - backend=backend, + cache=InMemoryCacheStore(), ) @app.get("/") @@ -148,11 +143,10 @@ def root(): def test_rate_limit_middleware_excludes_paths(): app = FastAPI() - backend = MemoryRateLimitBackend() app.add_middleware( RateLimitMiddleware, config=_config(requests=1, exclude_paths=["/health"]), - backend=backend, + cache=InMemoryCacheStore(), ) @app.get("/health") @@ -164,89 +158,25 @@ def health(): assert client.get("/health").status_code == 200 -@pytest.mark.asyncio -async def test_custom_backend_resolves_from_injectq(): - class MyRateLimitBackend(BaseRateLimitBackend): - async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: - return RateLimitDecision(allowed=True, remaining=limit - 1, reset_after=window) - - async def close(self) -> None: - return None - - container = InjectQ() - custom_backend = MyRateLimitBackend() - container.bind_instance(BaseRateLimitBackend, custom_backend) - - backend = build_backend(_config(backend="custom"), container=container) - - assert backend is custom_backend - assert (await backend.check("client", limit=2, window=60)).allowed is True - +def test_middleware_uses_the_cache_store_bound_after_it_was_added(): + from injectq import InjectQ -def test_custom_backend_requires_injectq_binding(): - with pytest.raises(ValueError, match="BaseRateLimitBackend"): - build_backend(_config(backend="custom"), container=InjectQ()) - - -def test_redis_backend_reuses_injectq_redis_client(): - class FakeRedis: - def register_script(self, script): - async def fake_script(*, keys, args): - return [1, 0, 60] - - return fake_script + from tenxgraph_api.src.app.core.cache_store import BaseCacheStore container = InjectQ() - redis = FakeRedis() - container.bind_instance("redis", redis) - - backend = build_backend( - _config( - backend="redis", - redis={"prefix": "10xgraph:test"}, - ), - container=container, - ) - - assert isinstance(backend, RedisRateLimitBackend) - assert backend._redis is redis - assert backend._close_redis is False - - -def test_redis_backend_requires_url_when_no_injected_client(): - with pytest.raises(ValueError, match="redis.url"): - build_backend(_config(backend="redis"), container=InjectQ()) - - -def test_redis_backend_from_url_builds_and_binds_the_client(): - """A Redis URL with no injected client must build the backend, not crash.""" - from redis.asyncio import Redis - - container = InjectQ() - backend = build_backend( - _config(backend="redis", redis={"url": "redis://localhost:6379/0"}), container=container - ) - - assert isinstance(backend, RedisRateLimitBackend) - assert container.try_get("redis") is backend._redis - assert container.try_get(Redis) is backend._redis - + app = FastAPI() + app.add_middleware(RateLimitMiddleware, config=_config(requests=1), container=container) + cache = InMemoryCacheStore() + container.bind_instance(BaseCacheStore, cache) -def test_redis_backend_from_url_requires_optional_extra(monkeypatch): - monkeypatch.setattr( - "tenxgraph_api.src.app.core.middleware.rate_limit.redis._REDIS_AVAILABLE", - False, - ) - monkeypatch.setattr( - "tenxgraph_api.src.app.core.middleware.rate_limit.redis.AsyncRedis", - None, - ) + @app.get("/") + def root(): + return {"ok": True} - with pytest.raises(ImportError, match=r"10xgraph-api\[redis\]"): - RedisRateLimitBackend.from_url( - redis_url="redis://localhost:6379/0", - prefix="10xgraph:test", - ) + client = TestClient(app) + assert client.get("/").status_code == 200 + assert client.get("/").status_code == 429 + assert cache._slots # the counters live in the bound store def test_rate_limit_middleware_ignores_forwarded_for_from_a_public_peer(): @@ -255,7 +185,7 @@ def test_rate_limit_middleware_ignores_forwarded_for_from_a_public_peer(): app.add_middleware( RateLimitMiddleware, config=_config(trusted_proxy_headers=True), - backend=MemoryRateLimitBackend(), + cache=InMemoryCacheStore(), ) @app.get("/") diff --git a/tests/unit_tests/test_rate_limit_keying.py b/tests/unit_tests/test_rate_limit_keying.py index f43ff195..a18f51f5 100644 --- a/tests/unit_tests/test_rate_limit_keying.py +++ b/tests/unit_tests/test_rate_limit_keying.py @@ -22,9 +22,6 @@ def _cfg(**overrides) -> RateLimitConfig: "requests": 100, "window": 60, "by": "ip", - "backend": "redis", - "redis_url": None, - "redis_prefix": "p", "exclude_paths": (), "trusted_proxy_headers": True, "trusted_proxy_hops": 1, @@ -103,7 +100,7 @@ def test_anonymous_falls_back_to_ip_not_one_shared_bucket(self, verified_as): class TestConfigValidation: def test_by_user_is_accepted(self): - cfg = RateLimitConfig.from_dict({"by": "user", "backend": "redis"}) + cfg = RateLimitConfig.from_dict({"by": "user"}) assert cfg.by == "user" def test_invalid_by_rejected(self): @@ -141,13 +138,12 @@ def test_explicit_list_replaces_the_default(self): def test_parsing_warns_when_headers_are_trusted_without_a_list(self, caplog): with caplog.at_level("WARNING", logger="tenxgraph_api"): - RateLimitConfig.from_dict({"backend": "redis", "trusted_proxy_headers": True}) + RateLimitConfig.from_dict({"trusted_proxy_headers": True}) assert "without rate_limit.trusted_proxies" in caplog.text caplog.clear() with caplog.at_level("WARNING", logger="tenxgraph_api"): RateLimitConfig.from_dict( { - "backend": "redis", "trusted_proxy_headers": True, "trusted_proxies": ["10.0.0.0/8"], } diff --git a/tests/unit_tests/test_rate_limit_memory_lru.py b/tests/unit_tests/test_rate_limit_memory_lru.py index 9754441a..9886812f 100644 --- a/tests/unit_tests/test_rate_limit_memory_lru.py +++ b/tests/unit_tests/test_rate_limit_memory_lru.py @@ -1,4 +1,5 @@ # ruff: noqa: S101, PLR2004, SLF001 +"""Rate limiting on the in-memory cache store: bounded memory with LRU eviction.""" import logging import time @@ -6,45 +7,60 @@ import pytest -from tenxgraph_api.src.app.core.middleware.rate_limit import MemoryRateLimitBackend +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore +from tenxgraph_api.src.app.core.middleware.rate_limit import check -async def _hit(backend: MemoryRateLimitBackend, key: str, window: int = 60): - return await backend.check(key, limit=5, window=window) +async def _hit(cache: InMemoryCacheStore, key: str, window: float = 60): + return await check(cache, key, 5, window) + + +def _clients(cache: InMemoryCacheStore) -> list[str]: + return [key.removeprefix("ratelimit:") for key in cache._slots] @pytest.mark.asyncio async def test_cap_evicts_exactly_the_least_recently_used_key(): - backend = MemoryRateLimitBackend(max_unique_keys=3) + cache = InMemoryCacheStore(max_slot_keys=3) for key in ("a", "b", "c"): - await _hit(backend, key) + await _hit(cache, key) - await _hit(backend, "d") + await _hit(cache, "d") - assert list(backend._buckets) == ["b", "c", "d"] + assert _clients(cache) == ["b", "c", "d"] @pytest.mark.asyncio async def test_recently_used_key_survives_eviction(): - backend = MemoryRateLimitBackend(max_unique_keys=3) + cache = InMemoryCacheStore(max_slot_keys=3) for key in ("a", "b", "c"): - await _hit(backend, key) - await _hit(backend, "a") + await _hit(cache, key) + await _hit(cache, "a") + + await _hit(cache, "d") - await _hit(backend, "d") + assert _clients(cache) == ["c", "a", "d"] + + +@pytest.mark.asyncio +async def test_live_leases_are_not_evicted_for_rate_limit_keys(): + cache = InMemoryCacheStore(max_slot_keys=2) + lease = await cache.acquire("concurrency:runs:thread:t1", 1, ttl=60) + assert lease.granted + for key in ("a", "b", "c"): + await _hit(cache, key) - assert "a" in backend._buckets - assert "b" not in backend._buckets - assert list(backend._buckets) == ["c", "a", "d"] + assert "concurrency:runs:thread:t1" in cache._slots + assert _clients(cache)[-1] == "c" @pytest.mark.asyncio async def test_decisions_keep_sliding_window_values(): - backend = MemoryRateLimitBackend() + cache = InMemoryCacheStore() - first = await backend.check("k", limit=2, window=60) - second = await backend.check("k", limit=2, window=60) - blocked = await backend.check("k", limit=2, window=60) + first = await check(cache, "k", 2, 60) + second = await check(cache, "k", 2, 60) + blocked = await check(cache, "k", 2, 60) assert (first.allowed, first.remaining, first.reset_after) == (True, 1, 60) assert (second.allowed, second.remaining, second.reset_after) == (True, 0, 60) @@ -54,66 +70,50 @@ async def test_decisions_keep_sliding_window_values(): @pytest.mark.asyncio -async def test_sweep_removes_only_stale_keys_from_the_front(): - backend = MemoryRateLimitBackend() - for key in ("old1", "old2", "fresh"): - await _hit(backend, key) - now = time.monotonic() - backend._buckets["old1"].last_seen = now - 1000 - backend._buckets["old2"].last_seen = now - 1000 - - await backend._sweep(60) - - assert list(backend._buckets) == ["fresh"] - - -@pytest.mark.asyncio -async def test_sweep_stops_at_first_fresh_key_without_scanning_the_rest(): - backend = MemoryRateLimitBackend() - for key in ("old", "fresh", "later"): - await _hit(backend, key) - backend._buckets["old"].last_seen = time.monotonic() - 1000 - # A stale entry behind a fresh one is out of order and must not be reached. - backend._buckets["later"].last_seen = time.monotonic() - 1000 +async def test_sweep_removes_only_expired_windows(): + cache = InMemoryCacheStore() + for key in ("old1", "old2"): + await _hit(cache, key, window=0.05) + await _hit(cache, "fresh") + time.sleep(0.1) - await backend._sweep(60) + await cache._sweep() - assert list(backend._buckets) == ["fresh", "later"] + assert _clients(cache) == ["fresh"] @pytest.mark.asyncio async def test_sweep_handles_more_than_one_batch(): - backend = MemoryRateLimitBackend() - backend._SWEEP_BATCH = 10 + cache = InMemoryCacheStore() + cache.SWEEP_BATCH = 10 for i in range(35): - await _hit(backend, f"k{i}") - for entry in backend._buckets.values(): - entry.last_seen = time.monotonic() - 1000 + await _hit(cache, f"k{i}", window=0.05) + time.sleep(0.1) - await backend._sweep(60) + await cache._sweep() - assert len(backend._buckets) == 0 + assert len(cache._slots) == 0 @pytest.mark.asyncio async def test_cap_warning_logged_once_per_minute(caplog): - backend = MemoryRateLimitBackend(max_unique_keys=2) - with caplog.at_level(logging.WARNING, logger="tenxgraph_api.rate_limit"): + cache = InMemoryCacheStore(max_slot_keys=2) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.cache_store"): for i in range(50): - await _hit(backend, f"k{i}") + await _hit(cache, f"k{i}") - warnings = [r for r in caplog.records if "bucket cap" in r.getMessage()] + warnings = [r for r in caplog.records if "slot cap" in r.getMessage()] assert len(warnings) == 1 - backend._last_cap_warning = time.monotonic() - 61 - with caplog.at_level(logging.WARNING, logger="tenxgraph_api.rate_limit"): - await _hit(backend, "again") - warnings = [r for r in caplog.records if "bucket cap" in r.getMessage()] + cache._last_cap_warning["slot"] = time.monotonic() - 61 + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.cache_store"): + await _hit(cache, "again") + warnings = [r for r in caplog.records if "slot cap" in r.getMessage()] assert len(warnings) == 2 @pytest.mark.asyncio -async def test_new_key_at_cap_does_not_scan_all_keys(monkeypatch): +async def test_new_key_at_cap_does_not_scan_all_keys(): class Counting(OrderedDict): scans = 0 @@ -121,34 +121,37 @@ def __iter__(self): type(self).scans += 1 return super().__iter__() - def values(self): + def items(self): type(self).scans += 1 - return super().values() + return super().items() - backend = MemoryRateLimitBackend(max_unique_keys=5_000) - backend._buckets = Counting() + cache = InMemoryCacheStore(max_slot_keys=5_000) + cache.SWEEP_EVERY_OPS = 10**9 # keep the background sweep out of the count + cache.SWEEP_INTERVAL_SECONDS = 10**9 + cache._slots = Counting() for i in range(5_000): - await _hit(backend, f"k{i}") + await _hit(cache, f"k{i}") scans_before = Counting.scans for i in range(20): - await _hit(backend, f"new{i}") + await _hit(cache, f"new{i}") - assert len(backend._buckets) == 5_000 - assert "k0" not in backend._buckets - assert Counting.scans == scans_before + assert len(cache._slots) == 5_000 + assert "ratelimit:k0" not in cache._slots + # One iteration per eviction, and each stops at the first (least recently used) key. + assert Counting.scans - scans_before <= 20 @pytest.mark.asyncio async def test_new_key_at_full_table_is_not_slower_than_an_existing_key(): - backend = MemoryRateLimitBackend(max_unique_keys=50_000) + cache = InMemoryCacheStore(max_slot_keys=50_000) for i in range(50_000): - await _hit(backend, f"k{i}") + await _hit(cache, f"k{i}") start = time.perf_counter() for i in range(500): - await _hit(backend, f"new{i}") + await _hit(cache, f"new{i}") elapsed = time.perf_counter() - start - # The old full scan cost about 2.5 ms per new key (1.25 s here). + # A full scan per new key would cost about 2.5 ms each (1.25 s here). assert elapsed < 0.5 diff --git a/tests/unit_tests/test_rate_limit_unavailable.py b/tests/unit_tests/test_rate_limit_unavailable.py index f4f9b8be..6af6e210 100644 --- a/tests/unit_tests/test_rate_limit_unavailable.py +++ b/tests/unit_tests/test_rate_limit_unavailable.py @@ -8,36 +8,36 @@ from fastapi import FastAPI from fastapi.testclient import TestClient +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore, SlotResult from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - BaseRateLimitBackend, - MemoryRateLimitBackend, - RateLimitDecision, - RateLimitMiddleware, - RedisRateLimitBackend, -) +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware, limiter + + +@pytest.fixture(autouse=True) +def _no_outage_in_progress(monkeypatch): + monkeypatch.setattr(limiter, "_failing", False) def _config(**overrides) -> RateLimitConfig: - data = {"enabled": True, "backend": "memory", "requests": 2, "window": 60, "by": "ip"} + data = {"enabled": True, "requests": 2, "window": 60, "by": "ip"} data.update(overrides) return RateLimitConfig.from_dict(data) -def _broken_redis_backend(*, fail_open: bool) -> RedisRateLimitBackend: - async def failing_script(*, keys, args): - raise ConnectionError("Error 111 connecting to localhost:6379. Connection refused.") +class _BrokenStore(InMemoryCacheStore): + """A cache store whose backend is down, until ``healed``.""" - backend = object.__new__(RedisRateLimitBackend) - backend._prefix = "10xgraph:test" - backend._fail_open = fail_open - backend._script = failing_script - return backend + healed = False + async def acquire(self, key: str, limit: int, ttl: float) -> SlotResult: + if not self.healed: + raise ConnectionError("Error 111 connecting to localhost:6379. Connection refused.") + return await super().acquire(key, limit, ttl) -def _client(backend: BaseRateLimitBackend, **overrides) -> TestClient: + +def _client(cache: BaseCacheStore, **overrides) -> TestClient: app = FastAPI() - app.add_middleware(RateLimitMiddleware, config=_config(**overrides), backend=backend) + app.add_middleware(RateLimitMiddleware, config=_config(**overrides), cache=cache) @app.get("/") def root(): @@ -47,7 +47,7 @@ def root(): def test_backend_error_with_fail_open_false_answers_503(): - resp = _client(_broken_redis_backend(fail_open=False)).get("/") + resp = _client(_BrokenStore(), fail_open=False).get("/") assert resp.status_code == 503 assert resp.headers["Retry-After"] == "5" error = resp.json()["error"] @@ -57,13 +57,13 @@ def test_backend_error_with_fail_open_false_answers_503(): def test_backend_error_with_fail_open_true_allows_the_request(): - resp = _client(_broken_redis_backend(fail_open=True)).get("/") + resp = _client(_BrokenStore(), fail_open=True).get("/") assert resp.status_code == 200 assert resp.json() == {"ok": True} def test_real_limit_still_answers_429(): - client = _client(MemoryRateLimitBackend()) + client = _client(InMemoryCacheStore()) assert client.get("/").status_code == 200 assert client.get("/").status_code == 200 resp = client.get("/") @@ -72,46 +72,53 @@ def test_real_limit_still_answers_429(): assert "Retry-After" in resp.headers -def test_custom_backend_can_signal_unavailable(): - class Down(BaseRateLimitBackend): - async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: - return RateLimitDecision.backend_unavailable(retry_after=7) +def test_plain_denial_from_a_custom_store_is_still_429(): + class Full(InMemoryCacheStore): + async def acquire(self, key: str, limit: int, ttl: float) -> SlotResult: + return SlotResult( + granted=False, count=limit, oldest_expiry=await self.now() + 2.5, slot_id=None + ) - async def close(self) -> None: - return None + resp = _client(Full()).get("/") + assert resp.status_code == 429 + assert resp.headers["Retry-After"] == "3" - resp = _client(Down()).get("/") - assert resp.status_code == 503 - assert resp.headers["Retry-After"] == "7" +def test_app_wiring_with_fail_open_false_and_a_failing_cache_store_answers_503(tmp_path): + """The startup check off, the bound cache store down: 503 RATE_LIMIT_UNAVAILABLE, not 429.""" + from injectq import InjectQ -def test_plain_denial_from_a_custom_backend_is_still_429(): - class Deny(BaseRateLimitBackend): - async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: - return RateLimitDecision(allowed=False, remaining=0, reset_after=3) + from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + from tenxgraph_api.src.app.core.config.setup_middleware import setup_middleware - async def close(self) -> None: - return None + path = tmp_path / "10xgraph.json" + path.write_text('{"agent": "g:app", "rate_limit": {"requests": 2, "fail_open": false}}') + graph_config = GraphConfig(str(path)) + container = InjectQ() + app = FastAPI() + setup_middleware(app, graph_config=graph_config, container=container) + container.bind_instance(BaseCacheStore, _BrokenStore()) # bound at startup, after setup - resp = _client(Deny()).get("/") - assert resp.status_code == 429 - assert resp.headers["Retry-After"] == "3" + @app.get("/") + def root(): + return {"ok": True} + + resp = TestClient(app).get("/", headers={"host": "localhost"}) + assert resp.status_code == 503 + assert resp.json()["error"]["code"] == "RATE_LIMIT_UNAVAILABLE" @pytest.mark.asyncio async def test_outage_logs_one_traceback_then_one_line_per_error(caplog): - backend = _broken_redis_backend(fail_open=False) + store = _BrokenStore() with caplog.at_level(logging.INFO, logger="tenxgraph_api.rate_limit"): for _ in range(3): - decision = await backend.check("client", limit=2, window=60) + decision = await limiter.check(store, "client", 2, 60, fail_open=False) assert decision.unavailable is True records = [r for r in caplog.records if r.name == "tenxgraph_api.rate_limit"] assert [r.exc_info is not None for r in records] == [True, False, False] assert "Connection refused" in records[1].getMessage() - async def working_script(*, keys, args): - return [1, 1, 60] - - backend._script = working_script - assert (await backend.check("client", limit=2, window=60)).allowed is True + store.healed = True + assert (await limiter.check(store, "client", 2, 60, fail_open=False)).allowed is True assert "recovered" in caplog.records[-1].getMessage() diff --git a/tests/unit_tests/test_rate_limit_user_key.py b/tests/unit_tests/test_rate_limit_user_key.py index 1e23c10f..ea0f6a4b 100644 --- a/tests/unit_tests/test_rate_limit_user_key.py +++ b/tests/unit_tests/test_rate_limit_user_key.py @@ -10,11 +10,9 @@ from starlette.requests import Request from tenxgraph_api.src.app.core.auth.base_auth import BaseAuth +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, RateLimitConfig -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - MemoryRateLimitBackend, - RateLimitMiddleware, -) +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware from tenxgraph_api.src.app.core.middleware.rate_limit.keying import client_key_for @@ -29,7 +27,7 @@ def authenticate(self, request, response, credential): def _config(**overrides) -> RateLimitConfig: - data = {"enabled": True, "backend": "memory", "requests": 2, "window": 60, "by": "user"} + data = {"enabled": True, "requests": 2, "window": 60, "by": "user"} data.update(overrides) return RateLimitConfig.from_dict(data) @@ -47,7 +45,7 @@ def auth_container(): def _app(config: RateLimitConfig) -> TestClient: app = FastAPI() - app.add_middleware(RateLimitMiddleware, config=config, backend=MemoryRateLimitBackend()) + app.add_middleware(RateLimitMiddleware, config=config, cache=InMemoryCacheStore()) @app.get("/") def root(): diff --git a/tests/unit_tests/test_readiness.py b/tests/unit_tests/test_readiness.py index 59ca212e..d1b665bd 100644 --- a/tests/unit_tests/test_readiness.py +++ b/tests/unit_tests/test_readiness.py @@ -3,6 +3,8 @@ # ruff: noqa: S101 import asyncio +import json +import logging from types import SimpleNamespace import pytest @@ -14,43 +16,30 @@ from tenxgraph.storage.store import BaseStore from tenxgraph_api.src.app.core.auth.route_guard import find_unprotected_routes +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore from tenxgraph_api.src.app.core.config import setup_middleware as sm +from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.core.middleware.rate_limit.middleware import HEALTH_CHECK_PATHS from tenxgraph_api.src.app.routers.ping import readiness from tenxgraph_api.src.app.routers.ping.router import router as ping_router class _Broken: - async def aget_thread_owner(self, thread_id): + async def aping(self): raise ConnectionError("postgres://user:secret@db is down") -class _PgLike: - def __init__(self, fail_redis=False): - self.queries = [] - self.redis = SimpleNamespace(ping=self._ping) - self._fail_redis = fail_redis - - async def _run_query(self, fn): - conn = SimpleNamespace(fetchval=self._fetchval) - return await fn(conn) - - async def _fetchval(self, sql): - self.queries.append(sql) - return 1 - - async def _ping(self): - if self._fail_redis: - raise ConnectionError("redis down") - return True +class _Pinged: + def __init__(self): + self.pings = 0 + async def aping(self): + self.pings += 1 -class _SelfChecking: - def __init__(self): - self.called = False - async def readiness_check(self): - self.called = True +class _DownCache(InMemoryCacheStore): + async def aping(self): + raise ConnectionError("redis down") # --------------------------------------------------------------------------- @@ -62,19 +51,20 @@ async def readiness_check(self): async def test_nothing_configured_is_ready(): assert await readiness.readiness(None, None, None) == { "checkpointer": "skipped", - "redis": "skipped", "store": "skipped", + "cache": "skipped", } @pytest.mark.asyncio -async def test_in_memory_checkpointer_answers_a_lookup(): - checks = await readiness.readiness(InMemoryCheckpointer(), None, None) - assert checks["checkpointer"] == "ok" +async def test_in_memory_components_answer_their_aping(): + checks = await readiness.readiness(InMemoryCheckpointer(), None, InMemoryCacheStore()) + assert checks == {"checkpointer": "ok", "store": "skipped", "cache": "ok"} @pytest.mark.asyncio -async def test_failing_checkpointer_fails_without_leaking_the_reason(caplog): +async def test_failing_checkpointer_fails_without_leaking_the_reason(caplog, monkeypatch): + monkeypatch.setattr(logging.getLogger("tenxgraph_api"), "propagate", True) checks = await readiness.readiness(_Broken(), None, None) assert checks["checkpointer"] == "fail" assert "secret" in caplog.text # logged for operators ... @@ -82,60 +72,44 @@ async def test_failing_checkpointer_fails_without_leaking_the_reason(caplog): @pytest.mark.asyncio -async def test_pg_style_checkpointer_runs_select_1_and_pings_its_redis(): - pg = _PgLike() - assert (await readiness.readiness(pg, None, None))["checkpointer"] == "ok" - assert pg.queries == ["SELECT 1"] - assert (await readiness.readiness(_PgLike(fail_redis=True), None, None))[ - "checkpointer" - ] == "fail" +async def test_only_aping_is_called_on_each_component(): + checkpointer, store, cache = _Pinged(), _Pinged(), _Pinged() + checks = await readiness.readiness(checkpointer, store, cache) + assert checks == {"checkpointer": "ok", "store": "ok", "cache": "ok"} + assert checkpointer.pings == store.pings == cache.pings == 1 @pytest.mark.asyncio -async def test_a_components_own_check_is_preferred(): - checkpointer, store = _SelfChecking(), _SelfChecking() - checks = await readiness.readiness(checkpointer, store, None) - assert checks["checkpointer"] == checks["store"] == "ok" - assert checkpointer.called and store.called - - -@pytest.mark.asyncio -async def test_store_without_a_probe_is_skipped_and_ping_is_used(): +async def test_component_without_aping_is_skipped(): assert (await readiness.readiness(None, object(), None))["store"] == "skipped" - async def aping(): - return True - - assert (await readiness.readiness(None, SimpleNamespace(aping=aping), None))["store"] == "ok" - @pytest.mark.asyncio async def test_slow_dependency_times_out(monkeypatch): monkeypatch.setattr(readiness, "CHECK_TIMEOUT_SECONDS", 0.05) class _Hangs: - async def readiness_check(self): + async def aping(self): await asyncio.sleep(5) assert (await readiness.readiness(_Hangs(), None, None))["checkpointer"] == "fail" -@pytest.mark.asyncio -async def test_redis_is_pinged_and_closed(monkeypatch): - calls = [] - - class _Client: - async def ping(self): - calls.append("ping") +def test_cache_counts_only_when_a_fail_closed_feature_needs_it(): + checks = {"checkpointer": "ok", "cache": "fail"} + assert readiness.is_ready(checks, cache_is_required=False) + assert not readiness.is_ready(checks, cache_is_required=True) + assert not readiness.is_ready({"checkpointer": "fail", "cache": "ok"}, cache_is_required=False) + assert not readiness.is_ready({"store": "fail"}, cache_is_required=False) - async def aclose(self): - calls.append("closed") - - monkeypatch.setattr( - "tenxgraph_api.src.app.core.redis_client.create_redis", lambda url, **kw: _Client() - ) - assert (await readiness.readiness(None, None, "redis://cache"))["redis"] == "ok" - assert calls == ["ping", "closed"] + on = SimpleNamespace(token_revocation=object(), rate_limit=None) + assert readiness.cache_required_by(on) == ["token_revocation"] + strict = SimpleNamespace(token_revocation=None, rate_limit=SimpleNamespace(fail_open=False)) + assert readiness.cache_required_by(strict) == ["rate_limit.fail_open=false"] + lenient = SimpleNamespace(token_revocation=None, rate_limit=SimpleNamespace(fail_open=True)) + assert not readiness.cache_required(lenient) + assert not readiness.cache_required(SimpleNamespace(token_revocation=None, rate_limit=None)) + assert not readiness.cache_required(None) # --------------------------------------------------------------------------- @@ -150,20 +124,16 @@ def _fresh_cache(): readiness.reset_cache() -@pytest.fixture(autouse=True) -def _no_redis(monkeypatch): - """The endpoint tests do not configure Redis, whatever the environment says.""" - from tenxgraph_api.src.app.core.config import settings as settings_mod - - monkeypatch.setattr(settings_mod, "get_settings", lambda: SimpleNamespace(REDIS_URL=None)) - - -def _client(checkpointer=None, store=None) -> TestClient: +def _client(checkpointer=None, store=None, cache=None, config=None) -> TestClient: container = InjectQ() if checkpointer is not None: container.bind_instance(BaseCheckpointer, checkpointer) if store is not None: container.bind_instance(BaseStore, store) + if cache is not None: + container.bind_instance(BaseCacheStore, cache) + if config is not None: + container.bind_instance(GraphConfig, config) app = FastAPI() setup_fastapi(container, app) app.include_router(ping_router) @@ -171,10 +141,22 @@ def _client(checkpointer=None, store=None) -> TestClient: def test_endpoint_is_200_when_ready(): - resp = _client(InMemoryCheckpointer()).get("/health/ready") + resp = _client(InMemoryCheckpointer(), cache=InMemoryCacheStore()).get("/health/ready") assert resp.status_code == 200 - assert resp.json()["status"] == "ready" - assert resp.json()["checks"]["checkpointer"] == "ok" + assert resp.json() == {"status": "ready", "checks": {"checkpointer": "ok", "cache": "ok"}} + + +def test_cache_outage_is_reported_and_fails_only_when_required(tmp_path): + path = tmp_path / "10xgraph.json" + path.write_text(json.dumps({"agent": "graph.agent:app"})) + resp = _client(InMemoryCheckpointer(), cache=_DownCache(), config=GraphConfig(str(path))) + assert resp.get("/health/ready").status_code == 200 + assert resp.get("/health/ready").json()["checks"]["cache"] == "fail" + + readiness.reset_cache() + path.write_text(json.dumps({"agent": "graph.agent:app", "token_revocation": True})) + resp = _client(InMemoryCheckpointer(), cache=_DownCache(), config=GraphConfig(str(path))) + assert resp.get("/health/ready").status_code == 503 def test_endpoint_is_503_when_a_dependency_fails(): @@ -208,7 +190,7 @@ async def test_results_are_cached_and_shared_by_concurrent_callers(monkeypatch): calls = [] class _Counting: - async def readiness_check(self): + async def aping(self): calls.append(1) await asyncio.sleep(0.01) @@ -224,34 +206,3 @@ async def readiness_check(self): monkeypatch.setattr(readiness, "CACHE_SECONDS", 0.0) await readiness.cached_readiness(probe, None, None) assert len(calls) == 2 - - -# --------------------------------------------------------------------------- -# schema version (G-164) -# --------------------------------------------------------------------------- - - -class _PgWithVersion(_PgLike): - def __init__(self, found, expected=4): - super().__init__() - self._found, self._expected = found, expected - - async def aget_schema_version(self): - return self._found - - def _get_current_schema_version(self): - return self._expected - - -@pytest.mark.asyncio -@pytest.mark.parametrize("found", [0, 3]) -async def test_old_or_missing_schema_is_not_ready(found): - checks = await readiness.readiness(_PgWithVersion(found), None, None) - assert checks["checkpointer"] == "fail" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("found", [4, 5]) -async def test_current_schema_is_ready(found): - checks = await readiness.readiness(_PgWithVersion(found), None, None) - assert checks["checkpointer"] == "ok" diff --git a/tests/unit_tests/test_redis_hardening.py b/tests/unit_tests/test_redis_hardening.py index c0df5638..008b8204 100644 --- a/tests/unit_tests/test_redis_hardening.py +++ b/tests/unit_tests/test_redis_hardening.py @@ -1,7 +1,5 @@ """Redis client limits (G-92), Redis clock (G-131) and revocation fixes (G-117, G-167).""" -# ruff: noqa: S101 - import time from typing import Any @@ -9,12 +7,9 @@ from tenxgraph_api.src.app.core import redis_client from tenxgraph_api.src.app.core.auth import revocation -from tenxgraph_api.src.app.core.auth.revocation import ( - DEFAULT_USER_REVOCATION_TTL, - InMemoryTokenRevocation, - RedisTokenRevocation, -) -from tenxgraph_api.src.app.core.concurrency import RedisConcurrencyLimiter +from tenxgraph_api.src.app.core.auth.revocation import DEFAULT_USER_REVOCATION_TTL +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore, redis_store +from tenxgraph_api.src.app.core.cache_store.redis_store import RedisCacheStore from tenxgraph_api.src.app.core.config.settings import get_settings @@ -69,26 +64,15 @@ def test_zero_turns_a_limit_off(monkeypatch): def test_every_api_redis_client_uses_the_shared_builder(monkeypatch): + """The cache store is the API's only Redis client (locks, caps, idempotency, ownership, + revocation and rate limits all go through it).""" built = [] monkeypatch.setattr( redis_client, "create_redis", lambda url, **kw: built.append(url) or object() ) - RedisConcurrencyLimiter.from_url("redis://a") - RedisTokenRevocation("redis://b") - from tenxgraph_api.src.app.core.middleware.rate_limit.redis import RedisRateLimitBackend - - class _Client: - def register_script(self, _): - return None - - monkeypatch.setattr( - redis_client, "create_redis", lambda url, **kw: built.append(url) or _Client() - ) - RedisRateLimitBackend.from_url("redis://c", "p") - from tenxgraph_api.src.app.loader import _build_ownership_redis - - _build_ownership_redis("redis://d") - assert built == ["redis://a", "redis://b", "redis://c", "redis://d"] + monkeypatch.setattr(redis_store, "create_redis", redis_client.create_redis) + RedisCacheStore.from_url("redis://a") + assert built == ["redis://a"] # --------------------------------------------------------------------------- @@ -96,53 +80,11 @@ def register_script(self, _): # --------------------------------------------------------------------------- -class _SkewedRedis: - """Redis whose clock is an hour ahead of this process.""" - - SKEW = 3600.0 - - def __init__(self): - self.scores: list[float] = [] - - async def time(self): - now = time.time() + self.SKEW - return int(now), int((now % 1) * 1_000_000) - - def pipeline(self, transaction=True): - scores = self.scores - - class _Pipe: - def zremrangebyscore(self, key, low, high): - self.high = high - - def zadd(self, key, mapping): - scores.extend(mapping.values()) - - def zcard(self, key): - pass - - def expire(self, key, seconds): - pass - - async def execute(self): - return [None, 1, 1, True] - - return _Pipe() - - async def zrem(self, key, member): - pass - - -@pytest.mark.asyncio -async def test_lease_expiry_uses_the_redis_clock(): - client = _SkewedRedis() - limiter = RedisConcurrencyLimiter(client, lease_ttl=60.0) - lease = await limiter.acquire("k", 1) - assert lease is not None - # With the pod's own clock the score would be about now + 60; the Redis clock is +3600. - assert client.scores[0] > time.time() + 3000 - lease.release() - await limiter.aclose() +def test_lease_expiry_uses_the_redis_clock(): + """Lease scores are computed inside the scripts from ``TIME``, never the pod's clock.""" + for script in (redis_store._ACQUIRE_LUA, redis_store._RENEW_LUA): + assert "redis.call('TIME')" in script + assert "now + tonumber(ARGV[2])" in script # --------------------------------------------------------------------------- @@ -150,100 +92,68 @@ async def test_lease_expiry_uses_the_redis_clock(): # --------------------------------------------------------------------------- -class _FakeRedis: +class _RecordingStore(InMemoryCacheStore): + """In-memory cache store that records TTLs and refuses multi-key reads.""" + def __init__(self): - self.data: dict[str, Any] = {} + super().__init__() self.ttl: dict[str, Any] = {} - self.mget_calls = 0 - - async def time(self): - now = time.time() - return int(now), int((now % 1) * 1_000_000) - - async def set(self, key, value, ex=None): - self.data[key] = value - self.ttl[key] = ex + self.modes: dict[str, str] = {} - async def get(self, key): - return self.data.get(key) + async def set(self, key, value, ttl, mode="always"): + written = await super().set(key, value, ttl, mode) + if written: + self.ttl[key] = ttl + self.modes[key] = mode + return written async def mget(self, keys): - self.mget_calls += 1 raise AssertionError("MGET over two keys fails with CROSSSLOT on Redis Cluster") - async def eval(self, script, numkeys, key, cutoff, ttl): - new = float(cutoff) if cutoff else time.time() - current = self.data.get(key) - if current is not None and float(current) >= new: - return 0 - self.data[key] = repr(new).encode() - self.ttl[key] = int(ttl) or None - return 1 - @pytest.mark.asyncio -async def test_check_does_not_use_mget(): - client = _FakeRedis() - backend = RedisTokenRevocation(client=client, prefix="p") - await backend.revoke_user("a", before=100.0) - assert await backend.is_revoked({"user_id": "a", "jti": "x", "iat": 50}) is True - assert client.mget_calls == 0 +async def test_check_reads_two_keys_one_by_one(): + store = _RecordingStore() + await revocation.revoke_user(store, "a", before=100.0) + assert await revocation.is_revoked(store, {"user_id": "a", "jti": "x", "iat": 50}) is True @pytest.mark.asyncio async def test_an_older_cutoff_does_not_replace_a_newer_one(): - client = _FakeRedis() - backend = RedisTokenRevocation(client=client, prefix="p") - await backend.revoke_user("a", before=2000.0) - await backend.revoke_user("a", before=1000.0) - assert await backend.is_revoked({"user_id": "a", "iat": 1500}) is True - - -def test_memory_older_cutoff_does_not_replace_a_newer_one(): - backend = InMemoryTokenRevocation() - backend.revoke_user("a", before=2000.0) - backend.revoke_user("a", before=1000.0) - assert backend._users["a"] == 2000.0 + store = _RecordingStore() + await revocation.revoke_user(store, "a", before=2000.0) + await revocation.revoke_user(store, "a", before=1000.0) + assert store.modes["revoked:user:a"] == "if_greater" + assert await store.get("revoked:user:a") == b"2000.000000" + assert await revocation.is_revoked(store, {"user_id": "a", "iat": 1500}) is True @pytest.mark.asyncio async def test_user_entries_expire_by_default(): - client = _FakeRedis() - backend = RedisTokenRevocation(client=client, prefix="p") - await backend.revoke_user("a") - assert client.ttl["p:user:a"] == DEFAULT_USER_REVOCATION_TTL + store = _RecordingStore() + await revocation.revoke_user(store, "a") + assert store.ttl["revoked:user:a"] == DEFAULT_USER_REVOCATION_TTL @pytest.mark.asyncio async def test_token_without_iat_is_dated_from_exp_when_the_lifetime_is_known(): now = time.time() - backend = InMemoryTokenRevocation(token_lifetime=3600) - backend.revoke_user("a", before=now) + store = InMemoryCacheStore() + await revocation.revoke_user(store, "a", before=now) old = {"user_id": "a", "exp": now + 100} # minted about 3500 s ago new = {"user_id": "a", "exp": now + 3600 + 100} # minted after the revoke - assert await backend.is_revoked(old) is True - assert await backend.is_revoked(new) is False + assert await revocation.is_revoked(store, old, token_lifetime=3600) is True + assert await revocation.is_revoked(store, new, token_lifetime=3600) is False # Without a lifetime an undated token cannot be told apart and stays revoked. - plain = InMemoryTokenRevocation() - plain.revoke_user("a", before=now) - assert await plain.is_revoked(new) is True - - -@pytest.mark.asyncio -async def test_exported_helpers_revoke_through_the_backend(): - backend = InMemoryTokenRevocation() - await revocation.revoke_token("j1", backend=backend) - await revocation.revoke_user("u1", before=time.time(), backend=backend) - assert await backend.is_revoked({"jti": "j1"}) is True - assert await backend.is_revoked({"user_id": "u1", "iat": 1}) is True - - redis = RedisTokenRevocation(client=_FakeRedis(), prefix="p") - await revocation.revoke_token("j2", backend=redis) - assert await redis.is_revoked({"jti": "j2"}) is True + assert await revocation.is_revoked(store, new) is True @pytest.mark.asyncio -async def test_helpers_need_a_backend(monkeypatch): - monkeypatch.setattr(revocation, "_bound_backend", lambda: None) - with pytest.raises(RuntimeError, match="token_revocation"): - await revocation.revoke_user("u") +async def test_token_entry_expires_with_the_token(): + store = _RecordingStore() + await revocation.revoke_token(store, "j1", expires_at=time.time() + 120) + assert 1 <= store.ttl["revoked:jti:j1"] <= 120 + await revocation.revoke_token(store, "j2") + assert store.ttl["revoked:jti:j2"] is None + assert await revocation.is_revoked(store, {"jti": "j1"}) is True + assert await revocation.is_revoked(store, {"jti": "j2"}) is True diff --git a/tests/unit_tests/test_reserved_user_id.py b/tests/unit_tests/test_reserved_user_id.py index 71319f3f..ba883b52 100644 --- a/tests/unit_tests/test_reserved_user_id.py +++ b/tests/unit_tests/test_reserved_user_id.py @@ -27,12 +27,9 @@ identity_is_reserved, is_reserved_user_id, ) +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, RateLimitConfig -from tenxgraph_api.src.app.core.middleware.rate_limit import ( - MemoryRateLimitBackend, - RateLimitMiddleware, - keying, -) +from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware, keying from tenxgraph_api.src.app.routers.graph.realtime_guard import ws_identity_still_valid @@ -79,9 +76,9 @@ def _app(auth: BaseAuth, *, rate_limit_by_user: bool = False) -> tuple[FastAPI, app = FastAPI() if rate_limit_by_user: config = RateLimitConfig.from_dict( - {"enabled": True, "backend": "memory", "requests": 100, "window": 60, "by": "user"} + {"enabled": True, "requests": 100, "window": 60, "by": "user"} ) - app.add_middleware(RateLimitMiddleware, config=config, backend=MemoryRateLimitBackend()) + app.add_middleware(RateLimitMiddleware, config=config, cache=InMemoryCacheStore()) setup_fastapi(container, app) @app.get("/v1/threads") diff --git a/tests/unit_tests/test_run_cleanup_order.py b/tests/unit_tests/test_run_cleanup_order.py index 0d83dae3..c7cb7f23 100644 --- a/tests/unit_tests/test_run_cleanup_order.py +++ b/tests/unit_tests/test_run_cleanup_order.py @@ -25,7 +25,7 @@ from tenxgraph.utils.constants import END from tenxgraph_api.src.app.core import shutdown -from tenxgraph_api.src.app.core.concurrency import Lease +from tenxgraph_api.src.app.core.leases import Lease from tenxgraph_api.src.app.routers.graph.run_limits import release_when_done from tenxgraph_api.src.app.routers.graph.services.graph_service import _bounded_stream diff --git a/tests/unit_tests/test_run_entry_guards.py b/tests/unit_tests/test_run_entry_guards.py index 797ed803..ea8f5623 100644 --- a/tests/unit_tests/test_run_entry_guards.py +++ b/tests/unit_tests/test_run_entry_guards.py @@ -4,8 +4,6 @@ lock and per-user run slot, hold them until the run is really over, and cancel cleanly. """ -# ruff: noqa: S101 - import asyncio import json import sys @@ -15,8 +13,9 @@ import pytest from fastapi import HTTPException, WebSocketDisconnect -from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.graph import run_limits, thread_lock from tenxgraph_api.src.app.routers.graph.router import ( realtime_graph_ws, @@ -40,9 +39,9 @@ def _router(): @pytest.fixture(autouse=True) def limiter(monkeypatch): """One in-process limiter behind both the thread lock and the run slot.""" - shared = MemoryConcurrencyLimiter() - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: shared) - monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: shared) + shared = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: shared) + monkeypatch.setattr(run_limits, "lease_manager", lambda: shared) monkeypatch.setattr(thread_lock, "THREAD_BUSY_WAIT_SECONDS", 0.2) router = _router() monkeypatch.setattr(router, "ws_identity_still_valid", lambda ws, user: True) diff --git a/tests/unit_tests/test_run_limits.py b/tests/unit_tests/test_run_limits.py index deb619ce..5781ced7 100644 --- a/tests/unit_tests/test_run_limits.py +++ b/tests/unit_tests/test_run_limits.py @@ -1,7 +1,5 @@ """Per-user run cap and run timeout for graph runs.""" -# ruff: noqa: S101 - import asyncio import gc from types import SimpleNamespace @@ -11,8 +9,9 @@ from fastapi import HTTPException from tenxgraph.core.state import StreamChunk, StreamEvent -from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.graph import run_limits from tenxgraph_api.src.app.routers.graph.run_limits import ( acquire_run_slot, @@ -30,7 +29,10 @@ ALICE = {"user_id": "alice"} -class _Slots(MemoryConcurrencyLimiter): +class _Slots(LeaseManager): + def __init__(self): + super().__init__(InMemoryCacheStore()) + def held(self, user_id): # tests count by user id return super().held(run_limits.run_slot_key(user_id)) @@ -38,7 +40,7 @@ def held(self, user_id): # tests count by user id @pytest.fixture(autouse=True) def fresh_slots(monkeypatch): slots = _Slots() - monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: slots) + monkeypatch.setattr(run_limits, "lease_manager", lambda: slots) return slots diff --git a/tests/unit_tests/test_setup_middleware.py b/tests/unit_tests/test_setup_middleware.py index 453feb48..c00eda5d 100644 --- a/tests/unit_tests/test_setup_middleware.py +++ b/tests/unit_tests/test_setup_middleware.py @@ -415,12 +415,13 @@ class MockSettings: def test_setup_middleware_all(): + from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware + app = FastAPI() settings = MockSettings() graph_config = MagicMock() rate_limit_config = MagicMock() - rate_limit_config.backend = "memory" rate_limit_config.requests = 100 rate_limit_config.window = 60 rate_limit_config.by = "ip" @@ -435,10 +436,6 @@ def test_setup_middleware_all(): "tenxgraph_api.src.app.core.config.setup_middleware.get_settings", return_value=settings ), patch("tenxgraph_api.src.app.core.config.setup_middleware.init_sentry") as mock_init_sentry, - patch( - "tenxgraph_api.src.app.core.config.setup_middleware.build_backend", - return_value="mock_backend", - ) as mock_build_backend, patch("tenxgraph_api.src.app.core.config.setup_middleware._setup_otel") as mock_setup_otel, patch( "tenxgraph_api.src.app.core.config.setup_middleware._attach_otel_publisher" @@ -447,8 +444,10 @@ def test_setup_middleware_all(): setup_middleware(app, graph_config=graph_config, container=container) mock_init_sentry.assert_called_once_with(settings) - mock_build_backend.assert_called_once_with(rate_limit_config, container=container) - assert app.state.rate_limit_backend == "mock_backend" + rate_limit = [m for m in app.user_middleware if m.cls is RateLimitMiddleware] + assert len(rate_limit) == 1 + assert rate_limit[0].kwargs == {"config": rate_limit_config, "container": container} + assert not hasattr(app.state, "rate_limit_backend") mock_setup_otel.assert_called_once_with(app, settings) mock_attach.assert_called_once_with(container, settings, graph_config) diff --git a/tests/unit_tests/test_shutdown_readiness.py b/tests/unit_tests/test_shutdown_readiness.py index cb586ae8..b144b333 100644 --- a/tests/unit_tests/test_shutdown_readiness.py +++ b/tests/unit_tests/test_shutdown_readiness.py @@ -17,6 +17,7 @@ from tenxgraph_api.cli.templates import defaults from tenxgraph_api.src.app import uvicorn_worker from tenxgraph_api.src.app.core import shutdown +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore from tenxgraph_api.src.app.routers.ping import readiness from tenxgraph_api.src.app.routers.ping.router import router as ping_router @@ -52,12 +53,10 @@ def test_dockerfile_and_compose_use_the_draining_worker(): # ---------------------------------------------------------------- readiness -def _client(monkeypatch, draining=False): - from tenxgraph_api.src.app.core.config import settings as settings_mod - - monkeypatch.setattr(settings_mod, "get_settings", lambda: SimpleNamespace(REDIS_URL=None)) +def _client(draining=False): container = InjectQ() container.bind_instance(BaseCheckpointer, InMemoryCheckpointer()) + container.bind_instance(BaseCacheStore, InMemoryCacheStore()) app = FastAPI() setup_fastapi(container, app) app.include_router(ping_router) @@ -74,28 +73,24 @@ def _fresh(): shutdown.reset() -def test_draining_app_is_not_ready(monkeypatch): - assert _client(monkeypatch).get("/health/ready").status_code == 200 - resp = _client(monkeypatch, draining=True).get("/health/ready") +def test_draining_app_is_not_ready(): + assert _client().get("/health/ready").status_code == 200 + resp = _client(draining=True).get("/health/ready") assert resp.status_code == 503 assert resp.json()["status"] == "draining" -def test_redis_outage_alone_stays_ready_unless_a_feature_fails_closed(): - checks = {"checkpointer": "ok", "redis": "fail"} - assert readiness.is_ready(checks, redis_is_required=False) - assert not readiness.is_ready(checks, redis_is_required=True) - assert not readiness.is_ready({"checkpointer": "fail", "redis": "ok"}, redis_is_required=False) - assert readiness.redis_required(SimpleNamespace(token_revocation="redis", rate_limit=None)) - assert readiness.redis_required( - SimpleNamespace( - token_revocation=None, rate_limit=SimpleNamespace(backend="redis", fail_open=False) - ) +def test_cache_outage_alone_stays_ready_unless_a_feature_fails_closed(): + checks = {"checkpointer": "ok", "cache": "fail"} + assert readiness.is_ready(checks, cache_is_required=False) + assert not readiness.is_ready(checks, cache_is_required=True) + assert not readiness.is_ready({"checkpointer": "fail", "cache": "ok"}, cache_is_required=False) + assert readiness.cache_required(SimpleNamespace(token_revocation=object(), rate_limit=None)) + assert readiness.cache_required( + SimpleNamespace(token_revocation=None, rate_limit=SimpleNamespace(fail_open=False)) ) - assert not readiness.redis_required( - SimpleNamespace( - token_revocation="memory", rate_limit=SimpleNamespace(backend="redis", fail_open=True) - ) + assert not readiness.cache_required( + SimpleNamespace(token_revocation=None, rate_limit=SimpleNamespace(fail_open=True)) ) diff --git a/tests/unit_tests/test_shutdown_wiring.py b/tests/unit_tests/test_shutdown_wiring.py index 8ab81d24..d1174f17 100644 --- a/tests/unit_tests/test_shutdown_wiring.py +++ b/tests/unit_tests/test_shutdown_wiring.py @@ -8,10 +8,14 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from injectq import InjectQ from tenxgraph.core.exceptions import GraphClosedError from tenxgraph_api.src.app import main as app_main +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore +from tenxgraph_api.src.app.core.cache_store.loader import bind_cache_store from tenxgraph_api.src.app.core.exceptions.handle_errors import init_errors_handler +from tenxgraph_api.src.app.core.leases import LeaseManager def _graph(order: list[str]) -> AsyncMock: @@ -66,6 +70,36 @@ async def sweep_loop(interval): assert order == ["sweeper_cancelled", "aclose"] +@pytest.mark.asyncio +async def test_graph_closes_first_then_leases_then_the_cache_store_once(): + order: list[str] = [] + graph = _graph(order) + + class _Store(InMemoryCacheStore): + async def aclose(self): + order.append("cache_store") + + container = InjectQ() + bind_cache_store(container, _Store()) + leases = container.get(LeaseManager) + leases_close = leases.aclose + + async def close_leases(): + order.append("leases") + await leases_close() + + leases.aclose = close_leases + with ( + patch.object(app_main, "container", container), + patch.object(app_main, "attach_all_modules", AsyncMock(return_value=graph)), + patch.object(app_main, "_prepare_checkpointer", AsyncMock()), + patch.object(app_main, "_cleanup_temp_media_cache", AsyncMock()), + ): + async with app_main.lifespan(FastAPI()): + pass + assert order == ["aclose", "leases", "cache_store"] + + @pytest.mark.asyncio async def test_a_failing_close_does_not_stop_the_rest_of_shutdown(): graph = AsyncMock() diff --git a/tests/unit_tests/test_startup_checks.py b/tests/unit_tests/test_startup_checks.py index a54f11d1..023e8d14 100644 --- a/tests/unit_tests/test_startup_checks.py +++ b/tests/unit_tests/test_startup_checks.py @@ -1,168 +1,237 @@ -"""The startup gate refuses to boot while a configured backend is unreachable.""" +"""The startup gate refuses to boot while a backend the server needs is unreachable.""" # ruff: noqa: S101, SLF001 import asyncio +import json import logging +import socket from types import SimpleNamespace from unittest.mock import AsyncMock +import httpx import pytest from fastapi import FastAPI from injectq import InjectQ -from tenxgraph.storage.checkpointer import BaseCheckpointer +from injectq.integrations.fastapi import setup_fastapi +from tenxgraph.storage.checkpointer import BaseCheckpointer, InMemoryCheckpointer from tenxgraph.storage.store import BaseStore from tenxgraph_api.src.app import main as app_main -from tenxgraph_api.src.app.core import redis_client -from tenxgraph_api.src.app.core.config import settings as settings_mod -from tenxgraph_api.src.app.routers.ping import startup_checks +from tenxgraph_api.src.app.core.cache_store import ( + BaseCacheStore, + InMemoryCacheStore, + RedisCacheStore, +) +from tenxgraph_api.src.app.core.cache_store.loader import ( + bind_cache_store, + load_and_bind_cache_store, +) +from tenxgraph_api.src.app.core.config.graph_config import GraphConfig +from tenxgraph_api.src.app.core.config.settings import get_settings +from tenxgraph_api.src.app.routers.ping import readiness, startup_checks +from tenxgraph_api.src.app.routers.ping.router import router as ping_router from tenxgraph_api.src.app.routers.ping.startup_checks import ( + HINT, StartupCheckError, Target, check_backends_at_startup, - host_port, startup_targets, wait_for_backends, ) -REDIS_URL = "redis://:s3cret-pass@127.0.0.1:6399/0" - +class _Pinged: + """A checkpointer or store: ``aping`` raises ``error`` when one is set.""" -class _FakeRedis: - def __init__(self, error: Exception | None = None, host="cache", port=6379): + def __init__(self, error: Exception | None = None, where: str | None = None): self.error = error - self.connection_pool = SimpleNamespace( - connection_kwargs={"host": host, "port": port, "password": "s3cret-pass"} - ) + self.pings = 0 + self._where = where - async def ping(self): + async def aping(self): + self.pings += 1 if self.error: raise self.error - return True - - async def aclose(self): - return None + def describe(self): + return self._where -class _FakePgCheckpointer: - """Looks like PgCheckpointer to the readiness probe.""" - def __init__(self, error: Exception | None = None): - self.error = error - self._pg_pool_config = {"postgres_dsn": "postgresql://app:s3cret-pass@db:5432/app"} - self.redis = _FakeRedis() - - async def _run_query(self, fn): - if self.error: - raise self.error - return 1 +class _NoPing: + pass -class _PingStore: - def __init__(self): - self.pings = 0 +class _DownCache(InMemoryCacheStore): + """A cache store that does not answer; describes itself as ``cache:6379``.""" async def aping(self): - self.pings += 1 + raise ConnectionError("Error 111 connecting to cache:6379. Connection refused.") + def describe(self): + return "cache:6379" -class _NoPingStore: - pass +def _config(tmp_path, data: dict) -> GraphConfig: + path = tmp_path / "10xgraph.json" + path.write_text(json.dumps({"agent": "graph.agent:app", **data})) + return GraphConfig(str(path)) -def _container(monkeypatch, *, checkpointer=None, store=None, redis_url=None) -> InjectQ: - monkeypatch.setattr(settings_mod, "get_settings", lambda: SimpleNamespace(REDIS_URL=redis_url)) + +def _container(*, checkpointer=None, store=None, cache=None, source=None, config=None) -> InjectQ: container = InjectQ() if checkpointer is not None: container.bind_instance(BaseCheckpointer, checkpointer) if store is not None: container.bind_instance(BaseStore, store) + if cache is not None: + bind_cache_store(container, cache, source) + if config is not None: + container.bind_instance(GraphConfig, config) return container -def _unreachable_redis(monkeypatch): - error = ConnectionError("Error 111 connecting to 127.0.0.1:6399. Connection refused.") - monkeypatch.setattr(redis_client, "create_redis", lambda url: _FakeRedis(error)) +def _closed_port() -> int: + """A local port nothing listens on: bound, then closed.""" + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] async def _no_sleep(_delay): await asyncio.sleep(0) -# ---------------------------------------------------------------- endpoints +@pytest.fixture(autouse=True) +def _propagate(monkeypatch): + """The app's logger stops propagating once logging is set up; caplog needs it on.""" + monkeypatch.setattr(logging.getLogger("tenxgraph_api"), "propagate", True) -def test_host_port_drops_credentials_path_and_query(): - assert host_port(REDIS_URL) == "127.0.0.1:6399" - assert host_port("postgresql://u:p@db.internal:5432/app?sslmode=require") == "db.internal:5432" - assert host_port("redis://[::1]:6379") == "[::1]:6379" - assert host_port("not a url") is None - assert host_port(None) is None +@pytest.fixture(autouse=True) +def _fresh_readiness(): + readiness.reset_cache() + yield + readiness.reset_cache() -def test_unconfigured_components_are_left_out(monkeypatch): - assert startup_targets(_container(monkeypatch)) == [] - names = [t.name for t in startup_targets(_container(monkeypatch, store=_PingStore()))] - assert names == ["store"] +@pytest.fixture +def unreachable_redis_url(monkeypatch): + """``REDIS_URL`` at a closed local port, with short client timeouts.""" + settings = get_settings() + monkeypatch.setattr(settings, "REDIS_CONNECT_TIMEOUT", 0.5) + monkeypatch.setattr(settings, "REDIS_SOCKET_TIMEOUT", 0.5) + monkeypatch.setattr(settings, "REDIS_POOL_TIMEOUT", 0.5) + return f"redis://:s3cret-pass@127.0.0.1:{_closed_port()}/0" -def test_rate_limiter_redis_is_checked_only_when_it_has_its_own_url(monkeypatch): - container = _container(monkeypatch, redis_url=REDIS_URL) - same = SimpleNamespace(redis_url=REDIS_URL) - other = SimpleNamespace(redis_url="redis://limits:6380/1") - bound_client = SimpleNamespace(redis_url=None) - assert [t.name for t in startup_targets(container, same)] == ["redis"] - assert [t.name for t in startup_targets(container, other)] == ["redis", "rate_limit redis"] - assert [t.name for t in startup_targets(container, bound_client)] == ["redis"] +# ---------------------------------------------------------------- targets -# ---------------------------------------------------------------- gate +def test_unbound_components_are_left_out(): + assert startup_targets(_container()) == [] + assert [t.name for t in startup_targets(_container(store=_Pinged()))] == ["store"] -@pytest.mark.asyncio -async def test_boot_check_passes_when_every_backend_answers(monkeypatch): - monkeypatch.setattr(redis_client, "create_redis", lambda url: _FakeRedis()) - store = _PingStore() +def test_cache_is_optional_unless_a_fail_closed_feature_uses_it(tmp_path): + def cache_target(config): + targets = startup_targets(_container(cache=_DownCache(), config=config)) + return next(t for t in targets if t.name == "cache") + + assert not cache_target(None).required + assert not cache_target(_config(tmp_path, {})).required + assert not cache_target(_config(tmp_path, {"rate_limit": {"enabled": True}})).required + assert not cache_target( + _config(tmp_path, {"rate_limit": {"enabled": False, "fail_open": False}}) + ).required + assert not cache_target(_config(tmp_path, {"token_revocation": False})).required + + target = cache_target(_config(tmp_path, {"token_revocation": True})) + assert target.required + assert target.details[-1] == "required by token_revocation" + target = cache_target(_config(tmp_path, {"rate_limit": {"enabled": True, "fail_open": False}})) + assert target.required + assert target.details[-1] == "required by rate_limit.fail_open=false" + + +def test_details_are_the_class_source_and_described_endpoint(): container = _container( - monkeypatch, checkpointer=_FakePgCheckpointer(), store=store, redis_url=REDIS_URL + checkpointer=_Pinged(where="db:5432"), + store=_Pinged(), + cache=_DownCache(), + source="from REDIS_URL", ) - await check_backends_at_startup(container, timeout=0) - assert store.pings == 1 + by_name = {t.name: t.details for t in startup_targets(container)} + assert by_name == { + "checkpointer": ["_Pinged", "db:5432"], + "store": ["_Pinged"], + "cache": ["_DownCache", "from REDIS_URL", "cache:6379"], + } + + +# ---------------------------------------------------------------- gate @pytest.mark.asyncio -async def test_boot_check_fails_naming_component_and_host_without_password(monkeypatch, caplog): - _unreachable_redis(monkeypatch) - container = _container(monkeypatch, redis_url=REDIS_URL) +async def test_boot_check_passes_when_every_backend_answers(caplog): + checkpointer, store = _Pinged(), _Pinged() + container = _container(checkpointer=checkpointer, store=store, cache=InMemoryCacheStore()) + with caplog.at_level(logging.INFO, logger="tenxgraph_api.startup"): + await check_backends_at_startup(container, timeout=0) + assert checkpointer.pings == store.pings == 1 + assert "Startup check passed: checkpointer, store, cache" in caplog.text + + +@pytest.mark.asyncio +async def test_failed_checkpointer_names_component_and_class(caplog): + error = ConnectionRefusedError("Connect call failed ('db', 5432)") + container = _container(checkpointer=_Pinged(error, where="db:5432")) with caplog.at_level(logging.WARNING, logger="tenxgraph_api.startup"): with pytest.raises(StartupCheckError) as info: await check_backends_at_startup(container, timeout=0) message = str(info.value) - assert "redis (127.0.0.1:6399)" in message - assert "Connection refused" in message - assert "STARTUP_CHECKS=false" in message - assert "s3cret" not in message - assert "s3cret" not in caplog.text + assert "checkpointer (_Pinged, db:5432): ConnectionRefusedError" in message + assert message.endswith(HINT) + assert HINT == ( + "Start the backend, fix its URL, or set STARTUP_CHECKS=false to skip this check." + ) assert any(r.levelno == logging.CRITICAL for r in caplog.records) assert any("attempt 1" in r.getMessage() for r in caplog.records) @pytest.mark.asyncio -async def test_failed_checkpointer_names_postgres_and_redis_hosts(monkeypatch): - error = ConnectionRefusedError("Connect call failed ('db', 5432)") - container = _container(monkeypatch, checkpointer=_FakePgCheckpointer(error)) +async def test_unreachable_optional_cache_is_one_warning_and_boot_goes_on(caplog): + checkpointer = _Pinged() + container = _container(checkpointer=checkpointer, cache=_DownCache(), source="from redis") + with caplog.at_level(logging.INFO, logger="tenxgraph_api.startup"): + await check_backends_at_startup(container, timeout=0) + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 1 + assert warnings[0].startswith( + "Startup check: cache store _DownCache (from redis, cache:6379) unreachable: locks and " + "caps fall back to per process" + ) + assert "Startup check passed: checkpointer" in caplog.text + assert "passed: checkpointer, cache" not in caplog.text + + +@pytest.mark.asyncio +async def test_required_cache_refuses_boot_naming_cache_and_source(tmp_path): + container = _container( + cache=_DownCache(), + source="from REDIS_URL", + config=_config(tmp_path, {"token_revocation": True}), + ) with pytest.raises(StartupCheckError) as info: await check_backends_at_startup(container, timeout=0) - message = str(info.value) - assert "checkpointer (postgres db:5432, redis cache:6379)" in message - assert "ConnectionRefusedError" in message - assert "s3cret" not in message + assert ( + "cache (_DownCache, from REDIS_URL, cache:6379, required by token_revocation): " + "ConnectionError" in str(info.value) + ) @pytest.mark.asyncio -async def test_check_that_recovers_within_the_window_passes(monkeypatch): +async def test_check_that_recovers_within_the_window_passes(): calls = 0 async def flaky(): @@ -171,7 +240,7 @@ async def flaky(): if calls < 3: raise ConnectionError("not up yet") - await wait_for_backends([Target("redis", flaky, ["r:6379"])], timeout=30, sleep=_no_sleep) + await wait_for_backends([Target("store", flaky)], timeout=30, sleep=_no_sleep) assert calls == 3 @@ -206,37 +275,73 @@ async def hang(): @pytest.mark.asyncio -async def test_store_without_ping_is_skipped_with_info(monkeypatch, caplog): - container = _container(monkeypatch, store=_NoPingStore()) +async def test_component_without_aping_is_skipped_with_info(caplog): + container = _container(store=_NoPing()) with caplog.at_level(logging.INFO, logger="tenxgraph_api.startup"): await check_backends_at_startup(container, timeout=0) assert "store has nothing to check; skipped" in caplog.text +# ---------------------------------------------------------------- real Redis client + + +@pytest.mark.asyncio +async def test_redis_cache_store_at_a_closed_port_names_the_endpoint_not_the_password( + tmp_path, unreachable_redis_url, caplog +): + config = _config(tmp_path, {"rate_limit": {"enabled": True, "fail_open": False}}) + container = _container(config=config) + settings = SimpleNamespace(REDIS_URL=unreachable_redis_url) + store = load_and_bind_cache_store(container, config, settings) + assert isinstance(store, RedisCacheStore) + try: + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.startup"): + with pytest.raises(StartupCheckError) as info: + await check_backends_at_startup(container, timeout=0, attempt_timeout=1.0) + finally: + await store.aclose() + message = str(info.value) + port = unreachable_redis_url.rsplit(":", 1)[1].split("/")[0] + assert f"cache (RedisCacheStore, from REDIS_URL, 127.0.0.1:{port}" in message + assert "required by rate_limit.fail_open=false" in message + assert "s3cret" not in message + assert "s3cret" not in caplog.text + + # ---------------------------------------------------------------- lifespan -def _lifespan_patches(monkeypatch, container): +def _lifespan_patches(monkeypatch, container, attach=None): monkeypatch.setattr(app_main, "container", container) - monkeypatch.setattr(app_main, "attach_all_modules", AsyncMock(return_value=None)) + monkeypatch.setattr(app_main, "attach_all_modules", attach or AsyncMock(return_value=None)) + monkeypatch.setattr(app_main, "_prepare_checkpointer", AsyncMock()) monkeypatch.setattr(app_main, "_cleanup_temp_media_cache", AsyncMock()) monkeypatch.setattr(app_main.settings, "STARTUP_CHECK_TIMEOUT_SECONDS", 0.0) + monkeypatch.setattr(app_main.settings, "STARTUP_CHECKS", True) + + +def _attach_with_redis(config, redis_url, checkpointer=None): + """Stand-in for ``attach_all_modules``: bind what the real one binds for these tests.""" + + async def attach(graph_config, container): + container.bind_instance(BaseCheckpointer, checkpointer or InMemoryCheckpointer()) + load_and_bind_cache_store(container, config, SimpleNamespace(REDIS_URL=redis_url)) + + return attach @pytest.mark.asyncio -async def test_lifespan_refuses_to_start_when_redis_is_unreachable(monkeypatch): - _unreachable_redis(monkeypatch) - _lifespan_patches(monkeypatch, _container(monkeypatch, redis_url=REDIS_URL)) - monkeypatch.setattr(app_main.settings, "STARTUP_CHECKS", True) - with pytest.raises(StartupCheckError, match=r"redis \(127\.0\.0\.1:6399\)"): +async def test_lifespan_refuses_to_start_when_the_checkpointer_is_down(monkeypatch): + container = _container(checkpointer=_Pinged(ConnectionError("db down"))) + _lifespan_patches(monkeypatch, container) + with pytest.raises(StartupCheckError, match=r"checkpointer \(_Pinged\)"): async with app_main.lifespan(FastAPI()): pytest.fail("the app must not start serving") @pytest.mark.asyncio async def test_startup_checks_false_skips_the_gate(monkeypatch, caplog): - _unreachable_redis(monkeypatch) - _lifespan_patches(monkeypatch, _container(monkeypatch, redis_url=REDIS_URL)) + _lifespan_patches(monkeypatch, _container(checkpointer=_Pinged(ConnectionError("down")))) monkeypatch.setattr(app_main.settings, "STARTUP_CHECKS", False) gate = AsyncMock() monkeypatch.setattr(startup_checks, "check_backends_at_startup", gate) @@ -245,3 +350,58 @@ async def test_startup_checks_false_skips_the_gate(monkeypatch, caplog): pass gate.assert_not_called() assert "STARTUP_CHECKS is off" in caplog.text + + +@pytest.mark.asyncio +async def test_leftover_redis_url_does_not_block_a_memory_only_server( + monkeypatch, tmp_path, unreachable_redis_url, caplog +): + """The rc3 bug: in-memory everything plus a stale REDIS_URL refused to boot in production.""" + config = _config(tmp_path, {"rate_limit": {"enabled": True, "requests": 100, "window": 60}}) + container = _container(config=config) + _lifespan_patches( + monkeypatch, container, attach=_attach_with_redis(config, unreachable_redis_url) + ) + monkeypatch.setattr(app_main.settings, "MODE", "production") + app = FastAPI() + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.startup"): + async with app_main.lifespan(app): + assert isinstance(container.get(BaseCacheStore), RedisCacheStore) + warnings = [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] + assert len(warnings) == 1 + assert "cache store RedisCacheStore (from REDIS_URL, 127.0.0.1:" in warnings[0] + assert "locks and caps fall back to per process" in warnings[0] + assert "s3cret" not in caplog.text + + setup_fastapi(container, app) + app.include_router(ping_router) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + resp = await client.get("/health/ready") + assert resp.status_code == 200 + assert resp.json() == {"status": "ready", "checks": {"checkpointer": "ok", "cache": "fail"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("data", "reason"), + [ + ({"token_revocation": True}, "required by token_revocation"), + ({"rate_limit": {"enabled": True, "fail_open": False}}, "rate_limit.fail_open=false"), + ], +) +async def test_fail_closed_feature_with_unreachable_redis_refuses_boot( + monkeypatch, tmp_path, unreachable_redis_url, data, reason +): + config = _config(tmp_path, data) + container = _container(config=config) + _lifespan_patches( + monkeypatch, container, attach=_attach_with_redis(config, unreachable_redis_url) + ) + with pytest.raises(StartupCheckError) as info: + async with app_main.lifespan(FastAPI()): + pytest.fail("the app must not start serving") + message = str(info.value) + assert "cache (RedisCacheStore, from REDIS_URL, 127.0.0.1:" in message + assert reason in message + assert "s3cret" not in message diff --git a/tests/unit_tests/test_stream_thread_naming.py b/tests/unit_tests/test_stream_thread_naming.py index 44749e0f..cd365306 100644 --- a/tests/unit_tests/test_stream_thread_naming.py +++ b/tests/unit_tests/test_stream_thread_naming.py @@ -1,7 +1,5 @@ """Thread naming on the stream path stays off the critical path (G-102).""" -# ruff: noqa: S101 - import asyncio from unittest.mock import AsyncMock, MagicMock @@ -10,7 +8,8 @@ from tenxgraph.core.state import Message, StreamChunk, StreamEvent from tenxgraph.core.state.message_block import TextBlock -from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.graph import thread_lock from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema from tenxgraph_api.src.app.routers.graph.services import graph_service @@ -27,8 +26,8 @@ def no_task_manager(monkeypatch): @pytest.fixture def locks(monkeypatch): - limiter = MemoryConcurrencyLimiter() - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: limiter) + limiter = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: limiter) return limiter diff --git a/tests/unit_tests/test_thread_lock.py b/tests/unit_tests/test_thread_lock.py index 9e398b79..f90c1293 100644 --- a/tests/unit_tests/test_thread_lock.py +++ b/tests/unit_tests/test_thread_lock.py @@ -1,7 +1,5 @@ """Per-thread run lock on invoke, stream, WebSocket and AG-UI, and production error chunks.""" -# ruff: noqa: S101 - import asyncio import gc import json @@ -13,8 +11,9 @@ from pydantic import ValidationError from tenxgraph.core.state import StreamChunk, StreamEvent -from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter, RedisConcurrencyLimiter +from tenxgraph_api.src.app.core.cache_store import InMemoryCacheStore from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.graph import thread_lock from tenxgraph_api.src.app.routers.graph.router import invoke_graph, stream_graph, websocket_graph from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import ( @@ -23,8 +22,6 @@ ) from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService -from .test_concurrency_limiter import _FakeRedis - ALICE = {"user_id": "alice"} SECRET = "redis://:SECRET@10.0.0.6" @@ -33,8 +30,8 @@ @pytest.fixture(autouse=True) def fresh_locks(monkeypatch): - limiter = MemoryConcurrencyLimiter() - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: limiter) + limiter = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: limiter) monkeypatch.setattr(thread_lock, "THREAD_BUSY_WAIT_SECONDS", 0.3) return limiter @@ -224,8 +221,8 @@ async def test_stream_lock_is_released_on_disconnect(fresh_locks): async def test_stream_slot_refusal_gives_the_thread_back(fresh_locks, monkeypatch): from tenxgraph_api.src.app.routers.graph import run_limits - slots = MemoryConcurrencyLimiter() - monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: slots) + slots = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(run_limits, "lease_manager", lambda: slots) service = _service(None) service.runs_policy.return_value = RunsConfig(max_concurrent_per_user=1) service.stream_graph = MagicMock(side_effect=lambda *a, **k: _items("a")) @@ -342,22 +339,22 @@ async def test_ag_ui_busy_thread_answers_409_and_waits_by_default(fresh_locks): # --------------------------------------------------------------------------- -# Redis-backed lock +# Lock shared through the cache store # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_lock_is_shared_between_workers_through_redis(monkeypatch): - redis = _FakeRedis() - worker_a = RedisConcurrencyLimiter(redis, lease_ttl=60) - worker_b = RedisConcurrencyLimiter(redis, lease_ttl=60) + shared = InMemoryCacheStore() # one store behind two workers, as Redis is + worker_a = LeaseManager(shared, lease_ttl=60) + worker_b = LeaseManager(shared, lease_ttl=60) monkeypatch.setattr(thread_lock, "THREAD_BUSY_WAIT_SECONDS", 0.2) - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: worker_a) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: worker_a) lease = await thread_lock.acquire_thread_lock("t1") - assert list(redis.zsets) == ["10xgraph:concurrency:runs:thread:t1"] + assert list(shared._slots) == ["concurrency:runs:thread:t1"] - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: worker_b) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: worker_b) with pytest.raises(HTTPException) as exc: await thread_lock.acquire_thread_lock("t1") assert exc.value.status_code == 409 @@ -373,12 +370,11 @@ async def test_lock_is_shared_between_workers_through_redis(monkeypatch): @pytest.mark.asyncio async def test_crashed_worker_lock_expires(monkeypatch): - redis = _FakeRedis() - limiter = RedisConcurrencyLimiter(redis, lease_ttl=0.2) - monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: limiter) + limiter = LeaseManager(InMemoryCacheStore(), lease_ttl=0.2) + monkeypatch.setattr(thread_lock, "lease_manager", lambda: limiter) monkeypatch.setattr(thread_lock, "THREAD_BUSY_WAIT_SECONDS", 2) lease = await thread_lock.acquire_thread_lock("t1") - for task in list(limiter._refreshers): # the worker dies: nothing renews the lease + for task in list(limiter._renewers): # the worker dies: nothing renews the lease task.cancel() again = await thread_lock.acquire_thread_lock("t1") again.release() @@ -534,8 +530,8 @@ async def test_busy_409_and_run_cap_429_carry_retry_after(fresh_locks, monkeypat from tenxgraph_api.src.app.routers.graph import run_limits - limiter = MemoryConcurrencyLimiter() - monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: limiter) + limiter = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(run_limits, "lease_manager", lambda: limiter) slot = await run_limits.acquire_run_slot(ALICE, RunsConfig(max_concurrent_per_user=1)) with pytest.raises(HTTPException) as capped: await run_limits.acquire_run_slot(ALICE, RunsConfig(max_concurrent_per_user=1)) diff --git a/tests/unit_tests/test_token_revocation.py b/tests/unit_tests/test_token_revocation.py index f95f4903..10695c82 100644 --- a/tests/unit_tests/test_token_revocation.py +++ b/tests/unit_tests/test_token_revocation.py @@ -1,172 +1,133 @@ -"""Token revocation backends and wiring.""" +"""Token revocation on the cache store, and its wiring.""" # ruff: noqa: S101 +import logging import time -from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock import pytest from injectq import InjectQ from tenxgraph_api.src.app.core.auth import revocation from tenxgraph_api.src.app.core.auth.revocation import ( - InMemoryTokenRevocation, - RedisTokenRevocation, - TokenRevocationBackend, + TokenRevocationConfig, TokenRevocationUnavailableError, - build_token_revocation, + is_revoked, + revoke_token, + revoke_user, token_revoked, ) +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore + + +@pytest.fixture(autouse=True) +def _propagate_logs(monkeypatch): + """The app's log setup (run by any test that builds the app) stops ``tenxgraph_api`` + from propagating; caplog listens on the root logger.""" + monkeypatch.setattr(logging.getLogger("tenxgraph_api"), "propagate", True) # Reference time for issued-at values only; token expiry is computed in each test. NOW = time.time() +ON = TokenRevocationConfig() # --------------------------------------------------------------------------- -# in-memory backend +# revoke / check on the cache store # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_memory_revokes_one_token_by_jti(): - backend = InMemoryTokenRevocation() - backend.revoke_token("t-1", expires_at=time.time() + 60) - assert await backend.is_revoked({"user_id": "a", "jti": "t-1"}) is True - assert await backend.is_revoked({"user_id": "a", "jti": "t-2"}) is False - assert await backend.is_revoked({"user_id": "a"}) is False +async def test_revokes_one_token_by_jti(): + store = InMemoryCacheStore() + await revoke_token(store, "t-1", expires_at=time.time() + 60) + assert await is_revoked(store, {"user_id": "a", "jti": "t-1"}) is True + assert await is_revoked(store, {"user_id": "a", "jti": "t-2"}) is False + assert await is_revoked(store, {"user_id": "a"}) is False @pytest.mark.asyncio -async def test_memory_drops_entries_once_the_token_expired(): - backend = InMemoryTokenRevocation() - backend.revoke_token("old", expires_at=time.time() - 1) - assert await backend.is_revoked({"user_id": "a", "jti": "old"}) is False - assert backend._tokens == {} +async def test_token_entry_is_dropped_once_the_token_expired(): + store = InMemoryCacheStore() + await revoke_token(store, "old", expires_at=time.time() - 1) # kept for the 1 s minimum + assert await store.get("revoked:jti:old") == b"1" + await store.delete("revoked:jti:old") + assert await is_revoked(store, {"user_id": "a", "jti": "old"}) is False @pytest.mark.asyncio -async def test_memory_revokes_a_users_older_tokens(): - backend = InMemoryTokenRevocation() - backend.revoke_user("a", before=NOW) - assert await backend.is_revoked({"user_id": "a", "iat": NOW - 10}) is True - assert await backend.is_revoked({"user_id": "a", "iat": NOW}) is True - assert await backend.is_revoked({"user_id": "a", "iat": NOW + 10}) is False # new login - assert await backend.is_revoked({"user_id": "a"}) is True # undated token - assert await backend.is_revoked({"user_id": "b", "iat": NOW - 10}) is False - - -# --------------------------------------------------------------------------- -# Redis backend (fake client) -# --------------------------------------------------------------------------- - - -class _FakeRedis: - def __init__(self): - self.data: dict[str, Any] = {} - self.ttl: dict[str, Any] = {} - self.closed = False - - async def set(self, key, value, ex=None): - self.data[key] = value - self.ttl[key] = ex - - async def get(self, key): - return self.data.get(key) - - async def eval(self, script, numkeys, key, cutoff, ttl): - """Python stand-in for the revoke_user script (keeps the newest cutoff).""" - new = float(cutoff) if cutoff else time.time() - current = self.data.get(key) - if current is not None and float(current) >= new: - return 0 - self.data[key] = repr(new).encode() - self.ttl[key] = int(ttl) or None - return 1 - - async def aclose(self): - self.closed = True +async def test_revokes_a_users_older_tokens(): + store = InMemoryCacheStore() + await revoke_user(store, "a", before=NOW) + assert await is_revoked(store, {"user_id": "a", "iat": NOW - 10}) is True + assert await is_revoked(store, {"user_id": "a", "iat": NOW}) is True + assert await is_revoked(store, {"user_id": "a", "iat": NOW + 10}) is False # new login + assert await is_revoked(store, {"user_id": "a"}) is True # undated token + assert await is_revoked(store, {"user_id": "b", "iat": NOW - 10}) is False + assert await is_revoked(store, {"jti": "nothing"}) is False + assert await is_revoked(store, {}) is False @pytest.mark.asyncio -async def test_redis_token_and_user_revocation(): - client = _FakeRedis() - backend = RedisTokenRevocation(client=client, prefix="p") - await backend.revoke_token("t-1", expires_at=time.time() + 120) - assert 1 <= client.ttl["p:jti:t-1"] <= 120 - assert await backend.is_revoked({"user_id": "a", "jti": "t-1"}) is True - assert await backend.is_revoked({"user_id": "a", "jti": "t-2", "iat": NOW}) is False - - await backend.revoke_user("a", before=NOW, ttl=3600) - assert client.ttl["p:user:a"] == 3600 - assert await backend.is_revoked({"user_id": "a", "iat": NOW - 1}) is True - assert await backend.is_revoked({"user_id": "a", "iat": NOW + 1}) is False - assert await backend.is_revoked({"jti": "nothing"}) is False - assert await backend.is_revoked({}) is False - - await backend.aclose() - assert client.closed - - -def test_redis_needs_a_url_or_client(): - with pytest.raises(ValueError, match="url or client"): - RedisTokenRevocation() +async def test_revoke_user_defaults_to_the_store_clock(): + store = InMemoryCacheStore() + store.now = AsyncMock(return_value=5000.0) # type: ignore[method-assign] + await revoke_user(store, "a") + assert await store.get("revoked:user:a") == b"5000.000000" # --------------------------------------------------------------------------- -# factory and lookup +# token_revoked: configuration and fail closed # --------------------------------------------------------------------------- -class _Custom(TokenRevocationBackend): - async def is_revoked(self, identity): - return False - - -CUSTOM_INSTANCE = _Custom() +@pytest.mark.asyncio +async def test_token_revoked_needs_an_identity_and_the_feature_on(monkeypatch): + store = InMemoryCacheStore() + await revoke_user(store, "a") + assert await token_revoked({}, store, ON) is False # auth off: nothing to revoke + assert await token_revoked({"user_id": "a"}, store, ON) is True + monkeypatch.setattr(revocation, "_bound", lambda: (None, store)) + assert await token_revoked({"user_id": "a"}) is False # not configured -def test_build_token_revocation(): - assert isinstance(build_token_revocation("memory"), InMemoryTokenRevocation) - assert isinstance( - build_token_revocation("redis", "redis://localhost:6379/0"), RedisTokenRevocation - ) - with pytest.raises(ValueError, match="needs a Redis URL"): - build_token_revocation("redis", None) - assert isinstance(build_token_revocation(f"{__name__}:_Custom"), _Custom) - assert build_token_revocation(f"{__name__}:CUSTOM_INSTANCE") is CUSTOM_INSTANCE - with pytest.raises(TypeError): - build_token_revocation(f"{__name__}:NOW") - with pytest.raises(ValueError, match="Unsupported"): - build_token_revocation("sometimes") +@pytest.mark.asyncio +async def test_token_revoked_fails_closed_when_the_store_fails(): + store = MagicMock(spec=BaseCacheStore) + store.get = AsyncMock(side_effect=ConnectionError("down")) + with pytest.raises(TokenRevocationUnavailableError): + await token_revoked({"user_id": "a"}, store, ON) @pytest.mark.asyncio -async def test_token_revoked_needs_an_identity_and_a_backend(monkeypatch): - backend = InMemoryTokenRevocation() - backend.revoke_user("a") - assert await token_revoked({}, backend) is False # auth off: nothing to revoke - assert await token_revoked({"user_id": "a"}, backend) is True - monkeypatch.setattr(revocation, "_bound_backend", lambda: None) - assert await token_revoked({"user_id": "a"}) is False # not configured +async def test_token_revoked_fails_closed_without_a_cache_store(monkeypatch): + monkeypatch.setattr(revocation, "_bound", lambda: (ON, None)) + with pytest.raises(TokenRevocationUnavailableError): + await token_revoked({"user_id": "a"}) @pytest.mark.asyncio -async def test_token_revoked_fails_closed(): - class _Down(TokenRevocationBackend): - async def is_revoked(self, identity): - raise ConnectionError("down") +async def test_token_revoked_uses_the_configured_token_lifetime(): + store = InMemoryCacheStore() + await revoke_user(store, "a", before=NOW) + fresh = {"user_id": "a", "exp": NOW + 3600 + 100} + assert await token_revoked(fresh, store, ON) is True + assert await token_revoked(fresh, store, TokenRevocationConfig(token_lifetime=3600)) is False - with pytest.raises(TokenRevocationUnavailableError): - await token_revoked({"user_id": "a"}, _Down()) + +def test_unbound_settings_mean_revocation_is_off(monkeypatch): + container = InjectQ() + monkeypatch.setattr(InjectQ, "get_instance", classmethod(lambda cls: container)) + assert revocation._bound() == (None, None) -def test_bound_backend_outside_a_request_uses_the_global_container(monkeypatch): +def test_bound_settings_outside_a_request_use_the_global_container(monkeypatch): container = InjectQ() - backend = InMemoryTokenRevocation() - container.bind_instance(TokenRevocationBackend, backend) + store = InMemoryCacheStore() + container.bind_instance(TokenRevocationConfig, ON) + container.bind_instance(BaseCacheStore, store) monkeypatch.setattr(InjectQ, "get_instance", classmethod(lambda cls: container)) - assert revocation._bound_backend() is backend + assert revocation._bound() == (ON, store) # --------------------------------------------------------------------------- @@ -206,56 +167,119 @@ def _config(spec): from tenxgraph_api.src.app.core.config.graph_config import GraphConfig config = MagicMock(spec=GraphConfig) - config.token_revocation = spec + config.token_revocation = TokenRevocationConfig.parse(spec) return config -def test_loader_binds_the_configured_backend(): - from tenxgraph_api.src.app.loader import _bind_token_revocation +def test_loader_binds_the_settings(): + from tenxgraph_api.src.app.loader import _enable_token_revocation + + container = InjectQ() + container.bind_instance(BaseCacheStore, InMemoryCacheStore()) + _enable_token_revocation(container, _config({"token_lifetime": 60}), True) + assert container.try_get(TokenRevocationConfig) == TokenRevocationConfig(token_lifetime=60) + + +@pytest.mark.parametrize(("spec", "auth"), [(None, True), (False, True), (True, False)]) +def test_loader_skips_when_off_or_without_auth(spec, auth): + from tenxgraph_api.src.app.loader import _enable_token_revocation container = InjectQ() - _bind_token_revocation(container, _config("memory"), True, None) - assert isinstance(container.try_get(TokenRevocationBackend), InMemoryTokenRevocation) + _enable_token_revocation(container, _config(spec), auth) + assert TokenRevocationConfig not in container -def test_loader_keeps_a_backend_the_developer_bound(): - from tenxgraph_api.src.app.loader import _bind_token_revocation +@pytest.mark.parametrize( + ("mode", "workers", "warned"), + [ + ("production", "1", True), + ("development", "4", True), + ("development", "1", False), + ], +) +def test_loader_warns_when_revocations_stay_in_one_process( + monkeypatch, caplog, mode, workers, warned +): + from tenxgraph_api.src.app import loader + monkeypatch.setattr(loader, "_is_production", lambda: mode == "production") + monkeypatch.setenv("WEB_CONCURRENCY", workers) container = InjectQ() - container.bind_instance(TokenRevocationBackend, CUSTOM_INSTANCE) - _bind_token_revocation(container, _config("memory"), True, None) - assert container.try_get(TokenRevocationBackend) is CUSTOM_INSTANCE + container.bind_instance(BaseCacheStore, InMemoryCacheStore()) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api"): + loader._enable_token_revocation(container, _config(True), True) + found = any("only the process" in r.getMessage() for r in caplog.records) + assert found is warned -@pytest.mark.parametrize(("spec", "auth"), [(None, True), ("memory", False)]) -def test_loader_skips_when_off_or_without_auth(spec, auth): - from tenxgraph_api.src.app.loader import _bind_token_revocation +def test_loader_does_not_warn_with_a_shared_cache_store(monkeypatch, caplog): + from tenxgraph_api.src.app import loader + from tenxgraph_api.src.app.core.cache_store import RedisCacheStore + monkeypatch.setattr(loader, "_is_production", lambda: True) container = InjectQ() - with patch.object(container, "bind_instance") as bind: - _bind_token_revocation(container, _config(spec), auth, None) - bind.assert_not_called() + container.bind_instance(BaseCacheStore, RedisCacheStore(client=MagicMock())) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api"): + loader._enable_token_revocation(container, _config(True), True) + assert not any("only the process" in r.getMessage() for r in caplog.records) + + +# --------------------------------------------------------------------------- +# config +# --------------------------------------------------------------------------- def test_graph_config_token_revocation(tmp_path): + from tenxgraph_api.src.app.core.config.graph_config import ( + DEFAULT_USER_REVOCATION_TTL, + GraphConfig, + ) + + path = tmp_path / "10xgraph.json" + + def parsed(value): + path.write_text(f'{{"agent": "g:app", "token_revocation": {value}}}') + return GraphConfig(str(path)).token_revocation + + assert parsed("true") == TokenRevocationConfig() + assert parsed("true").user_ttl == DEFAULT_USER_REVOCATION_TTL + assert parsed("false") is None + assert parsed("null") is None + assert parsed('{"enabled": false}') is None + assert parsed('{"token_lifetime": 3600, "user_ttl": 60}') == TokenRevocationConfig( + token_lifetime=3600, user_ttl=60 + ) + assert parsed('{"enabled": true, "user_ttl": null}').user_ttl is None + for bad in ("5", '{"token_lifetime": -1}', '{"user_ttl": "1d"}', '{"backend": "redis"}'): + with pytest.raises(ValueError, match="token_revocation"): + parsed(bad) + + +@pytest.mark.parametrize("value", ['"redis"', '"memory"', '"my.mod:backend"']) +def test_removed_token_revocation_strings_name_the_replacement(tmp_path, value): from tenxgraph_api.src.app.core.config.graph_config import GraphConfig path = tmp_path / "10xgraph.json" - path.write_text('{"agent": "g:app", "token_revocation": "redis"}') - assert GraphConfig(str(path)).token_revocation == "redis" - path.write_text('{"agent": "g:app", "token_revocation": 5}') - with pytest.raises(ValueError, match="token_revocation"): + path.write_text(f'{{"agent": "g:app", "token_revocation": {value}}}') + with pytest.raises(ValueError, match="cache_store") as exc: _ = GraphConfig(str(path)).token_revocation + assert '"token_revocation": true' in str(exc.value) -def test_config_editor_validates_token_revocation(): +def test_config_editor_validates_token_revocation(monkeypatch): from tenxgraph_api.cli.config_editor import validate_config - base = {"agent": "graph:app", "auth": "jwt"} - issues = validate_config({**base, "token_revocation": "memory"}) + monkeypatch.delenv("REDIS_URL", raising=False) + base = {"agent": "graph:app", "auth": "jwt", "redis": "redis://cache:6379/0"} + issues = validate_config({**base, "token_revocation": True}) assert not [i for i in issues if i["path"].startswith("token_revocation")] - assert any( - i["level"] == "error" for i in validate_config({**base, "token_revocation": "bad path"}) - ) - warn = validate_config({"agent": "graph:app", "token_revocation": "memory"}) + for removed in ("memory", "redis", "bad path"): + errors = [ + i + for i in validate_config({**base, "token_revocation": removed}) + if i["level"] == "error" + ] + assert [i["path"] for i in errors] == ["token_revocation"] + assert "cache_store" in errors[0]["message"] + warn = validate_config({"agent": "graph:app", "token_revocation": True}) assert [i["level"] for i in warn if i["path"] == "token_revocation"] == ["warning"] diff --git a/tests/unit_tests/test_websocket_guard.py b/tests/unit_tests/test_websocket_guard.py index fb0f69ff..cc931597 100644 --- a/tests/unit_tests/test_websocket_guard.py +++ b/tests/unit_tests/test_websocket_guard.py @@ -2,12 +2,13 @@ Covers what middleware cannot, because rate-limit / request-size middleware are HTTP-only: - the global rate limit applied at the WS handshake (shared backend/bucket with REST), - - the per-process concurrent-connection cap (websocket.max_connections), + - the concurrent-connection cap (websocket.max_connections), - bearer token via the Sec-WebSocket-Protocol sentinel (kept out of URLs/logs). These drive the guard through FastAPI's real DI on a @app.websocket route. """ +import asyncio from typing import Any import pytest @@ -25,8 +26,9 @@ RequirePermission, ws_bearer_subprotocol, ) +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore, SlotResult from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, WebSocketConfig -from tenxgraph_api.src.app.core.middleware.rate_limit.base import RateLimitDecision +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.routers.graph import realtime_guard from tenxgraph_api.src.app.routers.graph.realtime_guard import realtime_connection_guard @@ -44,17 +46,23 @@ async def authorize(self, user, resource, action, resource_id=None): # type: ig return True -class _FakeRateBackend: +class _FakeRateBackend(InMemoryCacheStore): + """A cache store whose rate-limit windows always grant (or always deny).""" + def __init__(self, allowed: bool): + super().__init__() self._allowed = allowed self.calls = 0 - async def check(self, key, *, limit, window): + async def acquire(self, key, limit, ttl): + if not key.startswith("ratelimit:"): + return await super().acquire(key, limit, ttl) self.calls += 1 - return RateLimitDecision(allowed=self._allowed, remaining=0, reset_after=5) - - async def close(self): - pass + if self._allowed: + return SlotResult(granted=True, count=1, oldest_expiry=None, slot_id="s") + return SlotResult( + granted=False, count=limit, oldest_expiry=await self.now() + 5, slot_id=None + ) class _StubConfig: @@ -81,10 +89,11 @@ def _build_app(config, rate_backend=None, *, auth=False): container.bind_instance(BaseAuth, _FakeAuth(), allow_none=True) container.bind_instance(AuthorizationBackend, _AllowAuthz(), allow_none=True) + if rate_backend is not None: + container.bind_instance(BaseCacheStore, rate_backend) + app = FastAPI() setup_fastapi(container, app) - if rate_backend is not None: - app.state.rate_limit_backend = rate_backend deps = [Depends(realtime_connection_guard)] if auth: @@ -99,17 +108,19 @@ async def ws( user: dict[str, Any] = (deps[1] if auth else Depends(lambda: {})), # noqa: PIE807 ): await websocket.accept(subprotocol=ws_bearer_subprotocol(websocket)) - await websocket.send_json({"active": realtime_guard._registry.active, "user": user}) + active = realtime_guard.lease_manager().held("ws:all") + await websocket.send_json({"active": active, "user": user}) await websocket.close() return app @pytest.fixture(autouse=True) -def _reset_registry(): - realtime_guard._registry._active = 0 - yield - realtime_guard._registry._active = 0 +def leases(monkeypatch): + """A fresh in-process lease manager behind the connection cap.""" + manager = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(realtime_guard, "lease_manager", lambda: manager) + return manager class TestConcurrencyCap: @@ -127,11 +138,12 @@ def test_slot_released_after_disconnect(self): for _ in range(2): with client.websocket_connect("/ws") as conn: assert conn.receive_json()["active"] == 1 - assert realtime_guard._registry.active == 0 + assert realtime_guard.lease_manager().held("ws:all") == 0 - def test_over_cap_rejected(self): + def test_over_cap_rejected(self, leases): app = _build_app(_StubConfig(None, 1)) - realtime_guard._registry._active = 1 # simulate one already-active connection + # Simulate one already-active connection (held by another worker, or another socket). + asyncio.run(leases.cache.acquire("concurrency:ws:all", 1, 60)) client = TestClient(app) with pytest.raises(WebSocketDisconnect) as exc: with client.websocket_connect("/ws") as conn: @@ -168,6 +180,7 @@ class _RL: requests = 100 window = 60 trusted_proxy_headers = False + fail_open = True class TestSubprotocolToken: diff --git a/tests/unit_tests/test_websocket_limits.py b/tests/unit_tests/test_websocket_limits.py index 57e1277c..2d4662d7 100644 --- a/tests/unit_tests/test_websocket_limits.py +++ b/tests/unit_tests/test_websocket_limits.py @@ -5,8 +5,6 @@ lifetime of the connection. """ -# ruff: noqa: S101, PLR2004 - from typing import Any import pytest @@ -16,9 +14,10 @@ from injectq.integrations import setup_fastapi from starlette.websockets import WebSocketDisconnect +from tenxgraph_api.src.app.core.cache_store import BaseCacheStore, InMemoryCacheStore, SlotResult from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, WebSocketConfig +from tenxgraph_api.src.app.core.leases import LeaseManager from tenxgraph_api.src.app.core.middleware.rate_limit import keying -from tenxgraph_api.src.app.core.middleware.rate_limit.base import RateLimitDecision from tenxgraph_api.src.app.routers.graph import realtime_guard from tenxgraph_api.src.app.routers.graph.realtime_guard import ( realtime_connection_guard, @@ -27,19 +26,23 @@ ) -class _Backend: - """Allows the first ``budget`` checks, then denies.""" +class _Backend(InMemoryCacheStore): + """A cache store whose rate-limit window grants the first ``budget`` requests.""" def __init__(self, budget: int): + super().__init__() self.budget = budget self.calls = 0 - async def check(self, key, *, limit, window): + async def acquire(self, key, limit, ttl): + if not key.startswith("ratelimit:"): + return await super().acquire(key, limit, ttl) self.calls += 1 - return RateLimitDecision(allowed=self.calls <= self.budget, remaining=0, reset_after=7) - - async def close(self): - pass + if self.calls <= self.budget: + return SlotResult(granted=True, count=self.calls, oldest_expiry=None, slot_id="s") + return SlotResult( + granted=False, count=limit, oldest_expiry=await self.now() + 7, slot_id=None + ) class _RL: @@ -49,6 +52,7 @@ class _RL: trusted_proxy_headers = False trusted_proxy_hops = 1 trusted_proxies = () + fail_open = True class _Config: @@ -71,10 +75,10 @@ def websocket(self): def _app(config, backend=None) -> TestClient: container = InjectQ() container.bind_instance(GraphConfig, config, allow_none=True) + if backend is not None: + container.bind_instance(BaseCacheStore, backend) app = FastAPI() setup_fastapi(container, app) - if backend is not None: - app.state.rate_limit_backend = backend @app.websocket("/ws") async def ws(websocket: WebSocket, _guard: None = Depends(realtime_connection_guard)): @@ -91,12 +95,11 @@ async def ws(websocket: WebSocket, _guard: None = Depends(realtime_connection_gu @pytest.fixture(autouse=True) -def _reset_registry(): - realtime_guard._registry._active = 0 - realtime_guard._registry._per_user.clear() - yield - realtime_guard._registry._active = 0 - realtime_guard._registry._per_user.clear() +def leases(monkeypatch): + """A fresh in-process lease manager behind the connection caps.""" + manager = LeaseManager(InMemoryCacheStore()) + monkeypatch.setattr(realtime_guard, "lease_manager", lambda: manager) + return manager @pytest.fixture @@ -154,7 +157,7 @@ def test_one_user_cannot_hold_every_slot(caller): second.send_text("x") second.receive_json() assert exc.value.code == realtime_guard.WS_TRY_AGAIN_LATER - assert realtime_guard._registry.active == 0 + assert realtime_guard.lease_manager().held("ws:all") == 0 def test_other_users_are_not_affected(caller): @@ -174,7 +177,7 @@ def test_user_slot_is_released_on_disconnect(caller): with client.websocket_connect("/ws") as conn: conn.send_text("x") conn.receive_json() - assert realtime_guard._registry._per_user == {} + assert realtime_guard.lease_manager()._held == {} # --------------------------------------------------------------------------- @@ -228,3 +231,38 @@ def test_identity_check_fails_if_token_now_belongs_to_someone_else(caller): def test_identity_check_skipped_without_auth(caller): user: dict[str, Any] = {} assert ws_identity_still_valid(_Ws(), user) is True + + +# --------------------------------------------------------------------------- +# the handshake and the runs count in the bound cache store (no TestClient socket) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_guard_and_runs_share_the_bound_cache_store(monkeypatch): + from types import SimpleNamespace + + from fastapi import WebSocketException + + class _Global(_RL): + by = "global" + requests = 2 + + store = InMemoryCacheStore() + monkeypatch.setattr(realtime_guard, "bound_cache_store", lambda: store) + monkeypatch.setattr(realtime_guard, "lease_manager", lambda: LeaseManager(store)) + monkeypatch.setattr(keying, "verified_user_id", lambda connection: None) + websocket = SimpleNamespace(state=SimpleNamespace()) + ws_config = WebSocketConfig(max_connections=None, max_connections_per_user=None) + config = _Config(ws_config, rate_limit=_Global()) + + guard = realtime_connection_guard(websocket, config) # type: ignore[arg-type] + await guard.__anext__() # handshake: one request + assert await ws_run_allowed(websocket) is None # run 1: the second request + retry_after = await ws_run_allowed(websocket) # run 2: over the limit + assert retry_after is not None and 1 <= retry_after <= 60 + assert (await store.acquire("ratelimit:__global__", 10, 60)).count == 3 + await guard.aclose() + + with pytest.raises(WebSocketException): # the next handshake is over the limit too + await realtime_connection_guard(websocket, config).__anext__() # type: ignore[arg-type] From b873eee3650a58a5cfa4212a769cc24bebcaa5d9 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Sat, 31 Oct 2026 21:42:41 +0600 Subject: [PATCH 2/5] Refactor thread name generation and cache store configuration - Removed unused thread name generator path from service configuration in test_stream_thread_naming.py. - Updated thread name generator path handling in test_thread_lock.py. - Modified token revocation handling in test_token_revocation.py to use cache_store. - Adjusted rate limit configuration in test_websocket_guard.py to include a cap. - Added comprehensive tests for cache_store configuration in test_cache_store_config.py. - Introduced tests for id_generator configuration in test_id_generator_config.py. - Created tests for thread name generator choices in test_thread_name_choices.py. --- 10xgraph.json | 19 +- CHANGELOG.md | 94 ++-- CLAUDE.md | 27 +- config-editor-ui/src/icons.js | 5 +- config-editor-ui/src/pages.jsx | 43 +- config-editor-ui/src/sidebar.jsx | 7 +- config-editor-ui/src/store.js | 12 +- config-editor-ui/src/widgets.jsx | 311 +++++++++++++- tenxgraph_api/cli/commands/init.py | 38 +- tenxgraph_api/cli/commands/revoke.py | 24 +- tenxgraph_api/cli/config_editor/schema.py | 404 +++++++++++------- .../cli/config_editor/static/app.css | 2 +- tenxgraph_api/cli/config_editor/static/app.js | 6 +- tenxgraph_api/cli/config_editor/validation.py | 153 ++++--- tenxgraph_api/cli/main.py | 6 +- .../cli/templates/prod/10xgraph.json | 25 +- .../10xgraph/references/api-configuration.md | 39 +- .../references/api-settings-and-middleware.md | 21 +- .../references/auth-and-authorization.md | 18 +- .../id-and-thread-name-generators.md | 33 +- .../10xgraph/references/production-runtime.md | 2 +- .../10xgraph/references/rate-limiting.md | 73 ++-- .../skills/copilot/10xgraph.instructions.md | 4 +- tenxgraph_api/src/app/core/auth/revocation.py | 15 +- .../src/app/core/cache_store/loader.py | 70 ++- .../src/app/core/cache_store/memory_store.py | 37 +- .../src/app/core/config/graph_config.py | 321 ++++++++++---- tenxgraph_api/src/app/core/config/settings.py | 9 +- .../src/app/core/config/setup_middleware.py | 2 +- .../app/core/middleware/rate_limit/limiter.py | 30 +- tenxgraph_api/src/app/loader.py | 208 ++++++++- tenxgraph_api/src/app/main.py | 3 - .../routers/graph/services/graph_service.py | 46 +- .../src/app/routers/ping/readiness.py | 42 +- .../src/app/routers/ping/startup_checks.py | 4 +- tenxgraph_api/src/app/utils/__init__.py | 7 +- .../src/app/utils/thread_name_generator.py | 32 ++ tests/cli/test_group6_cli_templates.py | 109 +++-- tests/cli/test_init_non_interactive.py | 27 +- tests/cli/test_init_prod.py | 8 +- tests/cli/test_production_readiness.py | 68 +-- tests/integration_tests/test_ag_ui_api.py | 4 +- .../cache_store/test_memory_cache_store.py | 24 ++ tests/unit_tests/test_cache_store_config.py | 215 ++++++++++ tests/unit_tests/test_closing_and_cancel.py | 1 - tests/unit_tests/test_concurrency_limiter.py | 104 +++-- tests/unit_tests/test_config_editor.py | 188 +++++--- tests/unit_tests/test_fix_graph.py | 4 +- tests/unit_tests/test_graph_config.py | 14 - tests/unit_tests/test_graph_service.py | 25 +- tests/unit_tests/test_id_generator_config.py | 313 ++++++++++++++ tests/unit_tests/test_loader.py | 14 +- .../test_ownership_authorization.py | 4 +- tests/unit_tests/test_rate_limit.py | 6 +- .../unit_tests/test_rate_limit_unavailable.py | 22 +- tests/unit_tests/test_readiness.py | 36 +- .../test_resume_binding_and_limits.py | 2 +- tests/unit_tests/test_run_limits.py | 4 +- tests/unit_tests/test_shutdown_readiness.py | 6 +- tests/unit_tests/test_startup_checks.py | 124 ++++-- tests/unit_tests/test_stream_thread_naming.py | 1 - tests/unit_tests/test_thread_lock.py | 2 +- tests/unit_tests/test_thread_name_choices.py | 198 +++++++++ tests/unit_tests/test_token_revocation.py | 31 +- tests/unit_tests/test_websocket_guard.py | 3 +- 65 files changed, 2877 insertions(+), 872 deletions(-) create mode 100644 tests/unit_tests/test_cache_store_config.py create mode 100644 tests/unit_tests/test_id_generator_config.py create mode 100644 tests/unit_tests/test_thread_name_choices.py diff --git a/10xgraph.json b/10xgraph.json index c692db8b..75e90da7 100644 --- a/10xgraph.json +++ b/10xgraph.json @@ -4,12 +4,15 @@ "store": "graph.dummy_store:store", "env": ".env", "auth": null, - "rate_limit": { - "enabled": true, - "requests": 100, - "window": 60, - "by": "ip", - "trusted_proxy_headers": false, - "exclude_paths": ["/health", "/docs", "/redoc", "/openapi.json"] + "cache_store": { + "type": "memory", + "rate_limit": { + "enabled": true, + "requests": 100, + "window": 60, + "by": "ip", + "trusted_proxy_headers": false, + "exclude_paths": ["/health", "/docs", "/redoc", "/openapi.json"] + } } -} \ No newline at end of file +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 27e333fd..fd1b1bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,9 +30,10 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [1.0.0rc4] - Unreleased The fourth release candidate of `10xgraph-api` 1.0. It requires `10xgraph` 1.0.0rc4 -(checkpointers have `aping()`). Everything the API shares between workers now lives in one -`BaseCacheStore`, and the startup check no longer refuses to boot over a Redis that nothing -configured needs. `10xgraph-client` stays at 1.0.0-rc.2. +(checkpointers have `aping()`). Everything the API shares between workers lives in one +`BaseCacheStore`, configured by one `cache_store` object in `10xgraph.json`, and the startup +check no longer refuses to boot over a Redis that nothing configured needs. +`10xgraph-client` stays at 1.0.0-rc.2. ### Breaking @@ -42,51 +43,84 @@ configured needs. `10xgraph-client` stays at 1.0.0-rc.2. `MemoryRateLimitBackend`, `RedisRateLimitBackend` and the rate-limit factory). Run locks and caps, idempotency, the ownership cache, token revocation and rate limiting all use the bound `BaseCacheStore`; implement one to replace Redis. -- `rate_limit.backend` and `rate_limit.redis` are removed. Either key is a config error naming - the replacement: the counters live in the cache store (`cache_store`, else `redis`, else - `REDIS_URL`, else in memory). Delete both keys; move a dedicated rate-limit Redis URL to - `redis` (one Redis now serves every feature). -- `token_revocation` is `true`, `false`, `null` or `{"enabled": true, "token_lifetime": ..., - "user_ttl": ...}`. The string values (`"redis"`, `"memory"`, `"module:attr"`) are config - errors naming `cache_store`. Replace `"redis"` with `true` and set `redis` (or `REDIS_URL`). +- `cache_store` is one object that holds the store and everything it backs: + `{"type": "memory" | "redis" | "custom", "url": "${REDIS_URL}", "path": "module:attr", + "rate_limit": {...}, "token_revocation": true}`. `url` goes with type `redis` (optional: + empty uses the `REDIS_URL` environment variable; `$VAR` / `${VAR}` expanded, an unset variable + is a config error) and `path` with type `custom` + (a `BaseCacheStore` instance or zero-argument factory). No `cache_store`, or no `type`, is the + in-memory store; `rate_limit` and `token_revocation` are off when absent. Unknown keys inside + `cache_store`, `rate_limit` and `token_revocation` are config errors listing the valid keys. +- The top-level `redis`, `rate_limit` and `token_revocation` keys are config errors at load that + say where each setting goes; a leftover `rate_limit` can no longer turn rate limiting off + without a word. `rate_limit` inside `cache_store` has no `backend` or `redis` key. + `token_revocation` is `true`, `false`, `null` or `{"enabled", "token_lifetime", "user_ttl"}`. +- `REDIS_URL` no longer selects the cache store, and `Settings.REDIS_URL` is gone: the store + comes only from `cache_store` (write `"url": "${REDIS_URL}"` to keep reading the variable). + This fixes rc3 refusing to boot an in-memory server in production with a stale `REDIS_URL`. +- `cache_store.type` `"redis"` without the `redis` package refuses to boot with + `pip install "10xgraph-api[redis]"`; it never falls back to a per-process store. - `GET /health/ready`: the `redis` key in `checks` is renamed `cache` and reports the bound cache store (`{"checkpointer": "ok", "store": "ok", "cache": "ok"}`). Readiness and the startup check only call `aping()` on the bound checkpointer, store and cache store; a component's `readiness_check()` hook and the probing of private checkpointer attributes are gone. -- `10xgraph init --rate-limit redis` writes `"redis": "${REDIS_URL}"` and, with auth, - `"token_revocation": true` (was `rate_limit.backend: "redis"`, `rate_limit.redis.url` and - `"token_revocation": "redis"`). `--rate-limit memory` turns rate limiting on without a - `redis` key. +- `10xgraph init --rate-limit redis` writes `"cache_store": {"type": "redis", "url": + "${REDIS_URL}", "rate_limit": {...}}`, plus `"token_revocation": true` inside with auth; + `--rate-limit memory` writes `{"type": "memory", "rate_limit": {...}}`; `none` writes no + `cache_store`. The production template uses the Redis form with both features. +- `thread_name_generator` takes `"random"` (the default), `"first_message"`, `"none"` or + `"module:attr"`. With the key unset, new threads are now named (randomly) and the name is + saved; before, the default name was generated and dropped. `GraphConfig.thread_name_generator_path` + is replaced by `GraphConfig.thread_name_generator`. ### Added - `BaseCacheStore` with `RedisCacheStore` (also Valkey, Dragonfly, KeyDB) and `InMemoryCacheStore`, exported from `tenxgraph_api`. A store has an abstract `aping()` and an optional `describe()` that names its endpoint (never credentials) in logs and startup errors. -- `"cache_store": "module:attr"` in `10xgraph.json`: a `BaseCacheStore` instance or zero-argument - factory; wins over `redis` and `REDIS_URL`. One INFO line at boot names the store and its - source, for example `Cache store: RedisCacheStore (from REDIS_URL, localhost:6379)`. + One INFO line at boot names the store and the config key it came from, for example + `Cache store: RedisCacheStore (cache_store.url, localhost:6379)`. - A contract suite for custom cache stores in `tenxgraph_api.testing.cache_store_contract`. -- The `redis` key expands `$VAR` and `${VAR}`; an unset variable is a config error. -- `10xgraph config` edits `cache_store`, and token revocation as a section with `enabled`, - `token_lifetime` and `user_ttl`. +- `id_generator` in `10xgraph.json`: `uuid`, `hex`, `short`, `int`, `bigint`, `timestamp`, + `snowflake` (`SnowFlakeIdGenerator`, needs the `snowflakekit` extra and reads the `SNOWFLAKE_*` + environment variables) or `"module:attr"` (a `BaseIDGenerator` instance, subclass or + zero-argument factory). The API binds it for the loaded graph, so new thread, message and + memory ids all come from it. Boot refuses when the graph is also compiled with its own + generator (`StateGraph(id_generator=...)`), and when a `PgCheckpointer`'s `id_type` cannot + hold its ids. Unset keeps the graph's own generator. +- `thread_name_generator: "first_message"`: the user's first message, whitespace collapsed, + cut to at most 60 characters on a word boundary (`FirstMessageThreadNameGenerator`). +- `10xgraph config` edits token revocation as a section with `enabled`, `token_lifetime` and + `user_ttl`. ### Changed - The startup check is fatal for the cache store only when a fail-closed feature uses it: - `token_revocation` on, or `rate_limit` with `fail_open: false`. Otherwise an unreachable - cache store is one WARNING (`... unreachable: locks and caps fall back to per process ...`) - and the server starts; `/health/ready` reports `cache: fail` and stays 200. This fixes rc3 - refusing to boot an in-memory server with a leftover `REDIS_URL`. Startup errors name each - component's class, where its setting came from (`REDIS_URL`, `redis in 10xgraph.json`, - `cache_store`) and its `describe()` endpoint. The hint now reads "Start the backend, fix its - URL, or set STARTUP_CHECKS=false to skip this check." -- `10xgraph revoke` writes to the cache store the server would pick (`cache_store`, `redis`, - `REDIS_URL`; `--redis-url` overrides) and refuses with a clear error when that is the - in-memory store. + `cache_store.token_revocation` on (with auth), or `cache_store.rate_limit` with + `fail_open: false`. Otherwise an unreachable cache store is one WARNING (`... unreachable: + locks and caps fall back to per process ...`) and the server starts; `/health/ready` reports + `cache: fail` and stays 200. Startup errors name each component's class, the config key of + the cache store (`cache_store.url`, `cache_store.path ...`) and its `describe()` endpoint. + The hint reads "Start the backend, fix its URL, or set STARTUP_CHECKS=false to skip this + check." +- `10xgraph revoke` writes to the cache store `cache_store` selects (`--redis-url` overrides) + and refuses with a clear error when that is the in-memory store. - Shutdown closes the cache store once, after the graph and after pending lease releases. The authorization backend no longer closes a Redis client. +- `token_revocation` makes the cache store required (startup check, `/health/ready`) only when + auth is on, the same rule the loader uses to enable it. +- `cache_store` is parsed once per raw value, so its config warnings (such as + `trusted_proxy_headers` without `trusted_proxies`) are not repeated per readiness probe. +- A rate-limit denial stays a 429 when reading the cache store clock for `Retry-After` fails; + it no longer turns into an allow or a 503. +- The in-memory cache store keeps slot and key expiries on the monotonic clock, so a wall clock + jump (host suspend, NTP step) no longer expires a held thread lock early. `now()` and + `oldest_expiry` are still epoch seconds. +- `revoke_user` stores the cutoff rounded up to the microsecond, so a token issued at the + cutoff itself is always revoked (it was rounded to the nearest). +- Thread names from `thread_name_generator` get the user's messages first, then the replies + (before: the run's output messages only). --- diff --git a/CLAUDE.md b/CLAUDE.md index 373d6d49..e4ed09a1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ Public exports from the package root (`from tenxgraph_api import ...`): `BaseAut | Command | Purpose | Notable options | |---|---|---| | `10xgraph dev` | The only server command: local Uvicorn dev server (reload on) via `commands/api.py::APICommand`, opens the hosted playground when ready. `api` and `play` were removed (use `dev --no-open` / `dev`); production runs Gunicorn in the `10xgraph build` image | `--config/-c` (default `10xgraph.json`), `--host/-H`, `--port/-p` (8000), `--reload/--no-reload`, `--open/--no-open`, `-v/-q` | -| `10xgraph init` | Interactively scaffold a project (guided prompts pick dev vs production, auth, rate limit) | `--path/-p`, `--force/-f`, `--name`, `--template` (quick-start\|production), `--auth` (none\|jwt\|custom), `--rate-limit` (none\|memory\|redis: `memory` = rate limit on, no `redis` key; `redis` = rate limit on plus `"redis": "${REDIS_URL}"` and, with auth, `"token_revocation": true`), `--provider` (google\|openai\|anthropic), `--websocket/--no-websocket` (default off; writes `websocket.enabled`). Prod `pyproject.toml` depends on `10xgraph[]` and `10xgraph-api[jwt,redis,websocket]` as chosen, `--yes/-y`, `--non-interactive`, `--dry-run`. There is **no `--prod` flag** | +| `10xgraph init` | Interactively scaffold a project (guided prompts pick dev vs production, auth, rate limit) | `--path/-p`, `--force/-f`, `--name`, `--template` (quick-start\|production), `--auth` (none\|jwt\|custom), `--rate-limit` (none\|memory\|redis: `none` = no `cache_store`; `memory` = `"cache_store": {"type": "memory", "rate_limit": {...}}`; `redis` = `{"type": "redis", "url": "${REDIS_URL}", "rate_limit": {...}}` plus `"token_revocation": true` inside with auth), `--provider` (google\|openai\|anthropic), `--websocket/--no-websocket` (default off; writes `websocket.enabled`). Prod `pyproject.toml` depends on `10xgraph[]` and `10xgraph-api[jwt,redis,websocket]` as chosen, `--yes/-y`, `--non-interactive`, `--dry-run`. There is **no `--prod` flag** | | `10xgraph build` | Generate a `Dockerfile` (and optionally `docker-compose.yml` / `k8s.yaml`). The builder stage strips CLI assets, runs `tenxgraph_api/_slim_image.py` (uninstalls typer/rich/questionary and what only they need, unless another package needs them) and removes pip, so the image carries no CLI code paths | `--output/-o`, `--force/-f`, `--python-version` (3.13), `--port`, `--docker-compose/--no-docker-compose`, `--k8s/--no-k8s`, `--service-name` | | `10xgraph eval` | Run agent evaluations; discovers `*_eval.py`/`eval_*.py`, runs cases (optionally `--parallel`), writes JSON and HTML reports to `eval_reports/` and opens the HTML report in the browser (only at an interactive terminal: skipped with `--no-open`, `--no-report`, `-q`, json/jsonl, `--non-interactive`, `CI`, non-TTY; `evaluation.open_report` in `10xgraph.json` sets the default). Simulations print per-turn step lines via `ProgressRun.step` | `--output/-o`, `--no-report`, `--threshold/-t`, `--parallel/-p`, `--max-concurrency/-c`, `--open/--no-open` | | `10xgraph test` | Run project tests via pytest (args after `--` forwarded verbatim) | `--coverage/-C`, `--html`, `-k`, path arg | @@ -86,30 +86,30 @@ Parsed by `tenxgraph_api/src/app/core/config/graph_config.py`. Supported keys: |---|---| | `agent` (required) | `"module:attribute"` resolving to a `CompiledGraph`. The loader accepts a `CompiledGraph` object, a sync/async factory returning one, or a callable. | | `env` | Path to a `.env` file, loaded at config-load time | -| `thread_name_generator` | `"module:attr"` -> a `ThreadNameGenerator` | +| `thread_name_generator` | `"random"` (default, also when unset: `DefaultThreadNameGenerator`), `"first_message"` (`FirstMessageThreadNameGenerator`: the user's first message, whitespace collapsed, cut to at most 60 characters on a word boundary; random when there is no text), `"none"` (threads stay unnamed but are saved) or `"module:attr"` -> a `ThreadNameGenerator` subclass or instance. Chosen by `loader.resolve_thread_name_generator`; `GraphService._save_thread_name` saves every generated name with `aput_thread`, in the background after the run. The generator gets the user's message texts first, then the replies | +| `id_generator` | Unset: the graph keeps the generator it was compiled with. A name (`graph_config.ID_GENERATOR_NAMES`: `uuid`, `hex`, `short`, `int`, `bigint`, `timestamp`, `snowflake` = `SnowFlakeIdGenerator`, needs the `snowflakekit` extra, reads `SNOWFLAKE_*` env vars) or `"module:attr"` (a `BaseIDGenerator` instance, subclass or zero-arg factory). `loader.apply_id_generator` rebinds `BaseIDGenerator`, `"generated_id_type"` and `"generated_id"` as factories in the graph's container, the API container and `InjectQ.get_instance()`, so thread ids (`GraphService._new_thread_id`), message ids (core `Message` default factory) and memory ids use it. Refuses to boot when the graph is compiled with a non-default generator too, or when a `PgCheckpointer`'s `id_type` cannot hold the ids | | `auth` | `null`, the string `"jwt"`, or `{"method": "custom", "path": "module:attr"}` | | `authorization` | `"module:attr"` -> an `AuthorizationBackend` (RBAC / per-tool access) | | `checkpointer` | `"module:attr"` -> a `BaseCheckpointer` | | `injectq` | `"module:attr"` -> an InjectQ container | | `store` | `"module:attr"` -> a `BaseStore` | -| `redis` | Redis URL (`$VAR` / `${VAR}` expanded; unset variable is a config error) of the API's cache store, a `RedisCacheStore` (also Valkey, Dragonfly, KeyDB). Unset falls back to `REDIS_URL`, then `InMemoryCacheStore` (per process) | -| `cache_store` | `"module:attr"` -> a `BaseCacheStore` instance or zero-arg factory (`core/cache_store/`); wins over `redis` / `REDIS_URL`. Selected and bound at boot by `core/cache_store/loader.py::load_and_bind_cache_store` (one INFO line `Cache store: (, )`), closed once in the lifespan. Holds the run locks and caps, idempotency, ownership cache L2, token revocation and rate-limit counters | -| `rate_limit` | Object (see below) | +| `cache_store` | One object, `graph_config.CacheStoreConfig`: `type` (`memory` default \| `redis` \| `custom`), `url` (type redis only, optional: empty uses the `REDIS_URL` environment variable, a set value wins; `$VAR` / `${VAR}` expanded when the store is built, unset variable is a config error; `RedisCacheStore`, also Valkey, Dragonfly, KeyDB), `path` (type custom only, required; `"module:attr"` -> a `BaseCacheStore` instance or zero-arg factory), `rate_limit` (object, see below), `token_revocation` (see its row). No key or no `type`: `InMemoryCacheStore` (per process). Unknown keys are config errors listing the valid ones. Nothing else picks the store (`REDIS_URL` alone does not). Selected and bound at boot by `core/cache_store/loader.py::load_and_bind_cache_store` (one INFO line `Cache store: (, )`, e.g. `cache_store.url`), closed once in the lifespan; type redis without the `redis` package refuses to boot. Holds the run locks and caps, idempotency, ownership cache L2, token revocation and rate-limit counters. Top-level `redis`, `rate_limit` and `token_revocation` keys are config errors at `GraphConfig` load (`check_cache_store_settings`) saying where the setting goes | | `ag_ui` | `{"enabled": bool}`, default off. Mounts `POST /v1/ag-ui` (AG-UI protocol, for CopilotKit and other AG-UI clients). Needs the `ag-ui` extra | | `websocket` | `enabled` (`false` leaves out `/v1/graph/ws` and `/v1/graph/live`; `true` without `websockets`/`wsproto` refuses to boot; unset keeps them and warns if no library; `routers/setup_router.py::_graph_router`), `max_connections` (1000), `max_connections_per_user` (10), `realtime_models` (models a `/v1/graph/live` client may pick), `realtime_client_overrides` (init options a client may set; default `voice`, `modalities`, `vad`; add `system_prompt` only for trusted callers) | | `runs` | `RunsConfig`: `thread_ids` (`server` default = only existing ids may be named on REST invoke/stream, `/v1/graph/ws` and checkpointer writes, a new thread is started by omitting the id; enforced by the ownership/rbac backends via `OwnershipAuthorizationBackend(server_thread_ids=...)`. AG-UI and `/v1/graph/live` are exempt: they pass `client_thread_ids=True` to `authorize` because their frontends make their own ids, and the caller owns a new one. `client` = any request may name a new thread, explicit opt-in), `config_keys` / `state_keys` (allow-lists for request config and `initial_state`/PUT state; 422 on other keys; `thread_id`, `run_id` always allowed), `max_concurrent_per_user` (HTTP invoke/stream/AG-UI runs in flight, default 10, 429 over it; `routers/graph/run_limits.py`), `timeout` (seconds; invoke 504, streams end with an error chunk), `max_parallel_tool_calls` (positive int, default unset = core's default 10 or env `TENXGRAPH_MAX_PARALLEL_TOOL_CALLS`; set into every run's config in `_prepare_input`, reserved from clients) | -| `token_revocation` | `true`, or `{"enabled", "token_lifetime", "user_ttl"}` (`graph_config.TokenRevocationConfig`); the old strings (`"redis"`, `"memory"`, `module:attr`) are config errors naming `cache_store`. Revocations live in the cache store (`core/auth/revocation.py`: `revoke_token`, `revoke_user`, `is_revoked`), checked by `RequirePermission` (401), before every `/v1/graph/ws` run, and before and every 30 s during a `/v1/graph/live` session (`REALTIME_RECHECK_INTERVAL`, close 1008); cache store errors fail closed (503), and an unreachable cache store refuses the boot | +| `cache_store.token_revocation` | `true`, `false`, `null` or `{"enabled", "token_lifetime", "user_ttl"}` (`graph_config.TokenRevocationConfig`; a string is a config error). Read through `GraphConfig.token_revocation`. Revocations live in the cache store (`core/auth/revocation.py`: `revoke_token`, `revoke_user`, `is_revoked`), checked by `RequirePermission` (401), before every `/v1/graph/ws` run, and before and every 30 s during a `/v1/graph/live` session (`REALTIME_RECHECK_INTERVAL`, close 1008); cache store errors fail closed (503), and an unreachable cache store refuses the boot | | `media` | `delete_with_thread` (default true): delete an uploaded file when the last message or thread that references it is deleted. Links are recorded in the file's `threads` metadata (`MediaService.link_threads` / `unlink_threads`); files without a `threads` key (uploaded before link tracking) and stores without `update_metadata` (core older than the release that added it) are never auto-deleted. `false` still records links. Parsed by `graph_config.MediaConfig` | | `graph_introspection` | `"open"` or `"scoped"` (default scoped in production): `GET /v1/graph`, `/v1/graph/tools`, `/v1/graph:StateSchema` need an explicit `graph:read` scope (`RequirePermission(..., introspection=True)`) | -`rate_limit` object: `enabled`, `requests` (default 100), `window` secs (60), `by` (`ip` | +`cache_store.rate_limit` object (read through `GraphConfig.rate_limit`, None when absent or +`enabled: false`): `enabled`, `requests` (default 100), `window` secs (60), `by` (`ip` | `user` | `global`), `trusted_proxy_headers` (honour `X-Forwarded-For` only when true), `trusted_proxies` (networks the header is honoured from; empty means loopback and private networks only, `DEFAULT_TRUSTED_PROXIES`), `trusted_proxy_hops`, `exclude_paths`, `fail_open` (on a cache store error: allow vs deny with 503 `RATE_LIMIT_UNAVAILABLE`; default true). Counters live in the cache store -(`core/middleware/rate_limit/limiter.py::check`). `backend` and `redis` were removed in rc4; -either key is a config error naming `cache_store`. +(`core/middleware/rate_limit/limiter.py::check`). Any other key is a config error listing the +valid ones (`RATE_LIMIT_KEYS`). ## Auth @@ -151,7 +151,7 @@ either key is a config error naming `cache_store`. which also leaves a tombstone for one L1 TTL (L1 + cache store) so the deleted id cannot be recreated by someone else while another worker still caches the old owner (`ThreadOwnershipResolver.recently_deleted`); the resolver never closes the cache store. -- **Identity reuse:** with `rate_limit.by == "user"` the limiter's verified identity is reused by +- **Identity reuse:** with `cache_store.rate_limit.by == "user"` the limiter's verified identity is reused by `RequirePermission` for backends with `BaseAuth.reusable_within_request = True` (`JwtAuth`), see `core/auth/identity_cache.py`. Look up the app container inside a request with `core/utils/container.py::current_container()`, not `InjectQ.get_instance()`. @@ -220,8 +220,9 @@ either key is a config error naming `cache_store`. graph `resume` input. `GraphInputSchema.resume` also resumes over `/v1/graph/invoke|stream`. - **Ping**: `GET /ping` (liveness). **Readiness**: `GET /health/ready` (`aping()` on the bound checkpointer, store and cache store; keys `checkpointer`, `store`, `cache`; 200 or 503 - with only ok/fail per component; a failed `cache` counts only when `token_revocation` is on - or `rate_limit.fail_open` is false; `routers/ping/readiness.py`). The startup check + with only ok/fail per component; a failed `cache` counts only when + `cache_store.token_revocation` is on (with auth) or `cache_store.rate_limit.fail_open` is + false; `routers/ping/readiness.py`). The startup check (`routers/ping/startup_checks.py`, `STARTUP_CHECKS`) applies the same rule at boot: required components retry for `STARTUP_CHECK_TIMEOUT_SECONDS` then refuse the boot; an optional unreachable cache store is one WARNING and the server starts. @@ -237,7 +238,7 @@ env vars are tolerated). Notable vars: `APP_NAME`, `APP_VERSION`, `MODE` (`produ anything else refuses to start, unset is `development`), `LOG_LEVEL`, `MAX_REQUEST_SIZE` (10MB default), security headers (`SECURITY_HEADERS_ENABLED`, `HSTS_*`, `FRAME_OPTIONS`, `CSP_POLICY`, ...), `ORIGINS` (CORS, default `*` with a wildcard warning), `ALLOWED_HOST`, `ROOT_PATH`/`DOCS_PATH`/`REDOCS_PATH`, -`REDIS_URL`, `SENTRY_DSN`, `SNOWFLAKE_*` (epoch/node/worker/bit layout), `JWT_SECRET_KEY`/ +`REDIS_*` client limits and `REDIS_KEY_PREFIX` (the URL lives in `cache_store.url`), `SENTRY_DSN`, `SNOWFLAKE_*` (epoch/node/worker/bit layout), `JWT_SECRET_KEY`/ `JWT_ALGORITHM`, `OTEL_ENABLED`/`OTEL_SERVICE_NAME`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_LEVEL`, `STRICT_PRODUCTION_CHECKS` (production refuses wildcard `ALLOWED_HOST`/`ORIGINS` or enabled docs). Debug follows `MODE` (on in development, off in production); a leftover `IS_DEBUG` is ignored with a warning. Media: `MEDIA_REQUIRE_OWNER` defaults on in diff --git a/config-editor-ui/src/icons.js b/config-editor-ui/src/icons.js index c3efb38e..27647a46 100644 --- a/config-editor-ui/src/icons.js +++ b/config-editor-ui/src/icons.js @@ -40,7 +40,8 @@ export const ICONS = { export const SECTION_ICONS = { "get-running": "flag", environment: "terminal", - core: "flag", auth: "lock", authorization: "shield", token_revocation: "key", graph_introspection: "code", rate_limit: "gauge", + core: "flag", auth: "lock", authorization: "shield", cache_store: "gauge", graph_introspection: "code", observability: "activity", runs: "play", websocket: "bolt", ag_ui: "external", remote_tools: "wrench", - media: "file", plugins: "layers", test: "flask", evaluation: "target", + media: "file", checkpointer: "layers", id_generator: "key", thread_name_generator: "file", store: "box", + injectq: "arrows", test: "flask", evaluation: "target", }; diff --git a/config-editor-ui/src/pages.jsx b/config-editor-ui/src/pages.jsx index b8e4be74..92f48a61 100644 --- a/config-editor-ui/src/pages.jsx +++ b/config-editor-ui/src/pages.jsx @@ -2,11 +2,13 @@ import { copyCommand, go, setSection } from "./actions.js"; import { Field, fieldGroups } from "./fields.jsx"; import { SECTION_ICONS } from "./icons.js"; import { - PAGES, WIDGET_FIELD_IDS, getPath, isAgentReady, isOn, issuesFor, navItem, section, store, + PAGES, WIDGET_FIELD_IDS, isAgentReady, isObject, isOn, issuesFor, navItem, section, store, valueOrDefault, } from "./store.js"; import { Callout, Icon, IssueCallout, Segmented } from "./ui.jsx"; -import { AuthWidget, AuthorizationWidget, RemoteToolsWidget } from "./widgets.jsx"; +import { + AuthWidget, AuthorizationWidget, CacheStoreWidget, ChoiceWidget, RemoteToolsWidget, +} from "./widgets.jsx"; const Heading = ({ children, code }) => (

@@ -52,17 +54,17 @@ function authSummary() { return { text: String(a) }; } -function rateSummary() { - const r = store.config.rate_limit; - if (r === undefined || r === null) return { text: offText("rate_limit") }; - const v = (k) => valueOrDefault(["rate_limit", k]); - if (v("enabled") === false) return { text: "Settings kept, not enforced." }; - const by = { ip: "per IP", user: "per user", global: "across all clients" }[v("by")] || v("by"); - const custom = ["requests", "window", "by"].some((k) => getPath(store.config, ["rate_limit", k]) !== undefined - && getPath(store.config, ["rate_limit", k]) !== store.fields.get(`rate_limit.${k}`)?.default); - return { - text: `${v("requests")} requests per ${v("window")} s, ${by}${custom ? "" : " (default)"}.`, - }; +function cacheSummary() { + const c = isObject(store.config.cache_store) ? store.config.cache_store : {}; + const parts = [`${c.type || "memory"} store`]; + const rate = c.rate_limit; + if (isObject(rate) && rate.enabled !== false) { + const by = { ip: "per IP", user: "per user", global: "across all clients" }[rate.by ?? "ip"] || rate.by; + parts.push(`${rate.requests ?? 100} requests per ${rate.window ?? 60} s, ${by}`); + } + const revoke = c.token_revocation; + if (revoke === true || (isObject(revoke) && revoke.enabled !== false)) parts.push("token revocation on"); + return { text: `${parts.join("; ")}.` }; } function traceSummary() { @@ -79,7 +81,7 @@ function GetRunning() { const core = section("core"); const ready = isAgentReady(); const general = core ? issuesFor("core").filter((i) => !core.fields.some((f) => f.path.join(".") === i.path)) : []; - const rows = [["auth", authSummary()], ["rate_limit", rateSummary()], ["observability", traceSummary()]]; + const rows = [["auth", authSummary()], ["cache_store", cacheSummary()], ["observability", traceSummary()]]; const all = store.nav.flatMap((g) => g.items).filter((id) => id !== "get-running"); return ( <> @@ -151,13 +153,16 @@ function SectionWidget({ sec }) { if (sec.widget === "auth") return ; if (sec.widget === "authorization") return ; if (sec.widget === "remote_tools") return ; + if (sec.widget === "cache_store") return ; + if (sec.widget === "id_generator" || sec.widget === "thread_name_generator") return ; return null; } // Issues that no rendered field or widget shows inline. export function generalIssues(sec) { const inline = new Set([...sec.fields.map((f) => f.path.join(".")), ...Object.keys(WIDGET_FIELD_IDS)]); - return issuesFor(sec.id).filter((i) => !inline.has(i.path)); + const tabbed = new Set(["cache_store.rate_limit", "cache_store.token_revocation"]); + return issuesFor(sec.id).filter((i) => !inline.has(i.path) && !(sec.id === "cache_store" && tabbed.has(i.path))); } // Fields and widget of a section, no heading. Shared by the page and search results. @@ -168,7 +173,7 @@ export function SectionFields({ sec }) { <> {widget ?
: null}
- {fieldGroups(sec.fields).map((g, i) => ( + {fieldGroups(sec.fields.filter((f) => !f.managed)).map((g, i) => (
{g.name ? (

{g.name}

@@ -239,9 +244,11 @@ function search(terms) { const results = []; for (const sec of store.sections) { const secHit = hit(`${sec.title} ${sec.description} ${sec.key || ""}`); - const fields = sec.fields.filter((f) => secHit || hit(`${f.label} ${f.help || ""} ${f.path.join(".")}`)); + const matches = (f) => secHit || hit(`${f.label} ${f.help || ""} ${f.path.join(".")}`); + // Fields the widget draws are found through the widget. + const fields = sec.fields.filter((f) => !f.managed && matches(f)); const env = (sec.env || []).filter((v) => hit(`${v.name} ${v.purpose}`)); - const widget = !!sec.widget && secHit; + const widget = !!sec.widget && (secHit || sec.fields.some((f) => f.managed && matches(f))); if (fields.length || env.length || widget) results.push({ id: sec.id, sec, fields, env, widget }); } const envGroups = (store.schema.environment || []) diff --git a/config-editor-ui/src/sidebar.jsx b/config-editor-ui/src/sidebar.jsx index 384aeca5..1df92c27 100644 --- a/config-editor-ui/src/sidebar.jsx +++ b/config-editor-ui/src/sidebar.jsx @@ -29,11 +29,14 @@ function NavLink({ id }) { href={`#${id}`} aria-current={current ? "page" : undefined} onClick={(e) => { e.preventDefault(); go(id); }} ref={(el) => { if (el && current && window.innerWidth < 1024) el.scrollIntoView({ block: "nearest", inline: "nearest" }); }} - class={`focus-ring flex min-h-10 items-center gap-2.5 rounded-lg px-3 font-medium transition-colors duration-150 max-lg:min-h-11 max-lg:shrink-0 max-lg:whitespace-nowrap ${current + class={`focus-ring flex min-h-10 items-center gap-2.5 rounded-lg px-3 py-1.5 font-medium transition-colors duration-150 max-lg:min-h-11 max-lg:shrink-0 max-lg:whitespace-nowrap ${current ? "bg-brand-soft text-ink" : "text-muted hover:bg-raised hover:text-ink"}`} > - {item.title} + + {item.title} + {item.subtitle ? {item.subtitle} : null} + ); diff --git a/config-editor-ui/src/store.js b/config-editor-ui/src/store.js index 27b2ff30..b85c61c2 100644 --- a/config-editor-ui/src/store.js +++ b/config-editor-ui/src/store.js @@ -7,9 +7,9 @@ // are pages of their own rather than schema sections. export const NAV = [ { label: null, items: ["get-running"] }, - { label: "Before production", items: ["auth", "authorization", "rate_limit", "observability", "token_revocation", "graph_introspection"] }, + { label: "Before production", items: ["auth", "authorization", "cache_store", "observability", "graph_introspection"] }, { label: "Runs and streaming", items: ["runs", "websocket", "ag_ui", "remote_tools", "media"] }, - { label: "Project wiring", items: ["plugins"] }, + { label: "Project wiring", items: ["checkpointer", "id_generator", "thread_name_generator", "store", "injectq"] }, { label: "Developer tools", items: ["test", "evaluation"] }, { label: "Reference", items: ["environment"] }, ]; @@ -108,6 +108,9 @@ export function setPath(obj, path, value) { target[path[path.length - 1]] = value; } +// Features that stay on when their last setting is reset: an empty object means "on, all defaults". +const KEEP_EMPTY = new Set(["cache_store.rate_limit", "cache_store.token_revocation"]); + export function deletePath(obj, path) { const parents = []; let target = obj; @@ -120,6 +123,7 @@ export function deletePath(obj, path) { // Drop nested objects left empty, but never the section key itself. for (let i = parents.length - 1; i >= 1; i--) { const [parent, k] = parents[i]; + if (KEEP_EMPTY.has(path.slice(0, i + 1).join("."))) break; if (isObject(parent[k]) && Object.keys(parent[k]).length === 0) delete parent[k]; else break; } @@ -167,6 +171,8 @@ export function unmetHint(f) { } export const navItem = (id) => { const sec = section(id); - return PAGES[id] ? { id, ...PAGES[id] } : sec ? { id, title: sec.title, description: sec.description, icon: null } : null; + return PAGES[id] + ? { id, ...PAGES[id] } + : sec ? { id, title: sec.title, subtitle: sec.subtitle, description: sec.description, icon: null } : null; }; export const isAgentReady = () => /^[\w.]+:[\w.]+$/.test(String(store.config.agent || "")); diff --git a/config-editor-ui/src/widgets.jsx b/config-editor-ui/src/widgets.jsx index 02dff124..9a2ae71d 100644 --- a/config-editor-ui/src/widgets.jsx +++ b/config-editor-ui/src/widgets.jsx @@ -1,6 +1,9 @@ import { useEffect, useState } from "preact/hooks"; -import { commit, isObject, issueAt, store } from "./store.js"; -import { ChoiceGroup, Chips, FieldIssue, FieldMeta, Group, Icon, Row, Select, TextRow } from "./ui.jsx"; +import { Field, fieldGroups } from "./fields.jsx"; +import { commit, isObject, issueAt, setPath, store } from "./store.js"; +import { + Callout, ChoiceGroup, Chips, FieldIssue, FieldMeta, Group, Icon, Pill, Row, Segmented, Select, TextRow, Toggle, +} from "./ui.jsx"; // ------------------------------------------------------------------ auth @@ -34,6 +37,310 @@ export function AuthWidget() { ); } +// ------------------------------------------------------------------ cache store + +const CACHE_CARDS = [ + ["memory", "Memory", "This process only"], + ["redis", "Redis", "Shared. Redis, Valkey, Dragonfly, KeyDB"], + ["custom", "Custom", "Your BaseCacheStore class"], +]; +const DEFAULT_RATE_LIMIT = { requests: 100, window: 60, by: "ip" }; + +const cacheStore = () => (isObject(store.config.cache_store) ? store.config.cache_store : {}); + +// Edit cache_store in place; the key goes away when nothing is left in it. +function editCache(mutate) { + commit((s) => { + const next = isObject(s.config.cache_store) ? s.config.cache_store : {}; + mutate(next); + if (Object.keys(next).length) s.config.cache_store = next; + else delete s.config.cache_store; + }); +} + +const rateLimitOn = () => { + const r = cacheStore().rate_limit; + return isObject(r) && r.enabled !== false; +}; +const revocationOn = () => { + const r = cacheStore().token_revocation; + return r === true || (isObject(r) && r.enabled !== false); +}; + +function setCacheType(kind) { + editCache((c) => { + delete c.url; + delete c.path; + if (kind === "memory") delete c.type; + else c.type = kind; + }); +} + +function setRateLimit(on) { + editCache((c) => { + if (!on) delete c.rate_limit; + else if (isObject(c.rate_limit)) delete c.rate_limit.enabled; + else c.rate_limit = { ...DEFAULT_RATE_LIMIT }; + }); +} + +function setRevocation(on) { + editCache((c) => { + if (!on) delete c.token_revocation; + else if (isObject(c.token_revocation)) delete c.token_revocation.enabled; + else c.token_revocation = true; + }); +} + +const putCache = (path, value) => commit((st) => { + setPath(st.config, path, value); +}); + +function TypeCards({ value, onPick }) { + const move = (e, i) => { + const d = e.key === "ArrowRight" || e.key === "ArrowDown" ? 1 : e.key === "ArrowLeft" || e.key === "ArrowUp" ? -1 : 0; + if (!d) return; + e.preventDefault(); + const next = CACHE_CARDS[(i + d + CACHE_CARDS.length) % CACHE_CARDS.length][0]; + onPick(next); + requestAnimationFrame(() => document.getElementById(`cache-type-${next}`)?.focus()); + }; + return ( +
+ {CACHE_CARDS.map(([v, title, detail], i) => { + const selected = v === value; + return ( + + ); + })} +
+ ); +} + +function InlineNote({ warn, children }) { + return ( +

+ {warn ? : null} + {children} +

+ ); +} + +function FeatureHead({ id, title, text, on, onChange }) { + return ( +
+
+

{title}

+

{text}

+
+ +
+ ); +} + +const LIMIT_BY = [["ip", "IP"], ["user", "User"], ["global", "Global"]]; + +function RateLimitPanel({ field, memory }) { + const [adv, setAdv] = useState(false); + const r = cacheStore().rate_limit; + const by = (isObject(r) && r.by) || "ip"; + const failOpen = !(isObject(r) && r.fail_open === false); + const advanced = ["trusted_proxy_headers", "trusted_proxy_hops", "trusted_proxies", "exclude_paths"]; + return ( + <> + {memory ? Counts per worker with the Memory store. : null} +
+ + +
+ Limit by +
+ putCache(["cache_store", "rate_limit", "by"], v)} + /> +
+ +
+
+ Fail open +
+ putCache(["cache_store", "rate_limit", "fail_open"], v)} + /> +

+ {failOpen ? "Allow requests if the store errors." : "Deny with 503 if the store errors."} +

+
+
+
+
+ + {adv ? ( +
+ {advanced.map((k) => )} +
+ ) : null} +
+ + ); +} + +const TABS = [["rate", "Rate limiting"], ["revoke", "Token revocation"]]; + +export function CacheStoreWidget({ sec }) { + const [tab, setTab] = useState("rate"); + const kind = cacheStore().type || "memory"; + const field = (path) => sec.fields.find((f) => f.path.join(".") === path); + const on = { rate: rateLimitOn(), revoke: revocationOn() }; + const memory = kind === "memory"; + const warned = { + rate: !!issueAt("cache_store.rate_limit"), + revoke: !!issueAt("cache_store.token_revocation"), + }; + const onKey = (e, i) => { + const d = e.key === "ArrowRight" ? 1 : e.key === "ArrowLeft" ? -1 : 0; + const to = e.key === "Home" ? 0 : e.key === "End" ? TABS.length - 1 : d ? (i + d + TABS.length) % TABS.length : -1; + if (to < 0) return; + e.preventDefault(); + setTab(TABS[to][0]); + document.getElementById(`cache-tab-${TABS[to][0]}`)?.focus(); + }; + return ( +
+ + + {kind === "redis" ? : null} + {kind === "custom" ? : null} + {memory ? ( +

+ Not shared between workers: each worker keeps its own locks, limits and revocations. +

+ ) : null} + + +
+ {TABS.map(([id, label], i) => { + const sel = tab === id; + return ( + + ); + })} +
+ +
+ {tab === "rate" ? ( + <> + + {on.rate ? + : warned.rate ? Off: one caller can exhaust the service. : null} + + ) : ( + <> + + {on.revoke ? ( + <> + {memory ? 10xgraph revoke cannot reach an in-memory store; pick Redis or Custom. : null} +
+ + +
+ + Fails closed: while the cache store is unreachable, authenticated requests get 503 and the server will not start. + + + ) : warned.revoke ? Off: a leaked token works until it expires. : null} + + )} +
+
+ ); +} + +// ------------------------------------------------------------------ choice (id generator, thread naming) + +// A dropdown of built-in names plus "custom" (a module:attr field). An unset key is the +// first option for id_generator ("Not set") and the default name for thread_name_generator. +export function ChoiceWidget({ sec }) { + const key = sec.fields[0].path[0]; + const names = sec.choices.map((c) => c[0]); + const value = store.config[key]; + const unsetValue = key === "thread_name_generator" ? "random" : ""; + const mode = value === undefined ? unsetValue : typeof value === "string" && names.includes(value) ? value : "custom"; + const options = [ + ...(key === "id_generator" ? [["", "Not set (the graph's own generator)"]] : []), + ...sec.choices.map(([v, label]) => [v, key === "thread_name_generator" && v === "random" ? `${label} (default)` : label]), + ["custom", "Custom (module:attr)"], + ]; + const detail = mode === "" ? "The graph keeps the generator it was compiled with." + : mode === "custom" ? "Your own class, as module:attribute." + : sec.choices.find((c) => c[0] === mode)?.[2]; + const pick = (v) => commit((s) => { + if (v === unsetValue) delete s.config[key]; + else s.config[key] = v === "custom" ? "" : v; + }); + const id = `f-${key}`; + return ( +
+ +
+ +