diff --git a/.oxlintrc.json b/.oxlintrc.json index acc0f053..6086861c 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -163,11 +163,13 @@ "LeaseRequestFailure", "LeaseRequestLimits", "SerializedDecision", + "UsageReader", "findCatalogModel", "fits", "modelClass", "newLeaseRequestId", - "pairedRuntimes" + "pairedRuntimes", + "tokenLabelMap" ], "message": "the gateway may import only the names its allow-list gives" }, diff --git a/docs/CLI.md b/docs/CLI.md index bcf6d609..0edee57d 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -3,7 +3,7 @@ Part of the user manual: every command the simlock CLI is expected to implement. Results are JSON on **stdout**; progress/diagnostics are JSON lines on **stderr** — this is the default output, not an opt-in, because -agents are the primary audience. `status`, `catalog`, and +agents are the primary audience. `status`, `catalog`, `stats`, and `daemon ` are the exception: they default to a human-oriented view for interactive/operator use and accept `--json` to switch to the structured form. Every other command's output is already @@ -80,6 +80,7 @@ command starts it again) to bring the platform up. | 12 | `INSUFFICIENT_DISK_SPACE` | not enough free disk space to install a component | | 12 | `LICENSE_NOT_ACCEPTED` | a required license (e.g. an Android SDK license) is not accepted | | 12 | `UNKNOWN_WORKER` | `worker drain`/`undrain` or `component install --worker` naming a worker the gateway does not know | +| 12 | `HISTORY_NOT_KEPT` | `stats` for a window that ends before the oldest event the history holds | | 12 | `DOWNLOADS_DISABLED` | `component install` on a machine whose `downloads.policy` is `"never"` | | 12 | `COMPONENT_NOT_OWNED` | `component remove` of a component Simlock did not install, or one that changed on disk since | | 12 | `COMPONENT_IN_USE` | `component remove` of a component a device uses, Simlock's or your own | @@ -1934,6 +1935,138 @@ gateway's order. A replay (`simlock events`, `--since`) prints events by millisecond. `--follow` prints each live push as it arrives, so a relayed event from a worker whose clock is behind prints after a later gateway event. +## `simlock stats [--since | --from [--to ]] [--json]` + +The usage figures for a window: how many requests there were, how long they +waited and held their devices, how full the host was, and what went wrong. They +are worked out when you ask, from the same event history `simlock events` +prints, so they agree with it for the same window and they survive a daemon +restart. Nothing is counted separately and nothing is kept beyond the history, +so a window reaches back only as far as `eventLog.retention` and +`eventLog.maxBytes` keep events (see [CONFIGURATION.md](CONFIGURATION.md)). + +- `--since 6h` is the last six hours, up to now. Durations take `ms`, `s`, `m`, + `h` and `d` units, as for `simlock events`. +- `--from 2026-10-04T00:00:00Z --to 2026-10-05T00:00:00Z` names the window. With + `--from` alone the window runs to now. +- With none of them, the window is the last 24 hours. +- `--since` with `--from`, `--to` without `--from`, and a `--from` that is not + earlier than `--to` are usage errors (exit 2). A window can be at most 90 + days long. +- `--json` prints the daemon's answer unchanged, as one JSON object; without it + you get a table. + +The table opens with the window, then the totals, then a row for each platform, +each worker and each requester: + +``` +Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z + +Totals + Requests: 12 (10 granted, 2 rejected) + Granted: warm 6, booted 3, provisioned 1 + Rejected: no-wait 1, timeout 1 + Declined: 2 + Wait: p50 1.2s, p95 4s, max 9.1s (11 samples) + Held: p50 5m 0s, p95 30m 0s, max 1h 2m (9 samples) + Turnaround: p50 5m 10s, p95 31m 40s, max 1h 3m (9 samples) + Provisioning: p50 1m 30s, p95 1m 30s, max 1m 30s (1 sample) + Boot: p50 20s, p95 40s, max 40s (2 samples) + Slots: peak 3 of 4, mean 1.5 + Queue: peak depth 2, mean 0.4 + Incidents: 0 quarantined, 1 recovered after a crash, 0 recovered from quarantine, 0 lost + +Platforms + ios 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3 + android 0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak - + +Workers + mac-mini-1 (wrk_1) 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3 + +Requesters + ci-bot (tok_a) 3 requests, 2 granted, 1 rejected, held 12m 5s +``` + +What the figures count: + +- **Requests** are the lease requests made in the window. A request belongs to + the window it was made in: its grant, rejection and end are joined from + events up to the end of the window, so a lease still held at the end of the + window counts as a request and a grant and gives no held or turnaround time. + A request made before the window that is granted or rejected inside it is in + no count. +- **Granted** is the grants of those requests, by how the device came to be + ready: `warm` was already ready, `booted` was started from shutdown, + `provisioned` was created for the request. +- **Rejected** are the requests that ended without a device, by reason + (`timeout`, `no-wait`, `cancelled`, ...), plus the requests refused before + they were stored (a requester that already holds a lease, one that asked + for a lease ID that is taken, or one that came while admission was closed + for `nuke` or maintenance). Those are in the window their rejection falls + in and are not requests, so granted plus rejected can be more than requests. +- **Wait** is the time from the request to its grant or rejection. **Held** is + the time from the grant to the release or expiry of the lease. **Turnaround** + is the time from the request to that end. **Provisioning** and **Boot** are the + durations of creating and starting devices in the window. Each shows its median + (`p50`), `p95` and longest, and how many samples it has; `-` or "no samples" + where there are none. +- **Slots** is how many devices were running or reserved against the most the + host allows: the highest value and the average over time. **RAM** appears when + the host keeps a RAM budget. **Queue** is how many requests were waiting. Where + the history has nothing before the first record in the window, that stretch of + time is left out of the figures rather than counted as zero. +- **Incidents** count devices quarantined, devices recovered after a crash, + devices recovered from quarantine, and devices lost (a recovery that failed, or + a quarantined device given up on). **Failures** counts the failure events in + the window by name: `device.purge-failed`, `device.recovery-failed`, + `component.install-failed`, and any other event whose name ends in + `-failed`. The row is left out when there were none. +- **Declined** counts the times a worker refused a request that a gateway sent + it, by the time of the refusal. It is a count of refusals, not of requests: a + gateway tries another worker after a refusal, so one request can be declined + by several workers. The row is left out when there were none. +- A requester that is a token shows the token's label beside its id. + +The window is rounded down to whole steps of the time series `--json` carries, +which is 1 minute for a window of a few hours and grows to 1 day for 90 days. +Neither end moves later than the time you asked for; the window in the answer is +the rounded one. + +Against a **gateway** the totals are the fleet's and there is a row for each +worker. Requests, waits and rejections come from the gateway's own record of +each request: it waits from the request until the gateway grants or rejects it, +and a request the gateway was still holding when it stopped counts as rejected +with the reason `daemon-restarted`. How the device came to be ready and how long +the lease was held come from the worker's record of the lease; a grant whose +worker's record the gateway never received counts as `unknown` and has no held +time. A worker's row has the requests granted on it or that failed on it +(`worker-failed`) and its device figures; a request that ended any other way, +or is still waiting, is in the totals and the platform rows only. The queue figure is the fleet queue's. + +A worker's own `simlock stats` covers that worker only: its row is itself. A +request that a gateway sent it is not one of the worker's requests: it is counted +under **Probes**, not under requests, and has no wait or turnaround of its own. +Its grant still counts as a grant, with its source and held time, under the +requester id the gateway gave it. + +If the history does not reach back to the start of the window (a new daemon, or +older events deleted by retention), the figures cover what it does hold and say +so: + +``` +Usage from 2026-09-05T12:00:00.000Z to 2026-10-05T12:00:00.000Z +Figures cover from 2026-10-05T11:59:00.000Z: the history does not reach back to the start of the window. +``` + +In `--json` that is `partial: true` and `coversFrom`. If the window ends before +the oldest event the history holds, there is nothing to show: the command fails +with `HISTORY_NOT_KEPT` (exit 12), and the message names the time the history +reaches back to. A daemon with no history at all has nothing to refuse and +answers with zeros. + +`stats` needs the daemon, and the admin credential (see +[Admin credential resolution](#admin-credential-resolution)). + ## `simlock daemon ` Manage the daemon explicitly. Other commands auto-start it on demand; `daemon` diff --git a/docs/CLIENT.md b/docs/CLIENT.md index fb21ee0e..aff33093 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -16,6 +16,7 @@ import { connectSimlockAdmin } from "simlock/admin"; // agent + admin role `connectSimlockAdmin` returns a superset of `connectSimlock`'s client — every agent-role method plus the admin-role ones (`list`, `runCleanup`, `runNuke`, `getConfig`, `stopDaemon`, `replayEvents`/`subscribeEvents` (each event carries an `id`), +`usage`, `createToken`/`listTokens`/`revokeToken`, `installComponent`, `removeComponent`). The split exists so `simlock/client` doesn't even show admin methods in a caller's editor; the daemon's own role check is what actually stops an agent-role session from @@ -571,6 +572,38 @@ const { waiting = [] } = await client.getStatus(); `list({ kind: "requests" })` lists both, each worker's entries with their `workerId`, the same list as `GET /v1/lease-requests`. +## How much was used: `usage` + +`simlock/admin` only. `usage({ from, to })` returns the usage figures for a +window, the same answer `simlock stats --json` prints (see +[CLI.md](CLI.md), under `simlock stats`, +for what each figure counts). `from` and `to` are epoch milliseconds, `from` +before `to`, at most 90 days apart; the client refuses anything else before it +sends a frame. + +```ts +const usage = await admin.usage({ from: Date.now() - 6 * 3_600_000, to: Date.now() }); + +usage.totals.requests; // lease requests made in the window +usage.totals.wait.p95; // milliseconds, or null when nothing waited +usage.totals.failures.byEvent; // a count per failure event, e.g. "device.purge-failed" +usage.workers[0]?.label; // one entry for each worker; a worker lists itself +usage.series; // one point per bucket, for a chart; `waiting` counts every + // request open at the bucket's end, leaving out a gateway's probes +``` + +The daemon computes the figures from its event history, so they cover only what +the history holds: `partial` is `true` and `coversFrom` says where they start when +it does not reach the start of the window. `window` in the answer is the window +asked for, rounded down to a whole number of `bucketMs` at both ends, and the series never has more +than 200 points. A window that ends before the oldest event the history holds +rejects with `HISTORY_NOT_KEPT`; its `details.oldestTs` is the oldest time the +history reaches. Against a gateway the totals are the fleet's and `workers` has +one entry for each worker; `bySource.unknown` counts grants whose source the +gateway never learned, and there is no `probes`. Against a worker, `probes` +counts the requests a gateway sent it, which `requests` leaves out. `declined` +counts the refusals of such requests. + ## One connection, no reconnect, no retry This is the one thing to internalize before building anything on top of this diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index ff689055..a34548b6 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -2113,6 +2113,29 @@ The CLI reads the file itself only for `simlock events --since` when no daemon answers; `--follow` subscribes first, replays, and drops replayed pushes, so the join neither loses nor repeats an event. +`usage.get` (ADR 0016) is the third reader of that history, and the only one that +turns it into numbers. `computeUsage` (`src/core/usage/`) is pure: it takes +the envelopes of a window, with the latest `capacity.changed`, `queue.changed` and +`daemon.started` at or before the window's start and the latest `lease.requested` +of each requester, that `EventHistory.read`'s `carry` adds, and the ids of the +requests made before it (`requestedBefore`, so a rejection of one is in no count) +and of those answered by then (`answeredBefore`, so a request still waiting when +the window opens counts in the series' `waiting`), and returns the figures. +`readEvents` reads them into one fact per request, joined by `requestId` and +`leaseId`, one per device event and one per `lease.declined`; `compute-usage.ts` +adds them up by platform, by worker and by requester. On a gateway (`fleet`) +request facts come from its own events and device facts from the events its +workers relayed (ADR 0021 §5): a request's outcome is the gateway's own +`request.granted` or `lease.rejected` for its request id, else its next own +`daemon.started` ends it as rejected `daemon-restarted`, else it is open; the +grant's source and held time are the relayed `lease.granted` of the worker and +lease it names. On a worker a request carrying `fleetRequestId` is a probe, +counted apart from its requests. That rule is in `readEvents` and nowhere else. One +`UsageReader` serves both dispatchers: it rounds the window down to the series +bucket at both ends, keeps its last answer by that window and the newest event id, and joins +token labels, so the daemon's two handlers differ only in `fleet`. `simlock stats` +prints what the operation returns and computes nothing. + ## Device requests A request names a device in one of three forms (ADR 0015 §1): an exact diff --git a/docs/internal/COMPONENTS.md b/docs/internal/COMPONENTS.md index b759861f..4c9f6554 100644 --- a/docs/internal/COMPONENTS.md +++ b/docs/internal/COMPONENTS.md @@ -62,6 +62,7 @@ pool reads from acquisition. | `Registry` | `src/core/registry.ts` | Devices, leases, lease requests and component records, written through one commit to `state.json`; the device transition function's only caller; emits the post-commit device and lease facts. | Decide a transition: callers do, inside a decision section. | | `SerializedDecision` | `src/core/serialized-decision.ts` | Serialising short read-decide-commit sections. | Hold driver work or other long I/O. | | `DeviceOperationClaims` | `src/core/device-operation-claims.ts` | Exclusive per-device operation claims: boot, eviction, cleanup, nuke, reclaim. | Any lifecycle, cleanup or leasing policy. | +| `UsageReader`, `computeUsage` | `src/core/usage/` (surface: `index.ts`) | `usage.get`'s one answerer for a worker and a gateway: reads the event history for a window rounded down to the series bucket, turns it into figures (`computeUsage`, `readEvents`, the timelines), joins token labels, keeps the last answer by window and newest event. | Write anything, or read the registry, capacity or lifecycle engine: events in, numbers out. | | `DeviceProvisioner` | `src/core/device-provisioner.ts` | Creating a device: the component-removal gate, the registry record, the driver's `provision`, readiness for the lease handoff. | Decide whether to provision. | | `ManagedDeviceLifecycle` | `src/core/managed-device-lifecycle.ts` | Registry-owned device operations with a claim and a revalidation each: boot for a lease, boot back to warm, shutdown, destroy, dispose, recover a leased device. Every driver verb on an existing device goes through here. | Decide when to run them. | | `ReclaimCoordinator` | `src/core/reclaim-coordinator.ts` | After a release: the driver's reclaim, committing the state it returns (`shutdown` on iOS, `ready` on Android) and waking the queue, handing a failed purge to quarantine, deleting a spent `fresh` device, recovering an interrupted reclaim at startup (a shutdown; for a device whose wipe a start put off, the full reclaim started in the background under a claim, and `settle` for a graceful stop). | Decide whether a device stays warm, boot anything, or read capacity or the queue. The warm pool (`src/core/warm-pool/`, ADR 0017) does. | diff --git a/docs/internal/EVENTS.md b/docs/internal/EVENTS.md index 0f932594..9788ccbb 100644 --- a/docs/internal/EVENTS.md +++ b/docs/internal/EVENTS.md @@ -52,7 +52,7 @@ in short: `subject.past-tense-fact`, emitted post-commit, facts not commands. | `lease.renewed` | lease id, new deadline | a `lease.renew` succeeded — whether it came from `simlock lease renew`, `POST /v1/leases/{id}/renew`, or the renew timer a running `simlock lease` / MCP session keeps over its own lease. There is one renew path and this is it | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.released` | lease id, device id, reason (explicit/killed/device-lost), owner id | an explicit `lease.release` (which is what a `simlock lease` holder does on its way out), (killed) an operator `release --all` or `nuke`, or (device-lost) a leased device could not be recovered after it stopped running outside simlock, or a daemon start found its device not running, and the device of that lease was wiped and returned to the pool, left waiting in `reclaiming` on a platform the daemon could not list, or marked missing. Closing a connection is not a release and never emits this | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.expired` | lease id, device id, owner id | the lease's deadline passed with no `lease.renew` behind it — the grant-time TTL, or the TTL of the last renew, simply ran out. This is the one way a lease ends without somebody asking, and the only bound on a holder that was killed outright | LeaseLifecycle | implemented (payload per ADR 0004 pending) | -| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted/worker-failed), `code` and `worker` on `worker-failed` (ADR 0021 §4, additive; `worker`, never `workerId`, which marks a relayed event, ADR 0014 §6) | a request ended without a grant. A worker emits it only for its local requests; a probe ends in `lease.declined` there (ADR 0021 §2). A gateway emits it for every ending that is not a grant (ADR 0021 §4): its own reasons, `unresolvable-spec` for the last cannot-serve refusal whether or not the request had queued, and `worker-failed` for any other failure on the worker it went to -- a terminal refusal, a failure after progress, `WORKER_UNREACHABLE`, `INTERNAL`, a dispatch timeout, a worker's own `REQUESTER_ALREADY_LEASED` or `LEASE_ID_TAKEN` -- with `code` the error code the caller got and `worker` the worker. `dispose()` emits nothing, so a request still open when a gateway stops has no ending event (ADR 0021 §5 plans for usage to close those at the next `daemon.started`; not built yet). A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never stored and has no `lease.requested` (except a gateway's `lease-id-taken` after a grant, below); it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec`, whether or not it had entered the gateway queue (the worker only declined it, ADR 0021 §4); `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | +| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted/worker-failed), `code` and `worker` on `worker-failed` (ADR 0021 §4, additive; `worker`, never `workerId`, which marks a relayed event, ADR 0014 §6) | a request ended without a grant. A worker emits it only for its local requests; a probe ends in `lease.declined` there (ADR 0021 §2). A gateway emits it for every ending that is not a grant (ADR 0021 §4): its own reasons, `unresolvable-spec` for the last cannot-serve refusal whether or not the request had queued, and `worker-failed` for any other failure on the worker it went to -- a terminal refusal, a failure after progress, `WORKER_UNREACHABLE`, `INTERNAL`, a dispatch timeout, a worker's own `REQUESTER_ALREADY_LEASED` or `LEASE_ID_TAKEN` -- with `code` the error code the caller got and `worker` the worker. `dispose()` emits nothing, so a request still open when a gateway stops has no ending event (ADR 0021 §5: usage closes those at the next `daemon.started`, as rejected with `daemon-restarted`). A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never stored and has no `lease.requested` (except a gateway's `lease-id-taken` after a grant, below); it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec`, whether or not it had entered the gateway queue (the worker only declined it, ADR 0021 §4); `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | | `lease.declined` | `{ requestId, fleetRequestId, requester, requestSpec, reason }` -- `reason` takes `lease.rejected`'s values; `requestSpec` is there for the same reason as on `lease.rejected`: a decline at admission has no `lease.requested` (ADR 0021 §2) | a worker refused or failed a probe, a `lease.request` carrying `fleetRequestId` (ADR 0021 §1), whatever the reason (`no-wait`, `unresolvable-spec`, `already-leased`, `lease-id-taken`, `boot-timeout`, `killed`, `daemon-restarted` at the next start) and however far the work got. The gateway owns the outcome of a probe, so the worker never rejects one and never queues one: where it would queue (a second failed provision) it declines with `no-wait` and answers `NO_CAPACITY`. The event depends only on the request being a probe, never on whether the gateway will retry. A local request is rejected exactly as before | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) | implemented | On a **gateway**, `lease.requested`, `lease.queued` and `lease.rejected` are its own fleet queue's facts (ADR 0005 §11/§14), diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts new file mode 100644 index 00000000..a593afc2 --- /dev/null +++ b/e2e/usage-stats.test.ts @@ -0,0 +1,299 @@ +import { describe, expect, it } from "vitest"; + +import { freeLoopbackPort, waitFor, withDaemon } from "./helpers/index.js"; +import type { RecordedEvent } from "./helpers/events.js"; + +/** + * ADR 0016 with real processes: `simlock stats` reads the figures from the event history, on a + * worker and on a gateway, and the figures are what a reader counts by hand from `simlock + * events` for the same window. + */ + +const LEASE_ARGS = ["lease", "--platform", "ios", "--device", "iPhone 16", "--os", "18.4"] as const; + +interface Samples { + readonly count: number; + readonly max: number | null; + readonly p50: number | null; +} + +interface Figures { + readonly requests: number; + readonly granted: number; + readonly bySource: Record; + readonly declined: number; + readonly probes?: number; + readonly rejected: { readonly total: number; readonly byReason: Record }; + readonly wait: Samples; + readonly held: Samples; + readonly turnaround: Samples; + readonly provisioning: Samples; + readonly boot: Samples; + readonly incidents: Record; +} + +interface Usage { + readonly window: { readonly from: number; readonly to: number }; + readonly partial: boolean; + readonly coversFrom: number; + readonly totals: Figures; + readonly workers: readonly (Figures & { readonly id: string; readonly label?: string })[]; + readonly requesters: readonly { readonly id: string; readonly requests: number }[]; +} + +/** The last hour, asked with `--since 1h`, as JSON. */ +async function stats(env: { + cli: (args: string[]) => Promise<{ code: number | null; json?: unknown }>; +}): Promise { + const result = await env.cli(["stats", "--since", "1h", "--json"]); + expect(result.code).toBe(0); + return result.json as Usage; +} + +/** + * The figures for the last hour once the minute that holds the newest event has closed: the + * window is rounded down to the series bucket (a minute here), so what happened in the minute now + * running is counted when that minute is over. Waits for it, up to a minute. + */ +async function settledStats( + env: { + cli: (args: string[]) => Promise<{ code: number | null; json?: unknown }>; + events: () => Promise; + }, + until: (usage: Usage) => boolean = () => true, +): Promise { + let usage: Usage | undefined; + await waitFor( + async () => { + const newest = Math.max(...(await env.events()).map((event) => event.timestamp)); + usage = await stats(env); + return usage.window.to >= newest && until(usage); + }, + { interval: 1_000, label: "the window closed over the newest event", timeout: 90_000 }, + ); + return usage as Usage; +} + +function countBy(values: readonly string[]): Record { + const counts: Record = {}; + for (const value of values) counts[value] = (counts[value] ?? 0) + 1; + return counts; +} + +function inWindow(events: readonly RecordedEvent[], window: Usage["window"]): RecordedEvent[] { + return events.filter((event) => event.timestamp > window.from && event.timestamp <= window.to); +} + +function payload(event: RecordedEvent): Record { + return event.payload as Record; +} + +describe("simlock stats", () => { + it("after a scripted run of leases against the fake driver, simlock stats --since 1h --json reports the counts a test derives independently from the same hour of simlock events --since 1h", async () => { + const env = await withDaemon({ + configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }, + }); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + const first = await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"]); + expect(first.code).toBe(0); + const firstLeaseId = (first.json as { lease: { id: string } }).lease.id; + // Refused for holding a lease already, refused for want of capacity, and timed out waiting. + expect((await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"])).code).toBe(13); + expect( + (await env.cli([...LEASE_ARGS, "--agent-id", "agent-b", "--no-wait", "--detach"])).code, + ).toBe(11); + expect( + ( + await env.cli([...LEASE_ARGS, "--agent-id", "agent-c", "--timeout", "300ms", "--detach"], { + timeout: 15_000, + }) + ).code, + ).toBe(10); + expect((await env.cli(["release", firstLeaseId])).code).toBe(0); + // A second lease on the device the first one left warm. + const second = await env.cli([...LEASE_ARGS, "--agent-id", "agent-d", "--detach"]); + expect(second.code).toBe(0); + expect( + (await env.cli(["release", (second.json as { lease: { id: string } }).lease.id])).code, + ).toBe(0); + + const usage = await settledStats(env); + const everything = await env.events(); + const history = inWindow(everything, usage.window); + const named = (name: string) => history.filter((event) => event.event === name); + + // The daemon is a minute old, so the history does not reach back an hour: the figures say + // where they start. + expect(usage.partial).toBe(true); + expect(usage.coversFrom).toBe(Math.min(...everything.map((event) => event.timestamp))); + expect(usage.totals.requests).toBe(named("lease.requested").length); + expect(usage.totals.granted).toBe(2); + expect(usage.totals.bySource).toMatchObject( + countBy(named("lease.granted").map((event) => payload(event).source as string)), + ); + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 1, warm: 1 }); + expect(usage.totals.rejected.total).toBe(named("lease.rejected").length); + expect(usage.totals.rejected.byReason).toEqual( + countBy(named("lease.rejected").map((event) => payload(event).reason as string)), + ); + expect(Object.keys(usage.totals.rejected.byReason).sort()).toEqual([ + "already-leased", + "no-wait", + "timeout", + ]); + // Held and turnaround are the two leases' own spans, as the events state them. + const heldByHand = named("lease.granted") + .map((granted) => { + const end = named("lease.released").find( + (event) => payload(event).leaseId === payload(granted).leaseId, + ); + return (end?.timestamp ?? NaN) - granted.timestamp; + }) + .sort((a, b) => a - b); + expect(usage.totals.held.count).toBe(2); + expect(usage.totals.held.max).toBe(heldByHand[1]); + expect(usage.totals.turnaround.count).toBe(2); + expect(usage.totals.wait.count).toBeGreaterThanOrEqual(3); + expect(usage.totals.provisioning.count).toBe(named("device.provisioned").length); + expect(usage.totals.boot.count).toBe(named("device.ready").length); + expect(usage.totals.incidents).toEqual({ + crashRecovered: 0, + lost: 0, + quarantineRecovered: 0, + quarantined: 0, + }); + expect(usage.workers).toHaveLength(1); + expect(usage.requesters.map((requester) => requester.id).sort()).toEqual([ + "agent-a", + "agent-b", + "agent-c", + "agent-d", + ]); + + const table = await env.cli(["stats", "--since", "1h"]); + expect(table.code).toBe(0); + expect(table.stdout).toContain("Totals"); + expect(table.stdout).toContain("Requests:"); + }, 240_000); + + it("after a --no-wait refusal, requests and rejected.byReason.no-wait each rise by one and granted is unchanged", async () => { + const env = await withDaemon({ + configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }, + }); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + expect((await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"])).code).toBe(0); + const before = await settledStats(env); + + expect( + (await env.cli([...LEASE_ARGS, "--agent-id", "agent-b", "--no-wait", "--detach"])).code, + ).toBe(11); + const after = await settledStats( + env, + (usage) => usage.totals.requests > before.totals.requests, + ); + + expect(after.totals.requests).toBe(before.totals.requests + 1); + expect(after.totals.rejected.byReason["no-wait"]).toBe( + (before.totals.rejected.byReason["no-wait"] ?? 0) + 1, + ); + expect(after.totals.granted).toBe(before.totals.granted); + }, 240_000); + + it("simlock stats --since 1h returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { + const env = await withDaemon(); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + const lease = await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"]); + expect( + (await env.cli(["release", (lease.json as { lease: { id: string } }).lease.id])).code, + ).toBe(0); + const before = await settledStats(env); + expect(before.totals.granted).toBe(1); + + await env.restartDaemon(); + const after = await stats(env); + expect(after.window.to).toBeGreaterThanOrEqual(before.window.to); + + expect(after.totals).toMatchObject({ + boot: before.totals.boot, + bySource: before.totals.bySource, + granted: before.totals.granted, + held: before.totals.held, + incidents: before.totals.incidents, + provisioning: before.totals.provisioning, + rejected: before.totals.rejected, + requests: before.totals.requests, + turnaround: before.totals.turnaround, + wait: before.totals.wait, + }); + expect(after.requesters).toEqual(before.requesters); + }, 240_000); + + it("on a gateway with two workers, simlock stats has fleet totals and one row per worker, and each worker's own simlock stats covers itself only", async () => { + const port = await freeLoopbackPort(); + const gateway = await withDaemon({ + configOverrides: { http: { host: "127.0.0.1", port }, mode: "gateway" }, + driver: "none", + }); + const minted = await gateway.cli(["token", "create", "--role", "worker"]); + const { secret } = minted.json as { secret: string }; + const oneAtATime = { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }; + const joinWith = (label: string) => + withDaemon({ + configOverrides: { + ...oneAtATime, + gateway: { label, token: secret, url: `ws://127.0.0.1:${port}` }, + }, + driverScript: { ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] } }, + }); + const workerA = await joinWith("worker-a"); + const workerB = await joinWith("worker-b"); + await waitFor( + async () => { + const list = await gateway.cli(["worker", "list", "--json"]); + const workers = (list.json as { workers: { connection: string }[] }).workers; + return workers.length === 2 && workers.every((worker) => worker.connection === "connected"); + }, + { label: "both workers joined", timeout: 30_000 }, + ); + + // One worker is full after its lease, so the second lease lands on the other. + for (const agent of ["agent-1", "agent-2"]) { + const leased = await gateway.cli([...LEASE_ARGS, "--agent-id", agent, "--detach"], { + timeout: 60_000, + }); + expect(leased.code).toBe(0); + } + + let fleet: Usage | undefined; + fleet = await settledStats( + gateway, + (usage) => usage.totals.granted === 2 && usage.workers.length === 2, + ); + expect(fleet?.totals.requests).toBe(2); + expect(fleet?.workers.map((worker) => worker.label).sort()).toEqual(["worker-a", "worker-b"]); + expect(fleet?.workers.map((worker) => worker.granted)).toEqual([1, 1]); + // The gateway knows each grant's source from its worker, counts no probes, and its declined + // figure is its workers'. + expect(fleet?.totals.bySource.unknown).toBe(0); + expect(fleet?.totals).not.toHaveProperty("probes"); + expect(fleet?.totals.declined).toBe( + fleet?.workers.reduce((total, worker) => total + worker.declined, 0), + ); + + for (const worker of [workerA, workerB]) { + const own = await settledStats(worker); + expect(own.workers).toHaveLength(1); + expect(own.totals.granted).toBe(1); + expect(own.workers[0]?.granted).toBe(1); + // What the gateway sent it are probes, not the worker's requests. + expect(own.totals.requests).toBe(0); + expect(own.totals.probes).toBeGreaterThanOrEqual(1); + } + }); +}); diff --git a/src/admin/index.ts b/src/admin/index.ts index 0b74a486..00f4f32b 100644 --- a/src/admin/index.ts +++ b/src/admin/index.ts @@ -66,6 +66,8 @@ export type { TokenListOutput, TokenRevokeInput, TokenRevokeOutput, + UsageGetInput, + UsageGetOutput, WorkerComponentProgress, WorkerDrainInput, WorkerDrainOutput, diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index 8007b23b..ea6ff105 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -198,6 +198,97 @@ describe("EventHistory", () => { { reason: "memory only" }, ]); }); + + it("reads the events and the oldest timestamp the file holds in one call, carrying the step in force", async () => { + const path = await eventFilePath(); + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const sink = new NodeFileLogSink({ path }); + const events = history({ bus, path, sink }); + const early = bus.emit("queue.changed", { depth: 2 }, "wait-queue"); + clock.advance(1_000); + bus.emit("daemon.stopping", { reason: "a" }, "daemon"); + clock.advance(1_000); + const late = bus.emit("daemon.stopping", { reason: "b" }, "daemon"); + sink.close(); + + const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_500 }); + + expect(read.oldestTs).toBe(1_000); + expect(read.events).toEqual([early, late]); + }); + + it("reads from the ring, with the carried step and its oldest timestamp, when there is no sink", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + const early = bus.emit("queue.changed", { depth: 2 }, "wait-queue"); + clock.advance(2_000); + const late = bus.emit("daemon.stopping", { reason: "b" }, "daemon"); + + const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_000 }); + + expect(read).toEqual({ + answeredBefore: new Set(), + events: [early, late], + oldestTs: 1_000, + requestedBefore: new Set(), + }); + expect(await events.replay({ carry: ["queue.changed"], sinceTs: 2_000 })).toEqual([ + early, + late, + ]); + }); + + it("reads the ring with the carried envelopes in time order, though the earlier of two kinds was first set", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + bus.emit("queue.changed", { depth: 1 }, "wait-queue"); + clock.advance(1_000); + const figures = { maxRunning: 2, reserved: 0, running: 1, warm: 0 }; + const capacity = bus.emit( + "capacity.changed", + { android: figures, global: figures, ios: figures }, + "capacity", + ); + clock.advance(1_000); + const queue = bus.emit("queue.changed", { depth: 3 }, "wait-queue"); + + const read = await events.read({ + carry: ["queue.changed", "capacity.changed"], + sinceTs: 5_000, + }); + + expect(read.events.map((envelope) => envelope.timestamp)).toEqual([2_000, 3_000]); + expect(read.events).toEqual([capacity, queue]); + }); + + it("reads the ring with an event at exactly sinceTs as the step in force and not as a newer event", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + clock.advance(1_000); + const step = bus.emit("queue.changed", { depth: 4 }, "wait-queue"); + bus.emit("daemon.stopping", { reason: "at the edge" }, "daemon"); + clock.advance(1); + const after = bus.emit("daemon.stopping", { reason: "after" }, "daemon"); + + const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_000 }); + + expect(read.events).toEqual([step, after]); + }); + + it("names the newest event's id, and none before the first event", () => { + const bus = new EventBus(new FakeClock(1_000)); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + expect(events.latestId()).toBeUndefined(); + + bus.emit("daemon.stopping", { reason: "a" }, "daemon"); + const last = bus.emit("daemon.stopping", { reason: "b" }, "daemon"); + + expect(events.latestId()).toBe(last.id); + }); }); describe("readEventFile", () => { @@ -339,6 +430,117 @@ describe("readEventFile", () => { expect(empty.oldestTs).toBeUndefined(); }); + it("reports the requestId of each lease.requested at or before sinceTs, and none from after it or from other events", async () => { + const requested = (seq: number, timestamp: number, requestId: string): EventEnvelope => + ({ + ...envelope(seq, timestamp), + event: "lease.requested", + payload: { requestId }, + }) as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + requested(1, 100, "old"), + envelope(2, 150), + requested(3, 200, "edge"), + requested(4, 300, "new"), + ), + }); + + const read = await readEventHistory(filesystem, "/data/events.jsonl", { sinceTs: 200 }); + + expect([...read.requestedBefore].sort()).toEqual(["edge", "old"]); + }); + + it("reports no requestId from a lease.requested that has none, or from another event that has one", async () => { + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + { ...envelope(1, 100), event: "lease.requested", payload: {} } as EventEnvelope, + { + ...envelope(2, 110), + event: "lease.granted", + payload: { requestId: "g" }, + } as EventEnvelope, + ), + }); + + const read = await readEventHistory(filesystem, "/data/events.jsonl", { sinceTs: 200 }); + + expect([...read.requestedBefore]).toEqual([]); + }); + + it("reports the requestId of each lease.granted, lease.rejected and request.granted at or before sinceTs as answered, and none from after it or from other events", async () => { + const answer = (seq: number, timestamp: number, event: string, requestId: string) => + ({ ...envelope(seq, timestamp), event, payload: { requestId } }) as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + answer(1, 100, "lease.granted", "granted"), + answer(2, 110, "lease.rejected", "rejected"), + answer(3, 120, "request.granted", "handed"), + answer(4, 130, "lease.declined", "declined"), + answer(5, 140, "lease.requested", "asked"), + answer(6, 200, "lease.granted", "edge"), + answer(7, 300, "lease.granted", "later"), + { ...envelope(8, 150), event: "lease.granted", payload: {} } as EventEnvelope, + ), + }); + + const read = await readEventHistory(filesystem, "/data/events.jsonl", { sinceTs: 200 }); + + expect([...read.answeredBefore].sort()).toEqual(["edge", "granted", "handed", "rejected"]); + expect([...read.requestedBefore]).toEqual(["asked"]); + }); + + it("reports the requests answered before sinceTs from the ring too", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + bus.emit("lease.rejected", { requestId: "old", requester: "a" } as never, "lease"); + clock.advance(2_000); + bus.emit("lease.rejected", { requestId: "new", requester: "a" } as never, "lease"); + + const read = await events.read({ carry: [], sinceTs: 2_000 }); + + expect([...read.answeredBefore]).toEqual(["old"]); + }); + + it("carries events whose worker and requester run together into the same text as two different ones", async () => { + const withPayload = (seq: number, payload: Record): EventEnvelope => + ({ + event: "lease.requested", + id: `evt_${seq}`, + module: "test", + payload, + seq, + timestamp: 100 + seq, + }) as unknown as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + withPayload(1, { requester: "w1x" }), + withPayload(2, { requester: "x", workerId: "w1" }), + ), + }); + + const read = await readEventFile(filesystem, "/data/events.jsonl", { + carry: ["lease.requested"], + sinceTs: 300, + }); + + expect(read.map((entry) => entry.seq)).toEqual([1, 2]); + }); + + it("reports the requests made before sinceTs from the ring too", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + bus.emit("lease.requested", { requestId: "old", requester: "a" } as never, "lease"); + clock.advance(2_000); + bus.emit("lease.requested", { requestId: "new", requester: "a" } as never, "lease"); + + const read = await events.read({ carry: [], sinceTs: 2_000 }); + + expect([...read.requestedBefore]).toEqual(["old"]); + }); + it("returns the generations when the current file is missing", async () => { const filesystem = await filesystemWith({ "/data/events.jsonl.1": lines(envelope(1, 100)), @@ -442,4 +644,131 @@ describe("readEventFile", () => { expect(read.map((entry) => entry.seq)).toEqual([1, 3]); }); + + it("readEventFile with carry keeps the later step when a line written afterwards is older, and the line written last on a tie", async () => { + const step = (id: string, seq: number, timestamp: number): EventEnvelope => + ({ + event: "capacity.changed", + id, + module: "test", + payload: {}, + seq, + timestamp, + }) as unknown as EventEnvelope; + const read = async (...written: EventEnvelope[]) => + ( + await readEventFile( + await filesystemWith({ "/data/events.jsonl": lines(...written) }), + "/data/events.jsonl", + { + carry: ["capacity.changed"], + sinceTs: 300, + }, + ) + ).map((entry) => entry.id); + + expect(await read(step("evt_2", 2, 200), step("evt_3", 3, 100))).toEqual(["evt_2"]); + expect(await read(step("evt_5", 5, 200), step("evt_6", 5, 200))).toEqual(["evt_6"]); + }); + + it("readEventFile with carry keeps the latest of a named event for each requester, so a request still open when the window opens is carried", async () => { + const forRequester = ( + seq: number, + timestamp: number, + event: "lease.requested" | "lease.granted", + requester: string, + ): EventEnvelope => + ({ + event, + id: `evt_${seq}`, + module: "test", + payload: { requester }, + seq, + timestamp, + }) as unknown as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + forRequester(1, 100, "lease.requested", "a"), + forRequester(2, 110, "lease.requested", "b"), + forRequester(3, 120, "lease.granted", "a"), + forRequester(4, 130, "lease.requested", "a"), + forRequester(5, 400, "lease.requested", "c"), + ), + }); + + const read = await readEventFile(filesystem, "/data/events.jsonl", { + carry: ["lease.requested", "lease.granted"], + sinceTs: 300, + }); + + expect(read.map((entry) => entry.seq)).toEqual([2, 3, 4, 5]); + }); + + it("readEventFile with carry returns the latest capacity.changed and queue.changed at or before sinceTs, one per worker id, and none when there is none", async () => { + const step = ( + seq: number, + timestamp: number, + event: "capacity.changed" | "queue.changed", + workerId?: string, + ): EventEnvelope => + ({ + event, + id: `evt_${seq}`, + module: "test", + payload: { + ...(event === "queue.changed" ? { depth: seq } : {}), + ...(workerId === undefined ? {} : { workerId }), + }, + seq, + timestamp, + }) as unknown as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + step(1, 100, "capacity.changed"), + step(2, 200, "capacity.changed"), + step(3, 150, "queue.changed", "w1"), + step(4, 180, "queue.changed", "w2"), + step(5, 190, "queue.changed", "w1"), + envelope(6, 250), + envelope(7, 400), + step(8, 500, "capacity.changed"), + ), + }); + const carry = ["capacity.changed", "queue.changed"] as const; + + const read = await readEventFile(filesystem, "/data/events.jsonl", { carry, sinceTs: 300 }); + const none = await readEventFile(filesystem, "/data/events.jsonl", { carry, sinceTs: 50 }); + const plain = await readEventFile(filesystem, "/data/events.jsonl", { sinceTs: 300 }); + + // The newest step of each kind and worker at or before 300, then everything after it. + expect(read.map((entry) => entry.id)).toEqual(["evt_4", "evt_5", "evt_2", "evt_7", "evt_8"]); + expect(plain.map((entry) => entry.id)).toEqual(["evt_7", "evt_8"]); + // Nothing is at or before 50, so nothing is carried and nothing is repeated. + expect(none.map((entry) => entry.id)).toEqual([ + "evt_1", + "evt_3", + "evt_4", + "evt_5", + "evt_2", + "evt_6", + "evt_7", + "evt_8", + ]); + }); + + it("carries a step stamped exactly sinceTs, and counts it once", async () => { + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines({ + ...envelope(1, 300), + event: "capacity.changed", + } as unknown as EventEnvelope), + }); + + const read = await readEventFile(filesystem, "/data/events.jsonl", { + carry: ["capacity.changed"], + sinceTs: 300, + }); + + expect(read.map((entry) => entry.id)).toEqual(["evt_1"]); + }); }); diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index 8f4be2ce..70f62cbe 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -1,6 +1,6 @@ import { type Filesystem, isMissingPathError, type Logger, type LogSink } from "../ports/index.js"; import { EVENT_ID_PATTERN } from "../contract/schemas.js"; -import type { EventBus, EventEnvelope } from "./index.js"; +import type { EventBus, EventEnvelope, EventName } from "./index.js"; import { byTimeThenSeq } from "./order.js"; /** The durable record of every business event, in the data directory (ADR 0006). */ @@ -8,6 +8,9 @@ export const EVENT_FILE_NAME = "events.jsonl"; /** * Every envelope in the event file newer than `sinceTs`, by `timestamp` then `seq` (ADR 0014 §5). + * With `carry`, also the latest envelope of each named event at or before `sinceTs` -- one per + * `workerId` and one per `requester` where the payload names one -- so a reader of a step function (ADR 0016 §3) is told + * the step in force when its window opens. * The current file is read first, then its generations `.1`, `.2` and so on until * one is missing (ADR 0016 §4). Newest first means a rotation landing mid-read can only make a * generation show up twice -- never make one go missing -- and the repeat is dropped by `id`. @@ -20,7 +23,7 @@ export const EVENT_FILE_NAME = "events.jsonl"; export async function readEventFile( filesystem: Filesystem, path: string, - options: { readonly sinceTs: number }, + options: { readonly sinceTs: number; readonly carry?: readonly EventName[] }, ): Promise { return (await readEventHistory(filesystem, path, options)).events; } @@ -33,22 +36,86 @@ export async function readEventFile( export async function readEventHistory( filesystem: Filesystem, path: string, - { sinceTs }: { readonly sinceTs: number }, -): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { + { sinceTs, carry = [] }: { readonly sinceTs: number; readonly carry?: readonly EventName[] }, +): Promise { const generations = await readGenerations(filesystem, path); const seen = new Set(); const events: EventEnvelope[] = []; + const carried = new Map(); + const before = { answered: new Set(), requested: new Set() }; let oldestTs: number | undefined; // Oldest generation first, so events that tie on time and seq keep the order they were written. for (const line of generations.reverse().flat()) { const envelope = parseEnvelope(line); if (envelope === undefined) continue; oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); - if (envelope.timestamp <= sinceTs || seen.has(eventKey(envelope))) continue; + if (envelope.timestamp <= sinceTs) { + keepIfCarried(carried, envelope, carry); + noteRequest(before, envelope); + continue; + } + if (seen.has(eventKey(envelope))) continue; seen.add(eventKey(envelope)); events.push(envelope); } - return { events: events.sort(byTimeThenSeq), oldestTs }; + return { + events: [...carried.values(), ...events].sort(byTimeThenSeq), + oldestTs, + answeredBefore: before.answered, + requestedBefore: before.requested, + }; +} + +/** What a history read answers: the events, how far back it reaches, and which requests were made + * before `sinceTs` or answered by then (a rejection in the window follows its request, ADR 0016 + * §2). */ +export interface EventHistoryRead { + readonly events: EventEnvelope[]; + readonly oldestTs: number | undefined; + /** The `requestId` of every `lease.requested` at or before `sinceTs`. */ + readonly requestedBefore: ReadonlySet; + /** The `requestId` of every `lease.granted`, `lease.rejected` and `request.granted` at or before + * `sinceTs`: a request made before it and in neither set was still waiting when it opened. */ + readonly answeredBefore: ReadonlySet; +} + +const ANSWERS: ReadonlySet = new Set([ + "lease.granted", + "lease.rejected", + "request.granted", +]); + +/** Notes the request id of a `lease.requested` in `requested`, and of an answer in `answered`. */ +function noteRequest( + { requested, answered }: { readonly requested: Set; readonly answered: Set }, + envelope: EventEnvelope, +): void { + const requestId = (envelope.payload as { readonly requestId?: unknown }).requestId; + if (typeof requestId !== "string") return; + if (envelope.event === "lease.requested") requested.add(requestId); + if (ANSWERS.has(envelope.event)) answered.add(requestId); +} + +/** + * Keeps `envelope` as the step in force for its event, worker and requester when it is a named + * event and no later one is kept already. Ties on time and seq go to the one met last, the one + * written last. + */ +function keepIfCarried( + carried: Map, + envelope: EventEnvelope, + carry: readonly EventName[], +): void { + if (!carry.includes(envelope.event)) return; + const { workerId, requester } = envelope.payload as { + readonly workerId?: unknown; + readonly requester?: unknown; + }; + const key = [envelope.event, workerId, requester] + .map((part) => (typeof part === "string" ? part : "")) + .join("\u0000"); + const kept = carried.get(key); + if (kept === undefined || byTimeThenSeq(kept, envelope) <= 0) carried.set(key, envelope); } /** The lines of the current file, then of each generation, newest first, until two in a row are missing. */ @@ -142,20 +209,69 @@ export class EventHistory { /** * Without `sinceTs`, the ring, as `simlock events` has always answered. With it, the event - * file while the writer is writing; the ring when there is no file to trust. + * file while the writer is writing; the ring when there is no file to trust. `carry` adds + * the step in force for each named event at `sinceTs` (see `readEventFile`). */ - async replay(input: { readonly sinceTs?: number } = {}): Promise { - const { bus, filesystem, logger, path } = this.#options; - if (input.sinceTs === undefined) return bus.replay(); - if (!this.#writing) return bus.replay({ sinceTs: input.sinceTs }); + async replay( + input: { readonly sinceTs?: number; readonly carry?: readonly EventName[] } = {}, + ): Promise { + if (input.sinceTs === undefined) return this.#options.bus.replay(); + return (await this.read({ carry: input.carry ?? [], sinceTs: input.sinceTs })).events; + } + + /** + * What `replay` answers for `sinceTs`, with the oldest timestamp the history holds -- not only + * what is newer than `sinceTs` -- so a reader can say how far back the history reaches (ADR + * 0016 §5). From the ring when there is no file to trust, and then it is the ring's oldest. + */ + async read(input: { + readonly sinceTs: number; + readonly carry: readonly EventName[]; + }): Promise { + const { filesystem, logger, path } = this.#options; + if (!this.#writing) return this.#readRing(input); try { - return await readEventFile(filesystem, path, { sinceTs: input.sinceTs }); + return await readEventHistory(filesystem, path, input); } catch (error: unknown) { logger.warn("Event file read failed; replaying from memory", { error: error instanceof Error ? error.message : String(error), path, }); - return bus.replay({ sinceTs: input.sinceTs }); + return this.#readRing(input); + } + } + + /** The id of the event published last, `undefined` before the first. */ + latestId(): string | undefined { + return this.#options.bus.latestId(); + } + + #readRing({ + sinceTs, + carry, + }: { + readonly sinceTs: number; + readonly carry: readonly EventName[]; + }): EventHistoryRead { + const ring = this.#options.bus.replay(); + const carried = new Map(); + const before = { answered: new Set(), requested: new Set() }; + let oldestTs: number | undefined; + for (const envelope of ring) { + oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); + if (envelope.timestamp > sinceTs) continue; + keepIfCarried(carried, envelope, carry); + noteRequest(before, envelope); } + const newer = ring.filter((envelope) => envelope.timestamp > sinceTs); + return { + // Every carried envelope is at or before `sinceTs`, so the carried ones, put in time order + // (a key keeps its first slot in the map), come before the newer ones, which the ring replays + // in order. + events: [...[...carried.values()].sort(byTimeThenSeq), ...newer], + oldestTs, + answeredBefore: before.answered, + requestedBefore: before.requested, + }; } } diff --git a/src/bus/index.ts b/src/bus/index.ts index e98723c4..1219ee8e 100644 --- a/src/bus/index.ts +++ b/src/bus/index.ts @@ -1,6 +1,8 @@ import { type Clock, CryptoIdGenerator, type IdGenerator } from "../ports/index.js"; import { byTimeThenSeq } from "./order.js"; +export { byTimeThenSeq }; + export interface CapacityFiguresPayload { readonly running: number; readonly maxRunning: number; @@ -435,6 +437,7 @@ export class EventBus { readonly #events: Array; #nextEventIndex = 0; #eventCount = 0; + #latestId: string | undefined; constructor( private readonly clock: Clock, @@ -520,7 +523,13 @@ export class EventBus { .sort(byTimeThenSeq); } + /** The id of the envelope published last, by arrival; `undefined` before the first. */ + latestId(): string | undefined { + return this.#latestId; + } + #append(envelope: EventEnvelope): void { + this.#latestId = envelope.id; this.#events[this.#nextEventIndex] = envelope; this.#nextEventIndex = (this.#nextEventIndex + 1) % this.capacity; if (this.#eventCount < this.capacity) { diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index b41d0f00..25872ffc 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -29,7 +29,12 @@ import { DaemonEndpointHost } from "../daemon/connection-host.js"; import { DaemonServer } from "../daemon/server.js"; import { AdminSecretManager } from "../daemon/admin-secret.js"; import { createCredentialRoleResolver } from "../daemon/session.js"; -import { fromWireError, SimlockError, type AnySimlockError } from "../contract/index.js"; +import { + fromWireError, + SimlockError, + type AnySimlockError, + type UsageOutput, +} from "../contract/index.js"; import type { CatalogGetOutput, DeviceRecoveredPush, @@ -5089,6 +5094,7 @@ function fakeClient(overrides: Partial = {}): SimlockAdminCl getConfig: () => Promise.resolve({} as SimlockConfig), stopDaemon: () => Promise.resolve({ stopping: true }), replayEvents: () => Promise.resolve([]), + usage: (window) => Promise.resolve(usageAnswer(window)), subscribeEvents: () => Promise.resolve(() => Promise.resolve()), createToken: (input) => Promise.resolve({ @@ -5708,3 +5714,233 @@ describe("simlock lease: a request names a model, a class, or nothing", () => { ); }); }); + +/** A `usage.get` answer with nothing counted, for the window it was asked for. */ +function usageAnswer( + window: { readonly from: number; readonly to: number }, + overrides: Partial = {}, +): UsageOutput { + const samples = { count: 0, max: null, p50: null, p95: null }; + const figures = { + boot: samples, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, + failures: { byEvent: {} }, + granted: 0, + held: samples, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: samples, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: samples, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: samples, + }; + return { + bucketMs: 60_000, + coversFrom: window.from, + partial: false, + platforms: { android: figures, ios: figures }, + requesters: [], + series: [], + totals: figures, + window, + workers: [], + ...overrides, + }; +} + +describe("CLI: stats", () => { + const NOW = Date.parse("2026-10-05T12:00:00.000Z"); + const HOUR = 3_600_000; + + function statsRun(argv: readonly string[], usage: SimlockAdminClient["usage"]) { + const output = outputCapture(); + const asked: { from: number; to: number }[] = []; + const run = runCli( + ["stats", ...argv], + output.environmentWith({ + clock: new FakeClock(NOW), + connectAdmin: async () => + fakeClient({ + usage: async (window) => { + asked.push(window); + return usage(window); + }, + }), + }), + ); + return { asked, output, run }; + } + + it("simlock stats rejects --since with --from, and --from later than --to, as usage errors", async () => { + // Each flag on its own is accepted, so the refusals below are the combinations'. + for (const argv of [["--since", "1h"], ["--from", "2026-10-05T10:00:00Z"], ["--json"]]) { + const { output, run } = statsRun(argv, async (window) => usageAnswer(window)); + await expect(run, argv.join(" ")).resolves.toBe(0); + expect(output.stderr, argv.join(" ")).not.toContain("USAGE"); + } + for (const argv of [ + ["--since", "1h", "--from", "2026-10-05T10:00:00Z"], + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-03T10:00:00Z"], + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T10:00:00Z"], + ["--to", "2026-10-04T10:00:00Z"], + ["--since", "1h", "--to", "2026-10-04T10:00:00Z"], + ["--since", "soon"], + ["--from", "last tuesday"], + ]) { + const { asked, output, run } = statsRun(argv, async (window) => usageAnswer(window)); + + await expect(run, argv.join(" ")).resolves.toBe(2); + expect(output.stderr, argv.join(" ")).toContain('"code":"USAGE"'); + expect(asked, argv.join(" ")).toEqual([]); + expect(output.stdout).toBe(""); + } + }); + + it("simlock stats names the flag combination or the time it refused in the usage error", async () => { + for (const [argv, message] of [ + [ + ["--since", "1h", "--from", "2026-10-05T10:00:00Z"], + "stats takes --since or --from, not both", + ], + [["--to", "2026-10-04T10:00:00Z"], "--to needs --from"], + [ + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-03T10:00:00Z"], + "--from must be earlier than --to, or than now when --to is left out", + ], + [["--from", "last tuesday"], "Invalid time: last tuesday"], + ] as const) { + const { output, run } = statsRun(argv, async (window) => usageAnswer(window)); + + await expect(run, argv.join(" ")).resolves.toBe(2); + expect(output.stderr, argv.join(" ")).toContain(`"message":"${message}"`); + } + }); + + it("simlock stats --json prints the operation's output unchanged", async () => { + const answer = usageAnswer( + { from: NOW - HOUR, to: NOW }, + { + partial: true, + requesters: [ + { granted: 1, heldTotalMs: 5, id: "tok_a", label: "ci", rejected: 0, requests: 1 }, + ], + }, + ); + const { output, run } = statsRun(["--since", "1h", "--json"], async () => answer); + + await expect(run).resolves.toBe(0); + + expect(JSON.parse(output.stdout)).toEqual(answer); + expect(output.stdout.endsWith("\n")).toBe(true); + }); + + it("asks for the last 24 hours by default, and for --since up to now", async () => { + const byDefault = statsRun([], async (window) => usageAnswer(window)); + const since = statsRun(["--since", "90m"], async (window) => usageAnswer(window)); + + await byDefault.run; + await since.run; + + expect(byDefault.asked).toEqual([{ from: NOW - 24 * HOUR, to: NOW }]); + expect(since.asked).toEqual([{ from: NOW - 90 * 60_000, to: NOW }]); + }); + + it("asks for the ISO window --from and --to name, and for --from up to now", async () => { + const both = statsRun( + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T12:00:00Z"], + async (window) => usageAnswer(window), + ); + const open = statsRun(["--from", "2026-10-05T10:00:00Z"], async (window) => + usageAnswer(window), + ); + + await both.run; + await open.run; + + expect(both.asked).toEqual([ + { from: Date.parse("2026-10-04T10:00:00Z"), to: Date.parse("2026-10-04T12:00:00Z") }, + ]); + expect(open.asked).toEqual([{ from: Date.parse("2026-10-05T10:00:00Z"), to: NOW }]); + }); + + it("prints the figures as a table, with the partial note above them", async () => { + const { output, run } = statsRun(["--since", "30d"], async (window) => + usageAnswer(window, { coversFrom: NOW - 60_000, partial: true }), + ); + + await expect(run).resolves.toBe(0); + + const lines = output.stdout.split("\n"); + expect(lines[0]).toBe("Usage from 2026-09-05T12:00:00.000Z to 2026-10-05T12:00:00.000Z"); + expect(lines[1]).toContain("Figures cover from 2026-10-05T11:59:00.000Z"); + expect(lines).toContain("Totals"); + }); + + it("prints the HISTORY_NOT_KEPT message and exits with its code when the window ends before the history", async () => { + const { output, run } = statsRun(["--since", "30d"], async () => { + throw new SimlockError( + "HISTORY_NOT_KEPT", + "domain", + "The event history does not reach back to the end of that window; its oldest event is from 2026-10-05T11:59:00.000Z.", + { oldestTs: NOW - 60_000 }, + ); + }); + + await expect(run).resolves.toBe(12); + + expect(output.stdout).toBe(""); + expect(output.stderr).toContain('"code":"HISTORY_NOT_KEPT"'); + expect(output.stderr).toContain("oldest event is from 2026-10-05T11:59:00.000Z"); + }); + + it("closes the connection after the answer, and after a failure", async () => { + let closed = 0; + const output = outputCapture(); + const connect = (usage: SimlockAdminClient["usage"]) => async () => + fakeClient({ + close: async () => { + closed += 1; + }, + usage, + }); + + await runCli( + ["stats"], + output.environmentWith({ connectAdmin: connect(async (window) => usageAnswer(window)) }), + ); + await runCli( + ["stats"], + output.environmentWith({ + connectAdmin: connect(async () => { + throw new Error("boom"); + }), + }), + ); + + expect(closed).toBe(2); + }); + + it("prints its usage on --help without connecting", async () => { + const output = outputCapture(); + let connected = false; + + const exitCode = await runCli( + ["stats", "--help"], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ); + + expect(exitCode).toBe(0); + expect(connected).toBe(false); + expect(output.stdout).toBe( + "Usage: simlock stats [--since | --from [--to ]] [--json]\n", + ); + }); +}); diff --git a/src/cli/index.ts b/src/cli/index.ts index 6eb0e68c..2ad29350 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -55,6 +55,7 @@ import { type LeaseRenewal, } from "../lease-policy/index.js"; import { followLog, type Signals } from "./follow-log.js"; +import { formatUsage } from "./stats.js"; import { spawnPassthrough } from "./passthrough.js"; import { ERROR_TABLE } from "../contract/index.js"; import { parseDurationMs } from "../contract/duration.js"; @@ -65,7 +66,7 @@ const USAGE = `Usage: simlock [options] Commands: lease, release, status, list, catalog, cleanup, doctor, nuke, events, - daemon, config, token + stats, daemon, config, token worker Inspect and manage the workers of a gateway; on a single host, list shows the host itself @@ -618,6 +619,8 @@ export async function runCli( return await runNuke(rest.slice(1), environment, token); case "events": return await runEvents(rest.slice(1), environment, token); + case "stats": + return await runStats(rest.slice(1), environment, token); case "daemon": return await runDaemon(rest.slice(1), environment, token); case "config": @@ -1558,6 +1561,69 @@ async function runEvents( } } +const DAY_MS = 24 * 60 * 60 * 1000; + +/** `simlock stats`: the usage figures for a window, from the daemon's event history (ADR 0016). */ +async function runStats( + argv: readonly string[], + environment: CliEnvironment, + token: string | undefined, +): Promise { + const values = commandArgs(argv, { + from: { type: "string" }, + help: { type: "boolean", short: "h" }, + json: { type: "boolean" }, + since: { type: "string" }, + to: { type: "string" }, + }); + if (values.help) { + environment.stdout.write( + "Usage: simlock stats [--since | --from [--to ]] [--json]\n", + ); + return 0; + } + const window = statsWindow(values, environment.clock.now()); + const client = await connectDaemonClient(environment, token); + try { + const usage = await client.usage(window); + if (values.json) writeResult(environment, usage); + else environment.stdout.write(`${formatUsage(usage)}\n`); + return 0; + } finally { + await client.close(); + } +} + +/** + * The window `simlock stats` asks for: `--since` back from now, or `--from` to `--to` (now when + * `--to` is left out), the last 24 hours when none is given. Flags that disagree are usage errors. + */ +function statsWindow( + values: Readonly>, + now: number, +): { readonly from: number; readonly to: number } { + const { from, since, to } = values; + if (from !== undefined && since !== undefined) { + throw new UsageError("stats takes --since or --from, not both"); + } + if (to !== undefined && from === undefined) throw new UsageError("--to needs --from"); + if (typeof from !== "string") { + return { from: now - (typeof since === "string" ? parseDuration(since) : DAY_MS), to: now }; + } + const window = { from: parseTime(from), to: typeof to === "string" ? parseTime(to) : now }; + if (window.from >= window.to) { + throw new UsageError("--from must be earlier than --to, or than now when --to is left out"); + } + return window; +} + +/** Parses an ISO time only at the CLI boundary. */ +function parseTime(value: string): number { + const milliseconds = Date.parse(value); + if (Number.isNaN(milliseconds)) throw new UsageError(`Invalid time: ${value}`); + return milliseconds; +} + /** * Connects for `simlock events`. History alone (`--since` without `--follow`, passed as * `historySinceTs`) needs no daemon: with nothing listening it prints the event file's history diff --git a/src/cli/stats.test.ts b/src/cli/stats.test.ts new file mode 100644 index 00000000..876f2e3c --- /dev/null +++ b/src/cli/stats.test.ts @@ -0,0 +1,380 @@ +import { describe, expect, it } from "vitest"; + +import type { UsageFigures, UsageOutput } from "../contract/index.js"; +import { formatUsage } from "./stats.js"; + +const NONE = { count: 0, max: null, p50: null, p95: null }; + +function figuresFixture(overrides: Partial = {}): UsageFigures { + return { + boot: NONE, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, + failures: { byEvent: {} }, + granted: 0, + held: NONE, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: NONE, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: NONE, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: NONE, + ...overrides, + }; +} + +function usageFixture(overrides: Partial = {}): UsageOutput { + return { + bucketMs: 900_000, + coversFrom: Date.parse("2026-10-04T10:00:00.000Z"), + partial: false, + platforms: { android: figuresFixture(), ios: figuresFixture() }, + requesters: [], + series: [], + totals: figuresFixture(), + window: { + from: Date.parse("2026-10-04T10:00:00.000Z"), + to: Date.parse("2026-10-05T10:00:00.000Z"), + }, + workers: [], + ...overrides, + }; +} + +const BUSY = figuresFixture({ + boot: { count: 2, max: 40_000, p50: 20_000, p95: 40_000 }, + bySource: { booted: 3, provisioned: 1, warm: 6 }, + failures: { byEvent: { "device.purge-failed": 2 } }, + granted: 10, + held: { count: 9, max: 3_725_000, p50: 300_000, p95: 1_800_000 }, + incidents: { crashRecovered: 1, lost: 4, quarantineRecovered: 3, quarantined: 2 }, + provisioning: { count: 1, max: 90_000, p50: 90_000, p95: 90_000 }, + queue: { meanDepth: 0.4, peakDepth: 2 }, + rejected: { byReason: { "no-wait": 1, timeout: 1 }, total: 2 }, + requests: 12, + turnaround: { count: 9, max: 3_800_000, p50: 310_000, p95: 1_900_000 }, + utilisation: { + ram: { limitBytes: 16 * 1024 ** 3, meanBytes: 1024 ** 3, peakBytes: 2 * 1024 ** 3 }, + slots: { max: 4, mean: 1.5, peak: 3 }, + }, + wait: { count: 11, max: 9_100, p50: 1_200, p95: 4_000 }, +}); + +describe("formatUsage", () => { + it("opens with the window and, when the history falls short of it, the line that says where the figures start", () => { + const partial = formatUsage( + usageFixture({ coversFrom: Date.parse("2026-10-05T09:59:00.000Z"), partial: true }), + ).split("\n"); + const whole = formatUsage(usageFixture()); + + expect(partial[0]).toBe("Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z"); + expect(partial[1]).toBe( + "Figures cover from 2026-10-05T09:59:00.000Z: the history does not reach back to the start of the window.", + ); + expect(whole).not.toContain("Figures cover from"); + }); + + it("prints every total on a line of its own", () => { + const lines = formatUsage(usageFixture({ totals: BUSY })).split("\n"); + + expect(lines).toContain("Totals"); + expect(lines).toContain(" Requests: 12 (10 granted, 2 rejected)"); + expect(lines).toContain(" Granted: warm 6, booted 3, provisioned 1"); + expect(lines).toContain(" Rejected: no-wait 1, timeout 1"); + expect(lines).toContain(" Wait: p50 1.2s, p95 4s, max 9.1s (11 samples)"); + expect(lines).toContain(" Held: p50 5m 0s, p95 30m 0s, max 1h 2m (9 samples)"); + expect(lines).toContain(" Turnaround: p50 5m 10s, p95 31m 40s, max 1h 3m (9 samples)"); + expect(lines).toContain(" Provisioning: p50 1m 30s, p95 1m 30s, max 1m 30s (1 sample)"); + expect(lines).toContain(" Boot: p50 20s, p95 40s, max 40s (2 samples)"); + expect(lines).toContain(" Slots: peak 3 of 4, mean 1.5"); + expect(lines).toContain(" RAM: peak 2.0 GiB of 16.0 GiB, mean 1.0 GiB"); + expect(lines).toContain(" Queue: peak depth 2, mean 0.4"); + expect(lines).toContain( + " Incidents: 2 quarantined, 1 recovered after a crash, 3 recovered from quarantine, 4 lost", + ); + expect(lines).toContain(" Failures: device.purge-failed 2"); + }); + + it("says plainly what it has no figure for, and leaves out rows it has nothing to say in", () => { + const text = formatUsage(usageFixture()); + + expect(text).toContain(" Requests: 0 (0 granted, 0 rejected)"); + expect(text).toContain(" Wait: no samples"); + expect(text).toContain(" Slots: not known"); + expect(text).toContain(" Queue: not known"); + expect(text).not.toContain("RAM:"); + expect(text).not.toContain("Failures:"); + expect(text).not.toContain("Rejected:"); + expect(text).not.toContain("Granted:"); + expect(text).not.toContain("Workers"); + expect(text).not.toContain("Requesters"); + }); + + it("prints a row for each platform, each worker with its label, and each requester with its label", () => { + const lines = formatUsage( + usageFixture({ + platforms: { android: figuresFixture(), ios: BUSY }, + requesters: [ + { + granted: 2, + heldTotalMs: 725_000, + id: "tok_a", + label: "ci-bot", + rejected: 1, + requests: 3, + }, + { granted: 0, heldTotalMs: 0, id: "tok_b", rejected: 0, requests: 1 }, + ], + totals: BUSY, + workers: [ + { ...BUSY, id: "wrk_1", label: "mac-mini-1" }, + { ...figuresFixture(), id: "wrk_2" }, + ], + }), + ).split("\n"); + + expect(lines).toContain("Platforms"); + expect(lines).toContain( + " ios 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3", + ); + expect(lines).toContain( + " android 0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak -", + ); + expect(lines).toContain("Workers"); + expect(lines).toContain( + " mac-mini-1 (wrk_1) 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3", + ); + expect(lines).toContain( + " wrk_2 0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak -", + ); + expect(lines).toContain("Requesters"); + expect(lines).toContain(" ci-bot (tok_a) 3 requests, 2 granted, 1 rejected, held 12m 5s"); + expect(lines).toContain(" tok_b 1 request, 0 granted, 0 rejected, held 0s"); + }); +}); + +const PLATFORM_ROW = "0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak -"; +const TOTAL_ROWS = [ + "Totals", + " Requests: 0 (0 granted, 0 rejected)", + " Wait: no samples", + " Held: no samples", + " Turnaround: no samples", + " Provisioning: no samples", + " Boot: no samples", + " Slots: not known", + " Queue: not known", + " Incidents: 0 quarantined, 0 recovered after a crash, 0 recovered from quarantine, 0 lost", +]; + +describe("formatUsage layout", () => { + it("separates the sections with one blank line each, in the order of the header, totals, platforms, workers and requesters", () => { + const text = formatUsage( + usageFixture({ + coversFrom: Date.parse("2026-10-05T09:59:00.000Z"), + partial: true, + requesters: [{ granted: 0, heldTotalMs: 0, id: "tok_b", rejected: 0, requests: 1 }], + workers: [{ ...figuresFixture(), id: "wrk_2" }], + }), + ); + + expect(text).toBe( + [ + "Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z", + "Figures cover from 2026-10-05T09:59:00.000Z: the history does not reach back to the start of the window.", + "", + ...TOTAL_ROWS, + "", + "Platforms", + ` android ${PLATFORM_ROW}`, + ` ios ${PLATFORM_ROW}`, + "", + "Workers", + ` wrk_2 ${PLATFORM_ROW}`, + "", + "Requesters", + " tok_b 1 request, 0 granted, 0 rejected, held 0s", + ].join("\n"), + ); + }); + + it("prints a whole report without the optional sections when there are no workers or requesters", () => { + expect(formatUsage(usageFixture())).toBe( + [ + "Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z", + "", + ...TOTAL_ROWS, + "", + "Platforms", + ` android ${PLATFORM_ROW}`, + ` ios ${PLATFORM_ROW}`, + ].join("\n"), + ); + }); +}); + +describe("formatUsage rows that depend on the figures", () => { + it("lists the error codes and the rejection reasons by name, whatever order they arrived in", () => { + const lines = formatUsage( + usageFixture({ + totals: figuresFixture({ + failures: { byEvent: { "c.zed": 1, "a.alpha": 2, "b.mid": 3 } }, + rejected: { byReason: { timeout: 1, "no-wait": 2, capacity: 3 }, total: 6 }, + }), + }), + ).split("\n"); + + expect(lines).toContain(" Failures: a.alpha 2, b.mid 3, c.zed 1"); + expect(lines).toContain(" Rejected: capacity 3, no-wait 2, timeout 1"); + }); + + it("prints the Failures row only when there are failure events", () => { + const without = formatUsage(usageFixture()); + const withOne = formatUsage( + usageFixture({ totals: figuresFixture({ failures: { byEvent: { "x.y": 1 } } }) }), + ); + + expect(without.split("\n").some((line) => line.startsWith(" Failures:"))).toBe(false); + expect(withOne.split("\n")).toContain(" Failures: x.y 1"); + }); + + it("prints the Granted row only when something was granted", () => { + const withGrant = formatUsage( + usageFixture({ + totals: figuresFixture({ bySource: { booted: 0, provisioned: 0, warm: 1 }, granted: 1 }), + }), + ).split("\n"); + const without = formatUsage(usageFixture()).split("\n"); + + expect(withGrant).toContain(" Granted: warm 1, booted 0, provisioned 0"); + expect(without.some((line) => line.startsWith(" Granted:"))).toBe(false); + }); + + it("prints the unknown grant source only when a gateway counted one", () => { + const unknown = formatUsage( + usageFixture({ + totals: figuresFixture({ + bySource: { booted: 0, provisioned: 0, unknown: 2, warm: 1 }, + granted: 3, + }), + }), + ).split("\n"); + const none = formatUsage( + usageFixture({ + totals: figuresFixture({ + bySource: { booted: 0, provisioned: 0, unknown: 0, warm: 1 }, + granted: 1, + }), + }), + ).split("\n"); + + expect(unknown).toContain(" Granted: warm 1, booted 0, provisioned 0, unknown 2"); + expect(none).toContain(" Granted: warm 1, booted 0, provisioned 0"); + }); + + it("prints the Declined row only when a worker declined something, and the Probes row only when a worker counted a probe", () => { + const both = formatUsage( + usageFixture({ totals: figuresFixture({ declined: 4, probes: 7 }) }), + ).split("\n"); + const without = formatUsage( + usageFixture({ totals: figuresFixture({ declined: 0, probes: 0 }) }), + ).split("\n"); + const gateway = formatUsage(usageFixture({ totals: figuresFixture({ declined: 1 }) })).split( + "\n", + ); + + expect(both).toContain(" Declined: 4"); + expect(both).toContain(" Probes: 7"); + expect(without.some((line) => /^ {2}(Declined|Probes):/.test(line))).toBe(false); + expect(gateway).toContain(" Declined: 1"); + expect(gateway.some((line) => line.startsWith(" Probes:"))).toBe(false); + }); + + it("prints the Rejected row only when something was rejected", () => { + const withReject = formatUsage( + usageFixture({ + totals: figuresFixture({ rejected: { byReason: { timeout: 1 }, total: 1 } }), + }), + ).split("\n"); + const without = formatUsage(usageFixture()).split("\n"); + + expect(withReject).toContain(" Rejected: timeout 1"); + expect(without.some((line) => line.startsWith(" Rejected:"))).toBe(false); + }); + + it("prints the RAM row only when the figures carry RAM", () => { + const withRam = formatUsage( + usageFixture({ + totals: figuresFixture({ + utilisation: { + ram: { limitBytes: 8 * 1024 ** 3, meanBytes: 0, peakBytes: 1024 ** 3 }, + slots: { max: null, mean: null, peak: null }, + }, + }), + }), + ).split("\n"); + const without = formatUsage(usageFixture()).split("\n"); + + expect(withRam).toContain(" RAM: peak 1.0 GiB of 8.0 GiB, mean 0.0 GiB"); + expect(without.some((line) => line.startsWith(" RAM:"))).toBe(false); + }); + + it.each([ + ["peak", { max: 4, mean: 1.5, peak: null }], + ["max", { max: null, mean: 1.5, peak: 3 }], + ["mean", { max: 4, mean: null, peak: 3 }], + ])("says slots are not known when only the slot %s is missing", (_name, slots) => { + const lines = formatUsage( + usageFixture({ totals: figuresFixture({ utilisation: { slots } }) }), + ).split("\n"); + + expect(lines).toContain(" Slots: not known"); + }); + + it.each([ + ["peak depth", { meanDepth: 0.4, peakDepth: null }], + ["mean depth", { meanDepth: null, peakDepth: 2 }], + ])("says the queue is not known when only the queue %s is missing", (_name, queue) => { + const lines = formatUsage(usageFixture({ totals: figuresFixture({ queue }) })).split("\n"); + + expect(lines).toContain(" Queue: not known"); + }); + + it.each([ + ["p50", { count: 1, max: 2, p50: null, p95: 1 }], + ["p95", { count: 1, max: 2, p50: 1, p95: null }], + ["max", { count: 1, max: null, p50: 1, p95: 2 }], + ])("says there are no samples when only the %s is missing", (_name, wait) => { + const lines = formatUsage(usageFixture({ totals: figuresFixture({ wait }) })).split("\n"); + + expect(lines).toContain(" Wait: no samples"); + }); +}); + +describe("formatUsage durations", () => { + const waitP50 = (ms: number): string => { + const lines = formatUsage( + usageFixture({ + totals: figuresFixture({ wait: { count: 1, max: ms, p50: ms, p95: ms } }), + }), + ).split("\n"); + const line = lines.find((candidate) => candidate.startsWith(" Wait:")); + return (line ?? "").replace(/^ {2}Wait: +p50 /, "").replace(/,.*$/, ""); + }; + + it.each([ + [0, "0s"], + [250, "250ms"], + [999, "999ms"], + [1_000, "1s"], + [59_999, "60s"], + [60_000, "1m 0s"], + [3_599_999, "59m 59s"], + [3_600_000, "1h 0m"], + [7_380_000, "2h 3m"], + ])("prints %i ms as %s", (ms, expected) => { + expect(waitP50(ms)).toBe(expected); + }); +}); diff --git a/src/cli/stats.ts b/src/cli/stats.ts new file mode 100644 index 00000000..2c45e1cc --- /dev/null +++ b/src/cli/stats.ts @@ -0,0 +1,175 @@ +import type { UsageFigures, UsageOutput } from "../contract/index.js"; + +/** The human rendering of `usage.get`'s answer for `simlock stats`. */ +export function formatUsage(usage: UsageOutput): string { + const lines = [`Usage from ${iso(usage.window.from)} to ${iso(usage.window.to)}`]; + if (usage.partial) { + lines.push( + `Figures cover from ${iso(usage.coversFrom)}: the history does not reach back to the start of the window.`, + ); + } + lines.push("", "Totals", ...totalRows(usage.totals)); + lines.push( + "", + "Platforms", + ...rows(Object.entries(usage.platforms), (figures) => summary(figures)), + ); + if (usage.workers.length > 0) { + lines.push( + "", + "Workers", + ...rows( + usage.workers.map((worker) => [ + worker.label === undefined ? worker.id : `${worker.label} (${worker.id})`, + worker, + ]), + (figures) => summary(figures), + ), + ); + } + if (usage.requesters.length > 0) { + lines.push( + "", + "Requesters", + ...rows( + usage.requesters.map((requester) => [ + requester.label === undefined ? requester.id : `${requester.label} (${requester.id})`, + requester, + ]), + (requester) => + `${plural(requester.requests, "request")}, ${requester.granted} granted, ` + + `${requester.rejected} rejected, held ${duration(requester.heldTotalMs)}`, + ), + ); + } + return lines.join("\n"); +} + +const iso = (ms: number): string => new Date(ms).toISOString(); + +const plural = (count: number, noun: string): string => `${count} ${noun}${count === 1 ? "" : "s"}`; + +/** One row for each `[name, item]`, the names set to a common width. */ +function rows( + entries: readonly (readonly [string, Item])[], + render: (item: Item) => string, +): string[] { + const width = Math.max(...entries.map(([name]) => name.length)); + return entries.map(([name, item]) => ` ${name.padEnd(width)} ${render(item)}`); +} + +/** A platform's or a worker's figures on one line. */ +function summary(figures: UsageFigures): string { + return ( + `${plural(figures.requests, "request")}, ${figures.granted} granted, ` + + `${figures.rejected.total} rejected, wait p50 ${duration(figures.wait.p50)}, ` + + `held p50 ${duration(figures.held.p50)}, slots peak ${figures.utilisation.slots.peak ?? "-"}` + ); +} + +const row = (label: string, value: string): string => ` ${label.padEnd(14)}${value}`; + +/** `name count` pairs, by name. */ +function list(counts: Readonly>): string { + return Object.entries(counts) + .sort(([left], [right]) => (left < right ? -1 : 1)) + .map(([name, count]) => `${name} ${count}`) + .join(", "); +} + +function totalRows(totals: UsageFigures): string[] { + return [ + ...requestRows(totals), + row("Wait:", samples(totals.wait)), + row("Held:", samples(totals.held)), + row("Turnaround:", samples(totals.turnaround)), + row("Provisioning:", samples(totals.provisioning)), + row("Boot:", samples(totals.boot)), + ...capacityRows(totals), + row( + "Incidents:", + `${totals.incidents.quarantined} quarantined, ${totals.incidents.crashRecovered} recovered after a crash, ` + + `${totals.incidents.quarantineRecovered} recovered from quarantine, ${totals.incidents.lost} lost`, + ), + ...(Object.keys(totals.failures.byEvent).length === 0 + ? [] + : [row("Failures:", list(totals.failures.byEvent))]), + ]; +} + +/** The requests, and what became of them, leaving out the rows with nothing in them. */ +function requestRows({ + bySource, + declined, + granted, + probes, + rejected, + requests, +}: UsageFigures): string[] { + return [ + row("Requests:", `${requests} (${granted} granted, ${rejected.total} rejected)`), + ...(granted === 0 + ? [] + : [ + row( + "Granted:", + `warm ${bySource.warm}, booted ${bySource.booted}, provisioned ${bySource.provisioned}` + + (bySource.unknown === undefined || bySource.unknown === 0 + ? "" + : `, unknown ${bySource.unknown}`), + ), + ]), + ...(rejected.total === 0 ? [] : [row("Rejected:", list(rejected.byReason))]), + ...(declined === 0 ? [] : [row("Declined:", String(declined))]), + ...(probes === undefined || probes === 0 ? [] : [row("Probes:", String(probes))]), + ]; +} + +function capacityRows({ queue, utilisation }: UsageFigures): string[] { + const { ram, slots } = utilisation; + return [ + row( + "Slots:", + slots.peak === null || slots.max === null || slots.mean === null + ? "not known" + : `peak ${figure(slots.peak)} of ${figure(slots.max)}, mean ${figure(slots.mean)}`, + ), + ...(ram === undefined + ? [] + : [ + row( + "RAM:", + `peak ${gibibytes(ram.peakBytes)} of ${gibibytes(ram.limitBytes)}, mean ${gibibytes(ram.meanBytes)}`, + ), + ]), + row( + "Queue:", + queue.peakDepth === null || queue.meanDepth === null + ? "not known" + : `peak depth ${figure(queue.peakDepth)}, mean ${figure(queue.meanDepth)}`, + ), + ]; +} + +function samples(figures: UsageFigures["wait"]): string { + if (figures.p50 === null || figures.p95 === null || figures.max === null) return "no samples"; + return ( + `p50 ${duration(figures.p50)}, p95 ${duration(figures.p95)}, max ${duration(figures.max)} ` + + `(${plural(figures.count, "sample")})` + ); +} + +/** A number to one decimal, without a trailing `.0`. */ +const figure = (value: number): string => String(Math.round(value * 10) / 10); + +const gibibytes = (bytes: number): string => `${(bytes / 1024 ** 3).toFixed(1)} GiB`; + +/** A time in the unit a person reads it in; `-` for a figure there is none of. */ +function duration(ms: number | null): string { + if (ms === null) return "-"; + if (ms === 0) return "0s"; + if (ms < 1_000) return `${ms}ms`; + if (ms < 60_000) return `${figure(ms / 1_000)}s`; + if (ms < 3_600_000) return `${Math.floor(ms / 60_000)}m ${Math.floor((ms % 60_000) / 1_000)}s`; + return `${Math.floor(ms / 3_600_000)}h ${Math.floor((ms % 3_600_000) / 60_000)}m`; +} diff --git a/src/contract/errors.test.ts b/src/contract/errors.test.ts index c044c2e6..ee0858f2 100644 --- a/src/contract/errors.test.ts +++ b/src/contract/errors.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; -import { ERROR_TABLE, fromWireError, isSimlockError } from "./errors.js"; +import { + CODES_WITH_DECLARED_DETAILS, + ERROR_TABLE, + fromWireError, + isSimlockError, +} from "./errors.js"; describe("SimlockError", () => { it("narrows details by code", () => { @@ -18,6 +23,22 @@ describe("SimlockError", () => { } }); + it("HISTORY_NOT_KEPT is a domain error with exit code 12, HTTP status 422, and the oldest held time in its details", () => { + expect(ERROR_TABLE.HISTORY_NOT_KEPT).toEqual({ + cliExitCode: 12, + code: "HISTORY_NOT_KEPT", + httpStatus: 422, + kind: "domain", + }); + + const error = fromWireError("HISTORY_NOT_KEPT", "not kept", { oldestTs: 42 }); + + expect(error.code).toBe("HISTORY_NOT_KEPT"); + if (error.code === "HISTORY_NOT_KEPT") expect(error.details.oldestTs).toBe(42); + else throw new Error("expected HISTORY_NOT_KEPT"); + expect(CODES_WITH_DECLARED_DETAILS.has("HISTORY_NOT_KEPT")).toBe(true); + }); + it("wraps an unrecognized code as UNKNOWN_DAEMON_ERROR instead of throwing", () => { const error = fromWireError("SOME_FUTURE_CODE", "a newer daemon said so"); expect(isSimlockError(error)).toBe(true); diff --git a/src/contract/errors.ts b/src/contract/errors.ts index 4961aa6e..65b80e30 100644 --- a/src/contract/errors.ts +++ b/src/contract/errors.ts @@ -93,6 +93,11 @@ export interface ErrorDetailsMap { COMPONENT_IN_USE: { readonly devices: number; readonly foreignDevices: number }; /** ADR 0010 §8: a `component.remove` while an install or removal runs or waits on the platform. */ COMPONENT_BUSY: Record; + /** + * ADR 0016 §5: a `usage.get` window that ends before the oldest event the history holds. + * `oldestTs` is the oldest time the history reaches, so a caller can ask for a window it has. + */ + HISTORY_NOT_KEPT: { readonly oldestTs: number }; UNKNOWN_LEASE: { readonly leaseId: string }; /** A `simlock ` verb the owning driver will not proxy (ADR 0001, decision 7). Carries * the tool so a caller can say which wrapper refused without re-parsing the message. */ @@ -203,6 +208,7 @@ const CODES_WITH_DECLARED_DETAILS_BY_CODE: Record DOWNLOAD_TIMEOUT: true, DOWNLOADS_DISABLED: true, COMPONENT_IN_USE: true, + HISTORY_NOT_KEPT: true, UNKNOWN_LEASE: true, PASSTHROUGH_REFUSED: true, UNKNOWN_PASSTHROUGH_TOOL: true, @@ -390,6 +396,8 @@ export const ERROR_TABLE: { readonly [Code in SimlockErrorCode]: ErrorTableEntry WORKER_CONNECTED: { code: "WORKER_CONNECTED", kind: "domain", cliExitCode: 2, httpStatus: 409 }, // Exit 12, beside `UNKNOWN_MODEL`: the "what you named does not exist" class, which a script // branches on differently from "the fleet is busy" or "that cannot apply here". + // Exit 12 and 422, with the other "what you asked for is not there" codes (ADR 0016 §5). + HISTORY_NOT_KEPT: { code: "HISTORY_NOT_KEPT", kind: "domain", cliExitCode: 12, httpStatus: 422 }, UNKNOWN_WORKER: { code: "UNKNOWN_WORKER", kind: "domain", cliExitCode: 12, httpStatus: 404 }, WORKER_UNREACHABLE: { code: "WORKER_UNREACHABLE", diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 322203b0..4e779631 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -41,6 +41,7 @@ const ROLE_MATRIX: ReadonlyArray<{ { name: "events.replay", input: {}, role: "admin" }, { name: "events.subscribe", input: {}, role: "admin" }, { name: "events.unsubscribe", input: {}, role: "admin" }, + { name: "usage.get", input: { from: 0, to: 60_000 }, role: "admin" }, { name: "token.create", input: { role: "agent" }, role: "admin" }, { name: "driver.passthrough", input: { args: ["devices"], tool: "adb" }, role: "agent" }, { @@ -112,6 +113,7 @@ const EFFECT_MATRIX: ReadonlyArray<{ { name: "events.replay", input: {}, effect: "read" }, { name: "events.subscribe", input: {}, effect: "read" }, { name: "events.unsubscribe", input: {}, effect: "read" }, + { name: "usage.get", input: { from: 0, to: 60_000 }, effect: "read" }, { name: "token.create", input: { role: "agent" }, effect: "write" }, { name: "driver.passthrough", input: { args: ["devices"], tool: "adb" }, effect: "read" }, { @@ -1248,3 +1250,94 @@ describe("lease.request names a model, a class, or nothing (ADR 0015 §1)", () = expect(requestedClass({ model: "iPhone 16" })).toBeUndefined(); }); }); + +describe("usage.get input", () => { + const parse = (input: unknown) => OPERATIONS["usage.get"].input.safeParse(input); + const DAY = 24 * 60 * 60 * 1000; + + it("takes epoch milliseconds with from before to, up to ninety days apart", () => { + expect(parse({ from: 0, to: 1 }).success).toBe(true); + expect(parse({ from: 1_000, to: 1_000 + 90 * DAY }).success).toBe(true); + expect(parse({ from: 1_000, to: 1_001 + 90 * DAY }).success).toBe(false); + expect(parse({ from: 5, to: 5 }).success).toBe(false); + expect(parse({ from: 6, to: 5 }).success).toBe(false); + }); + + it("refuses a bound that is not an integer, is negative, or no date can hold", () => { + expect(parse({ from: 0.5, to: 10 }).success).toBe(false); + expect(parse({ from: -1, to: 10 }).success).toBe(false); + expect(parse({ from: 0, to: 8.64e15 + 1 }).success).toBe(false); + expect(parse({ from: 8.64e15 - 1, to: 8.64e15 }).success).toBe(true); + expect(parse({ from: "0", to: 10 }).success).toBe(false); + expect(parse({ from: 0 }).success).toBe(false); + }); +}); + +describe("usage.get contract details", () => { + const issues = (input: unknown) => { + const result = OPERATIONS["usage.get"].input.safeParse(input); + return result.success ? [] : result.error.issues.map((issue) => issue.message); + }; + + it("usage.get names the bound it refused: from before to, and no more than ninety days", () => { + expect(issues({ from: 6, to: 5 })).toEqual(["from must be before to"]); + expect(issues({ from: 0, to: 91 * 24 * 60 * 60 * 1000 })).toEqual([ + "the window must not be longer than 90 days", + ]); + }); + + it("every operation carries the name it is registered under", () => { + for (const [key, operation] of Object.entries(OPERATIONS)) expect(operation.name).toBe(key); + }); + + it("usage.get output keeps the RAM figures of a scope and the RAM use of a series point", () => { + const samples = { count: 0, max: null, p50: null, p95: null }; + const figures = { + boot: samples, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, + failures: { byEvent: {} }, + granted: 0, + held: samples, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: samples, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: samples, + utilisation: { + ram: { limitBytes: 100, meanBytes: 40, peakBytes: 60 }, + slots: { max: 2, mean: 1, peak: 2 }, + }, + wait: samples, + }; + const point = { + at: 60_000, + queueDepth: 0, + ramUsedBytes: 60, + slotsMax: 2, + slotsUsed: 1, + waiting: 0, + }; + const output = { + bucketMs: 60_000, + coversFrom: 0, + partial: false, + platforms: { android: figures, ios: figures }, + requesters: [], + series: [point], + totals: figures, + window: { from: 0, to: 60_000 }, + workers: [], + }; + + const parsed = OPERATIONS["usage.get"].output.parse(output); + + expect(parsed.totals.utilisation.ram).toEqual({ + limitBytes: 100, + meanBytes: 40, + peakBytes: 60, + }); + expect(parsed.series).toEqual([point]); + }); +}); diff --git a/src/contract/operations.ts b/src/contract/operations.ts index a0108bd1..fb569f6b 100644 --- a/src/contract/operations.ts +++ b/src/contract/operations.ts @@ -38,6 +38,8 @@ import { statusLeaseSchema, tokenRecordSchema, tokenRoleSchema, + usageOutputSchema, + usageWindowSchema, waitingRequestSchema, workerViewSchema, } from "./schemas.js"; @@ -707,6 +709,22 @@ export const eventsReplay = defineOperation({ output: z.array(eventEnvelopeSchema), }); +// ---- usage.get ---------------------------------------------------------------------------- + +/** + * ADR 0016 §1: the figures for a window, computed from the event history when asked. Admin-only, + * as `events.replay` is: it reads the same history. A window that ends before the oldest held + * event is refused with `HISTORY_NOT_KEPT`. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const usageGet = defineOperation({ + name: "usage.get", + role: "admin", + effect: "read", + input: usageWindowSchema, + output: usageOutputSchema, +}); + // ---- events.subscribe --------------------------------------------------------------------- // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. @@ -1013,6 +1031,7 @@ export const OPERATIONS = { "events.replay": eventsReplay, "events.subscribe": eventsSubscribe, "events.unsubscribe": eventsUnsubscribe, + "usage.get": usageGet, "token.create": tokenCreate, "token.list": tokenList, "token.revoke": tokenRevoke, diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 182892b9..7d20bf29 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -1098,3 +1098,120 @@ export const statusLeaseSchema = leaseRecordSchema.extend({ /** Which worker holds this lease; absent on a worker's own answer, set by a gateway. */ workerId: z.string().optional(), }); + +// ---- usage.get (ADR 0016) ----------------------------------------------------------------- + +/** The widest window `usage.get` answers: ninety days, so the history one call reads is bounded. */ +const USAGE_MAX_WINDOW_MS = 90 * 24 * 60 * 60 * 1000; + +/** + * `usage.get`'s input. The window is wire input: both ends are epoch milliseconds a date can + * hold, `from` is before `to`, and the span is bounded before anything is read (safety rule 10). + */ +export const usageWindowSchema = z + .object({ + from: z.number().int().min(0).max(MAX_DATE_MS), + to: z.number().int().min(0).max(MAX_DATE_MS), + }) + .refine((window) => window.from < window.to, { message: "from must be before to" }) + .refine((window) => window.to - window.from <= USAGE_MAX_WINDOW_MS, { + message: "the window must not be longer than 90 days", + }); + +/** Milliseconds, `null` for a percentile over no samples. */ +const usageSamplesSchema = z.object({ + count: z.number().int().nonnegative(), + max: z.number().nullable(), + p50: z.number().nullable(), + p95: z.number().nullable(), +}); + +const usageCountsSchema = z.record(z.string(), z.number().int().nonnegative()); + +/** One scope's figures: the fleet's, a platform's, or a worker's. */ +const usageFiguresSchema = z.object({ + bySource: z.object({ + booted: z.number().int().nonnegative(), + provisioned: z.number().int().nonnegative(), + /** A gateway only: a grant whose worker's own `lease.granted` was not relayed (ADR 0021 §5). */ + unknown: z.number().int().nonnegative().optional(), + warm: z.number().int().nonnegative(), + }), + boot: usageSamplesSchema, + /** `lease.declined` events by their timestamp: a worker's own, or a gateway's relayed ones. Not + * requests (ADR 0021 §5). */ + declined: z.number().int().nonnegative(), + failures: z.object({ byEvent: usageCountsSchema }), + granted: z.number().int().nonnegative(), + held: usageSamplesSchema, + incidents: z.object({ + crashRecovered: z.number().int().nonnegative(), + lost: z.number().int().nonnegative(), + quarantineRecovered: z.number().int().nonnegative(), + quarantined: z.number().int().nonnegative(), + }), + /** A worker's own figure, absent from a gateway's answer: the gateway dispatches it received + * (`lease.requested` carrying `fleetRequestId`), which `requests` leaves out (ADR 0021 §5). */ + probes: z.number().int().nonnegative().optional(), + provisioning: usageSamplesSchema, + queue: z.object({ meanDepth: z.number().nullable(), peakDepth: z.number().nullable() }), + rejected: z.object({ + byReason: usageCountsSchema, + total: z.number().int().nonnegative(), + }), + requests: z.number().int().nonnegative(), + turnaround: usageSamplesSchema, + utilisation: z.object({ + /** Present only when `capacity.changed` carried a RAM budget in the window. */ + ram: z + .object({ + limitBytes: z.number(), + meanBytes: z.number(), + peakBytes: z.number(), + }) + .optional(), + slots: z.object({ + max: z.number().nullable(), + mean: z.number().nullable(), + peak: z.number().nullable(), + }), + }), + wait: usageSamplesSchema, +}); + +export const usageOutputSchema = z.object({ + /** The width of one series point, chosen by the daemon (ADR 0016 §8). */ + bucketMs: z.number().int().positive(), + /** The later of the window's start and the oldest timestamp the history holds (ADR 0016 §5). */ + coversFrom: z.number(), + /** True when `coversFrom` is later than the window's start: the history does not reach it. */ + partial: z.boolean(), + platforms: z.object({ android: usageFiguresSchema, ios: usageFiguresSchema }), + requesters: z.array( + z.object({ + granted: z.number().int().nonnegative(), + heldTotalMs: z.number().nonnegative(), + id: z.string(), + label: z.string().optional(), + rejected: z.number().int().nonnegative(), + requests: z.number().int().nonnegative(), + }), + ), + /** One point per bucket, each as it stands at the bucket's end; `null` where nothing is known. */ + series: z.array( + z.object({ + at: z.number(), + queueDepth: z.number().nullable(), + ramUsedBytes: z.number().optional(), + slotsMax: z.number().nullable(), + slotsUsed: z.number().nullable(), + waiting: z.number().int().nonnegative(), + }), + ), + totals: usageFiguresSchema, + window: z.object({ from: z.number(), to: z.number() }), + workers: z.array(usageFiguresSchema.extend({ id: z.string(), label: z.string().optional() })), +}); + +export type UsageFigures = z.infer; +export type UsageOutput = z.infer; diff --git a/src/core/index.ts b/src/core/index.ts index 9ec85617..eacb77c1 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -31,6 +31,7 @@ export { type CapacityReservation, plannedCapacityDevice, } from "./capacity/index.js"; +export { tokenLabelMap, UsageReader } from "./usage/index.js"; export { type CleanupRule, type RegistryView } from "./cleanup/types.js"; export { automaticCleanupRules } from "./cleanup/rules.js"; export { CleanupReaper } from "./reaper.js"; diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts new file mode 100644 index 00000000..d0c53b63 --- /dev/null +++ b/src/core/usage/compute-usage.test.ts @@ -0,0 +1,1327 @@ +import { describe, expect, it } from "vitest"; + +import type { EventEnvelope, EventMap, EventName } from "../../bus/index.js"; +import { computeUsage, seriesBucketMs, type UsageOptions } from "./compute-usage.js"; + +const SECOND = 1_000; +const MINUTE = 60 * SECOND; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; +const T0 = 1_000_000_000; +const WINDOW = { from: T0, to: T0 + HOUR }; +const WORKER: UsageOptions = { fleet: false, workers: [{ id: "self" }] }; +const FLEET: UsageOptions = { fleet: true }; + +let sequence = 0; + +/** One envelope; `extra` is merged into the payload, which is how a relayed event gets its `workerId`. */ +function at(timestamp: number, event: EventName, payload: Record): EventEnvelope { + sequence += 1; + return { + event, + id: `evt_${sequence}`, + module: "test", + payload: payload as unknown as EventMap[EventName], + seq: sequence, + timestamp, + } as EventEnvelope; +} + +const requested = (ts: number, requestId: string, requester = "agent-a", platform = "ios") => + at(ts, "lease.requested", { + requestId, + requestSpec: { platform }, + requester, + waitPolicy: "wait", + }); + +const granted = ( + ts: number, + requestId: string, + leaseId: string, + source = "warm", + requester = "agent-a", + extra: Record = {}, +) => + at(ts, "lease.granted", { + deviceId: `dev-${leaseId}`, + leaseId, + requestId, + requester, + source, + ...extra, + }); + +const released = (ts: number, leaseId: string, extra: Record = {}) => + at(ts, "lease.released", { + deviceId: `dev-${leaseId}`, + leaseId, + ownerId: "owner", + reason: "explicit", + ...extra, + }); + +const rejected = ( + ts: number, + requestId: string, + reason: string, + requester = "agent-a", + extra: Record = {}, +) => + at(ts, "lease.rejected", { + reason, + requestId, + requestSpec: { platform: "ios" }, + requester, + ...extra, + }); + +const figures = (running: number, reserved: number, maxRunning: number, warm = 0) => ({ + maxRunning, + reserved, + running, + warm, +}); + +const capacity = ( + ts: number, + global: ReturnType, + ramBudget?: { usedBytes: number; limitBytes: number }, + extra: Record = {}, +) => + at(ts, "capacity.changed", { + android: figures(0, 0, global.maxRunning), + global, + ios: global, + ...(ramBudget === undefined ? {} : { ramBudget }), + ...extra, + }); + +const queueChanged = (ts: number, depth: number, extra: Record = {}) => + at(ts, "queue.changed", { depth, ...extra }); + +/** The gateway's own record that it handed a fleet request's grant to its caller (ADR 0021 §4). */ +const handed = (ts: number, requestId: string, worker: string, workerLeaseId: string) => + at(ts, "request.granted", { leaseId: `gw-${workerLeaseId}`, requestId, worker, workerLeaseId }); + +/** A worker's refusal of a gateway's dispatch, as the worker records it (ADR 0021 §2). */ +const declined = (ts: number, extra: Record = {}) => + at(ts, "lease.declined", { + fleetRequestId: "f", + reason: "no-wait", + requestId: "worker-side", + requestSpec: { platform: "ios" }, + requester: "gw:g1:agent-a", + ...extra, + }); + +const restarted = (ts: number, extra: Record = {}) => + at(ts, "daemon.started", { configSnapshot: {}, version: "0", ...extra }); + +/** A worker's `lease.requested` for a gateway's dispatch: it carries the fleet request id. */ +const probe = (ts: number, requestId: string, requester = "gw:g1:a") => + at(ts, "lease.requested", { + fleetRequestId: `f-${requestId}`, + requestId, + requestSpec: { platform: "ios" }, + requester, + waitPolicy: "noWait", + }); + +describe("computeUsage", () => { + it("counts a request granted, held and released as one request, one grant, and one sample in wait, held and turnaround with the expected milliseconds", () => { + const usage = computeUsage( + [requested(T0 + 1_000, "r1"), granted(T0 + 3_000, "r1", "l1"), released(T0 + 13_000, "l1")], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + expect(usage.totals.held).toEqual({ count: 1, max: 10_000, p50: 10_000, p95: 10_000 }); + expect(usage.totals.turnaround).toEqual({ count: 1, max: 12_000, p50: 12_000, p95: 12_000 }); + expect(usage.platforms.ios.requests).toBe(1); + expect(usage.platforms.android.requests).toBe(0); + }); + + it("ends a lease's held time at its expiry as at its release", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1"), + granted(T0 + 2_000, "r1", "l1"), + at(T0 + 9_000, "lease.expired", { deviceId: "dev-l1", leaseId: "l1", ownerId: "owner" }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.held).toEqual({ count: 1, max: 7_000, p50: 7_000, p95: 7_000 }); + expect(usage.totals.turnaround.p50).toBe(8_000); + }); + + it("reports p50, p95 and max of a set of twenty known waits", () => { + const events: EventEnvelope[] = []; + for (let index = 1; index <= 20; index += 1) { + // Request `index` waits index * 100 ms: 100, 200, ... 2000. + events.push(requested(T0 + index * 10_000, `r${index}`, `agent-${index}`)); + events.push(granted(T0 + index * 10_000 + index * 100, `r${index}`, `l${index}`)); + } + + const { wait } = computeUsage(events, WINDOW, WORKER).totals; + + expect(wait).toEqual({ count: 20, max: 2_000, p50: 1_000, p95: 1_900 }); + }); + + it("reports a percentile over no samples as null", () => { + const { wait, held } = computeUsage([], WINDOW, WORKER).totals; + + expect(wait).toEqual({ count: 0, max: null, p50: null, p95: null }); + expect(held).toEqual({ count: 0, max: null, p50: null, p95: null }); + }); + + it("counts a rejected request under its reason and not under grants", () => { + const usage = computeUsage( + [requested(T0 + 1_000, "r1"), rejected(T0 + 6_000, "r1", "timeout")], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { timeout: 1 }, total: 1 }); + expect(usage.totals.wait.p50).toBe(5_000); + expect(usage.totals.turnaround.count).toBe(0); + }); + + it("attributes grants to warm, booted and provisioned from lease.granted.source", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "a"), + granted(T0 + 2_000, "r1", "l1", "warm", "a"), + requested(T0 + 1_000, "r2", "b"), + granted(T0 + 2_000, "r2", "l2", "booted", "b"), + requested(T0 + 1_000, "r3", "c"), + granted(T0 + 2_000, "r3", "l3", "provisioned", "c"), + requested(T0 + 1_000, "r4", "d"), + granted(T0 + 2_000, "r4", "l4", "provisioned", "d"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.granted).toBe(4); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 2, warm: 1 }); + }); + + it("reads provisioning and boot durations from device.provisioned and device.ready", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.provisioned", { + deviceId: "d1", + driver: "fake", + duration: 90_000, + spec: { platform: "ios" }, + }), + at(T0 + 2_000, "device.ready", { bootDuration: 20_000, deviceId: "d1" }), + at(T0 + 3_000, "device.provisioned", { + deviceId: "d2", + driver: "fake", + duration: 30_000, + spec: { platform: "android" }, + }), + at(T0 + 4_000, "device.ready", { bootDuration: 40_000, deviceId: "d2" }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.provisioning).toEqual({ count: 2, max: 90_000, p50: 30_000, p95: 90_000 }); + expect(usage.totals.boot).toEqual({ count: 2, max: 40_000, p50: 20_000, p95: 40_000 }); + expect(usage.platforms.ios.provisioning.p50).toBe(90_000); + expect(usage.platforms.android.boot.p50).toBe(40_000); + }); + + it("derives peak and time-weighted mean slot utilisation from capacity.changed steps, including the step in force before the window starts", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(1, 0, 4)), + capacity(T0 + 10 * MINUTE, figures(3, 1, 4)), + capacity(T0 + 30 * MINUTE, figures(0, 0, 4)), + ], + WINDOW, + WORKER, + ); + + // 1 slot for 10 minutes, 4 for 20, 0 for 30: (10 + 80 + 0) / 60. + expect(usage.totals.utilisation.slots).toEqual({ max: 4, mean: 1.5, peak: 4 }); + }); + + it("reports RAM utilisation when capacity.changed carries ramBudget and omits it otherwise", () => { + const withRam = computeUsage( + [ + capacity(T0 - 1_000, figures(0, 0, 4), { limitBytes: 1_000, usedBytes: 100 }), + capacity(T0 + 30 * MINUTE, figures(1, 0, 4), { limitBytes: 1_000, usedBytes: 500 }), + ], + WINDOW, + WORKER, + ); + const withoutRam = computeUsage([capacity(T0 - 1_000, figures(0, 0, 4))], WINDOW, WORKER); + + expect(withRam.totals.utilisation.ram).toEqual({ + limitBytes: 1_000, + meanBytes: 300, + peakBytes: 500, + }); + expect(withoutRam.totals.utilisation).not.toHaveProperty("ram"); + }); + + it("derives peak and mean queue depth from queue.changed", () => { + const usage = computeUsage( + [ + queueChanged(T0 - 1_000, 0), + queueChanged(T0 + 10 * MINUTE, 2), + queueChanged(T0 + 40 * MINUTE, 0), + ], + WINDOW, + WORKER, + ); + + // 0 for 10 minutes, 2 for 30, 0 for 20: 60 / 60. + expect(usage.totals.queue).toEqual({ meanDepth: 1, peakDepth: 2 }); + }); + + it("counts quarantined, crashRecovered, quarantineRecovered and lost from the named device events and nothing else", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.quarantined", { deviceId: "d1", maxRetries: 3, nextRetryAt: 0 }), + at(T0 + 2_000, "device.recovered", { + attempts: 1, + deviceId: "d2", + duration: 5, + leaseId: "l", + }), + at(T0 + 3_000, "device.quarantine-recovered", { + attempts: 1, + deviceId: "d1", + strategy: "erase", + }), + at(T0 + 4_000, "device.recovery-failed", { + attempts: 3, + deviceId: "d3", + error: "x", + leaseId: "l", + reason: "attempts-exhausted", + }), + at(T0 + 5_000, "device.quarantine-abandoned", { attempts: 3, deviceId: "d4" }), + at(T0 + 6_000, "device.quarantine-stranded", { attempts: 3, deviceId: "d5", error: "x" }), + // Facts that sit next to those and are not any of the four. + at(T0 + 7_000, "device.crash-detected", { + deviceId: "d2", + leaseId: "l", + observed: "stopped", + platform: "ios", + }), + at(T0 + 8_000, "device.deleted", { deviceId: "d4", initiator: "x" }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.incidents).toEqual({ + crashRecovered: 1, + lost: 3, + quarantineRecovered: 1, + quarantined: 1, + }); + }); + + it("counts a request whose lease.requested is before from and whose grant is inside the window under neither requests nor grants", () => { + const usage = computeUsage( + [ + requested(T0 - 5_000, "r1"), + granted(T0 + 1_000, "r1", "l1"), + released(T0 + 5_000, "l1"), + // A request inside the window, so the window is seen counting at all. + requested(T0 + 6_000, "r2", "agent-b"), + granted(T0 + 6_500, "r2", "l2", "booted", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, warm: 0 }); + expect(usage.totals.wait).toMatchObject({ count: 1, p50: 500 }); + expect(usage.totals.held.count).toBe(0); + expect(usage.requesters.map((requester) => requester.id)).toEqual(["agent-b"]); + }); + + it("gives no held or turnaround sample for a lease still open at to, and counts its request and grant", () => { + const usage = computeUsage( + [ + requested(WINDOW.to - 10_000, "r1"), + granted(WINDOW.to - 5_000, "r1", "l1"), + // The end falls after the window: it is not seen. + released(WINDOW.to + 5_000, "l1"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait.count).toBe(1); + expect(usage.totals.held.count).toBe(0); + expect(usage.totals.turnaround.count).toBe(0); + }); + + it("counts a rejection with no preceding lease.requested under rejected and not under requests", () => { + const usage = computeUsage( + [rejected(T0 + 1_000, "r-refused", "already-leased")], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { "already-leased": 1 }, total: 1 }); + expect(usage.totals.wait.count).toBe(0); + }); + + it("counts a killed rejection that names no request spec, with no preceding lease.requested, under rejected", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "lease.rejected", { + reason: "killed", + requestId: "r-killed", + requester: "agent-a", + }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { killed: 1 }, total: 1 }); + expect(usage.platforms.ios.rejected.total).toBe(0); + expect(usage.platforms.android.rejected.total).toBe(0); + }); + + it("counts a request from before from that is rejected inside the window under neither requests nor rejections", () => { + const usage = computeUsage( + [ + requested(T0 - 5_000, "r1"), + rejected(T0 + 1_000, "r1", "timeout"), + // A request inside the window, so the window is seen counting at all. + requested(T0 + 2_000, "r2", "agent-b"), + rejected(T0 + 3_000, "r2", "cancelled", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.rejected).toEqual({ byReason: { cancelled: 1 }, total: 1 }); + + // The history a window is read from holds nothing from before it, so a rejection whose + // request is not there is told from one refused at admission by its reason. + const unseen = computeUsage( + [ + rejected(T0 + 1_000, "r-old", "timeout"), + rejected(T0 + 2_000, "r-old-2", "boot-timeout"), + rejected(T0 + 3_000, "r-refused", "already-leased", "agent-b"), + ], + WINDOW, + WORKER, + ); + expect(unseen.totals.rejected).toEqual({ byReason: { "already-leased": 1 }, total: 1 }); + }); + + it("reports null series points and excludes them from peak and mean before the first step when no step precedes the window", () => { + const usage = computeUsage( + [capacity(T0 + 30 * MINUTE, figures(2, 0, 4)), queueChanged(T0 + 30 * MINUTE, 3)], + WINDOW, + WORKER, + ); + + expect(usage.series).toHaveLength(60); + // Point 28 ends at the 29th minute, point 29 at the 30th, where the first step lands. + expect(usage.series[28]).toMatchObject({ queueDepth: null, slotsMax: null, slotsUsed: null }); + expect(usage.series[29]).toMatchObject({ queueDepth: 3, slotsMax: 4, slotsUsed: 2 }); + expect(usage.totals.utilisation.slots).toEqual({ max: 4, mean: 2, peak: 2 }); + expect(usage.totals.queue).toEqual({ meanDepth: 3, peakDepth: 3 }); + }); + + it("reports slots, RAM, queue depth and waiting requests in each series point as they stand at the bucket's end", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(1, 0, 4), { limitBytes: 1_000, usedBytes: 250 }), + queueChanged(T0 - 1_000, 0), + requested(T0 + 90 * SECOND, "r1"), + queueChanged(T0 + 90 * SECOND, 1), + granted(T0 + 150 * SECOND, "r1", "l1"), + queueChanged(T0 + 150 * SECOND, 0), + ], + { from: T0, to: T0 + 5 * MINUTE }, + WORKER, + ); + + expect(usage.bucketMs).toBe(MINUTE); + expect(usage.series.map((point) => point.at)).toEqual( + [1, 2, 3, 4, 5].map((n) => T0 + n * MINUTE), + ); + expect(usage.series.map((point) => point.waiting)).toEqual([0, 1, 0, 0, 0]); + expect(usage.series.map((point) => point.queueDepth)).toEqual([0, 1, 0, 0, 0]); + expect(usage.series[0]).toMatchObject({ ramUsedBytes: 250, slotsMax: 4, slotsUsed: 1 }); + }); + + it("groups by platform and by workerId", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "a", "ios"), + granted(T0 + 2_000, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), + handed(T0 + 2_100, "r1", "w1", "l1"), + requested(T0 + 3_000, "r2", "b", "android"), + granted(T0 + 4_000, "w2-r", "l2", "provisioned", "gw:g1:b", { workerId: "w2" }), + handed(T0 + 4_100, "r2", "w2", "l2"), + at(T0 + 4_500, "device.provisioned", { + deviceId: "dev-l2", + driver: "fake", + duration: 7_000, + spec: { platform: "android" }, + workerId: "w2", + }), + ], + WINDOW, + { ...FLEET, workers: [{ id: "w1", label: "mac-1" }] }, + ); + + expect(usage.platforms.ios).toMatchObject({ granted: 1, requests: 1 }); + expect(usage.platforms.android).toMatchObject({ granted: 1, requests: 1 }); + expect(usage.platforms.android.provisioning.p50).toBe(7_000); + expect(usage.workers.map((worker) => [worker.id, worker.label, worker.granted])).toEqual([ + ["w1", "mac-1", 1], + ["w2", undefined, 1], + ]); + expect(usage.workers[1]?.provisioning.p50).toBe(7_000); + expect(usage.workers[1]).not.toHaveProperty("label"); + expect(usage.workers[0]?.provisioning.count).toBe(0); + expect(usage.totals.granted).toBe(2); + }); + + it("with fleet: true counts a fleet request once, from the gateway's own events, and attributes its grant to the worker of its request.granted", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + // The worker's own copy of the same request, relayed from a worker the grant is not on. + at(T0 + 1_300, "lease.requested", { + fleetRequestId: "gw-r1", + requestId: "w-r1", + requestSpec: { platform: "ios" }, + requester: "gw:g1:agent-a", + waitPolicy: "noWait", + workerId: "w9", + }), + // A grant on that other worker under the same lease id: not the one the request got. + granted(T0 + 3_900, "w9-r", "l1", "warm", "gw:g1:agent-b", { workerId: "w9" }), + granted(T0 + 4_000, "w-r1", "l1", "booted", "gw:g1:agent-a", { workerId: "w1" }), + handed(T0 + 4_200, "gw-r1", "w1", "l1"), + released(T0 + 9_000, "l1", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, unknown: 0, warm: 0 }); + expect(usage.workers).toHaveLength(1); + expect(usage.workers[0]).toMatchObject({ granted: 1, id: "w1", requests: 1 }); + }); + + it("with fleet: true ignores relayed lease.requested, lease.queued, queue.changed, lease.rejected and lease.declined as request facts", () => { + const usage = computeUsage( + [ + queueChanged(T0 + 1_000, 1), + at(T0 + 1_500, "queue.changed", { depth: 9, workerId: "w1" }), + at(T0 + 2_000, "lease.requested", { + requestId: "w-r1", + requestSpec: { platform: "ios" }, + requester: "gw:g1:ghost", + waitPolicy: "noWait", + workerId: "w1", + }), + at(T0 + 2_100, "lease.queued", { queuePosition: 1, requestId: "w-r1", workerId: "w1" }), + rejected(T0 + 2_200, "w-r1", "no-wait", "gw:g1:ghost", { workerId: "w1" }), + declined(T0 + 2_300, { requestId: "w-r1", workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: {}, total: 0 }); + expect(usage.totals.queue.peakDepth).toBe(1); + expect(usage.requesters).toEqual([]); + }); + + it("with fleet: true does not count a relayed lease.granted carrying a request's fleetRequestId as its outcome when the gateway rejected it", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + granted(T0 + 1_500, "w-r1", "l1", "warm", "gw:g1:agent-a", { + fleetRequestId: "gw-r1", + workerId: "w1", + }), + // The grant came too late: the gateway had settled the request as a timeout. + rejected(T0 + 1_200, "gw-r1", "timeout", "agent-a"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.granted).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { timeout: 1 }, total: 1 }); + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 0, unknown: 0, warm: 0 }); + }); + + it("with fleet: true does not let a stale worker's relayed refusal reject a request another worker granted", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + rejected(T0 + 1_100, "w1-r", "no-wait", "gw:g1:agent-a", { workerId: "w1" }), + granted(T0 + 1_400, "w2-r", "l1", "warm", "gw:g1:agent-a", { workerId: "w2" }), + handed(T0 + 1_500, "gw-r1", "w2", "l1"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.granted).toBe(1); + expect(usage.totals.rejected.total).toBe(0); + expect( + usage.workers.map((worker) => [worker.id, worker.granted, worker.rejected.total]), + ).toEqual([["w2", 1, 0]]); + }); + + it("with fleet: true does not close an open fleet request at a relayed worker daemon.started", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + at(T0 + 2_000, "daemon.started", { configSnapshot: {}, version: "0", workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.rejected.total).toBe(0); + expect(usage.totals.wait.count).toBe(0); + expect(usage.series.at(-1)?.waiting).toBe(1); + }); + + it("with fleet: true settles a request as rejected from the gateway's own lease.rejected, and a worker-failed one under its worker", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1", "a"), + rejected(T0 + 2_000, "gw-r1", "timeout", "a"), + requested(T0 + 3_000, "gw-r2", "b"), + rejected(T0 + 3_500, "gw-r2", "worker-failed", "b", { code: "INTERNAL", worker: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(2); + expect(usage.totals.granted).toBe(0); + expect(usage.totals.rejected).toEqual({ + byReason: { timeout: 1, "worker-failed": 1 }, + total: 2, + }); + expect(usage.workers).toHaveLength(1); + expect(usage.workers[0]).toMatchObject({ + id: "w1", + rejected: { byReason: { "worker-failed": 1 }, total: 1 }, + requests: 1, + wait: { count: 1, p50: 500 }, + }); + }); + + it("with fleet: true counts a fleet request under the worker its request.granted or worker-failed rejection names, with its wait and turnaround, and a timeout rejection under no worker", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "g1", "a"), + granted(T0 + 1_800, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), + handed(T0 + 2_000, "g1", "w1", "l1"), + released(T0 + 7_800, "l1", { workerId: "w1" }), + requested(T0 + 3_000, "g2", "b"), + rejected(T0 + 3_400, "g2", "worker-failed", "b", { + code: "WORKER_UNREACHABLE", + worker: "w2", + }), + requested(T0 + 4_000, "g3", "c"), + rejected(T0 + 9_000, "g3", "timeout", "c"), + ], + WINDOW, + FLEET, + ); + + expect(usage.workers.map((worker) => worker.id)).toEqual(["w1", "w2"]); + expect(usage.workers[0]).toMatchObject({ + granted: 1, + requests: 1, + turnaround: { count: 1, p50: 7_000 }, + wait: { count: 1, p50: 1_000 }, + }); + expect(usage.workers[1]).toMatchObject({ + granted: 0, + rejected: { total: 1 }, + requests: 1, + wait: { count: 1, p50: 400 }, + }); + expect(usage.totals).toMatchObject({ + granted: 1, + rejected: { byReason: { timeout: 1, "worker-failed": 1 }, total: 2 }, + requests: 3, + wait: { count: 3 }, + }); + expect(usage.workers.reduce((total, worker) => total + worker.requests, 0)).toBe(2); + }); + + it("with fleet: true measures a fleet wait from the gateway's lease.requested to its request.granted, ignoring a relayed lease.granted stamped earlier", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + granted(T0 + 800, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), + handed(T0 + 3_000, "gw-r1", "w1", "l1"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + }); + + it("with fleet: true takes the grant source and held time from the relayed lease.granted whose workerId and leaseId match the request.granted's worker and workerLeaseId, and counts source unknown with no held sample when it is missing", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "g1", "a"), + // The same lease id on w1 and w2, and other leases on w2: only (w2, l1) is the one g1 names. + granted(T0 + 1_050, "w2-r0", "l0", "warm", "gw:g1:z", { workerId: "w2" }), + granted(T0 + 1_100, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), + granted(T0 + 1_200, "w2-r", "l1", "provisioned", "gw:g1:a", { workerId: "w2" }), + granted(T0 + 1_300, "w2-r2", "l2", "booted", "gw:g1:y", { workerId: "w2" }), + handed(T0 + 1_500, "g1", "w2", "l1"), + released(T0 + 4_000, "l0", { workerId: "w2" }), + released(T0 + 6_200, "l1", { workerId: "w2" }), + released(T0 + 7_300, "l2", { workerId: "w2" }), + released(T0 + 9_000, "l1", { workerId: "w1" }), + requested(T0 + 2_000, "g2", "b"), + handed(T0 + 2_400, "g2", "w3", "l9"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 1, unknown: 1, warm: 0 }); + expect(usage.totals.granted).toBe(2); + expect(usage.totals.held).toEqual({ count: 1, max: 5_000, p50: 5_000, p95: 5_000 }); + expect(usage.totals.turnaround).toEqual({ count: 1, max: 5_500, p50: 5_500, p95: 5_500 }); + expect(usage.totals.wait.count).toBe(2); + expect(usage.requesters.map((requester) => [requester.id, requester.heldTotalMs])).toEqual([ + ["a", 5_000], + ["b", 0], + ]); + }); + + it("gives a lease id chosen again after its release the end of its own lease on a worker, not the later one's", () => { + const usage = computeUsage( + [ + requested(T0 + 100_000, "r1"), + granted(T0 + 120_000, "r1", "ci-1"), + released(T0 + 130_000, "ci-1"), + requested(T0 + 140_000, "r2"), + granted(T0 + 150_000, "r2", "ci-1"), + released(T0 + 190_000, "ci-1"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.held).toEqual({ count: 2, max: 40_000, p50: 10_000, p95: 40_000 }); + }); + + it("with fleet: true gives two fleet requests granted on one worker under one lease id each their own source and held time", () => { + const usage = computeUsage( + [ + requested(T0 + 100_000, "g1", "a"), + granted(T0 + 120_000, "w-r1", "ci-1", "warm", "gw:g1:a", { workerId: "w1" }), + handed(T0 + 120_100, "g1", "w1", "ci-1"), + released(T0 + 130_000, "ci-1", { workerId: "w1" }), + requested(T0 + 140_000, "g2", "b"), + granted(T0 + 150_000, "w-r2", "ci-1", "booted", "gw:g1:b", { workerId: "w1" }), + handed(T0 + 150_100, "g2", "w1", "ci-1"), + released(T0 + 190_000, "ci-1", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, unknown: 0, warm: 1 }); + expect(usage.totals.held).toEqual({ count: 2, max: 40_000, p50: 10_000, p95: 40_000 }); + expect(usage.requesters.map((requester) => [requester.id, requester.heldTotalMs])).toEqual([ + ["a", 10_000], + ["b", 40_000], + ]); + }); + + it("with fleet: true counts each relayed lease.declined under its worker's and platform's declined, two declines of one request as two", () => { + const usage = computeUsage( + [ + declined(T0 + 1_000, { workerId: "w1" }), + declined(T0 + 2_000, { workerId: "w1" }), + declined(T0 + 3_000, { requestSpec: { platform: "android" }, workerId: "w2" }), + declined(T0 - 1_000, { workerId: "w1" }), + // The gateway's own events have no decline of this kind: nothing counts for no worker. + declined(T0 + 4_000), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.declined).toBe(3); + expect(usage.platforms.ios.declined).toBe(2); + expect(usage.platforms.android.declined).toBe(1); + expect(usage.workers.map((worker) => [worker.id, worker.declined])).toEqual([ + ["w1", 2], + ["w2", 1], + ]); + expect(usage.totals.requests).toBe(0); + }); + + it("with fleet: true ends a request with no outcome at the gateway's next daemon.started as rejected daemon-restarted, and leaves one with no later start open", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gone", "a"), + restarted(T0 + 5_000), + requested(T0 + 6_000, "open", "b"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(2); + expect(usage.totals.rejected).toEqual({ byReason: { "daemon-restarted": 1 }, total: 1 }); + expect(usage.totals.wait).toEqual({ count: 1, max: 4_000, p50: 4_000, p95: 4_000 }); + expect(usage.workers).toEqual([]); + expect(usage.series.slice(0, 7).map((point) => point.waiting)).toEqual([1, 1, 1, 1, 1, 1, 1]); + expect(usage.series.at(-1)?.waiting).toBe(1); + }); + + it("with fleet: true answers no probes field in totals, platforms or worker entries", () => { + const usage = computeUsage( + [capacity(T0 + 1_000, figures(0, 0, 2), undefined, { workerId: "w1" })], + WINDOW, + FLEET, + ); + + expect(usage.totals).not.toHaveProperty("probes"); + expect(usage.platforms.ios).not.toHaveProperty("probes"); + expect(usage.workers[0]).not.toHaveProperty("probes"); + expect(usage.totals.bySource).toHaveProperty("unknown", 0); + }); + + it("on a worker counts a lease.requested carrying fleetRequestId under probes, not requests, its grant under granted, its lease.declined under declined by the decline's timestamp, and gives it no wait sample", () => { + const usage = computeUsage( + [ + probe(T0 + 1_000, "p1"), + granted(T0 + 2_000, "p1", "l1", "booted", "gw:g1:a", { fleetRequestId: "f-p1" }), + released(T0 + 8_000, "l1"), + probe(T0 + 3_000, "p2"), + declined(T0 + 3_100, { requestId: "p2" }), + // A decline of a probe made before the window counts by its own timestamp. + declined(T0 + 4_000, { requestId: "old-probe" }), + requested(T0 + 5_000, "local", "agent-b"), + granted(T0 + 5_500, "local", "l2", "warm", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals).toMatchObject({ + declined: 2, + granted: 2, + probes: 2, + rejected: { total: 0 }, + requests: 1, + }); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, warm: 1 }); + expect(usage.totals.bySource).not.toHaveProperty("unknown"); + expect(usage.totals.wait).toEqual({ count: 1, max: 500, p50: 500, p95: 500 }); + expect(usage.totals.held).toMatchObject({ count: 1, p50: 6_000 }); + expect(usage.workers[0]).toMatchObject({ declined: 2, probes: 2, requests: 1 }); + expect(usage.platforms.ios).toMatchObject({ declined: 2, probes: 2 }); + }); + + it("on a worker gives a probe no turnaround sample and lists its gw: requester with its grant and held time and no request", () => { + const usage = computeUsage( + [ + probe(T0 + 1_000, "p1"), + granted(T0 + 2_000, "p1", "l1", "warm", "gw:g1:a", { fleetRequestId: "f-p1" }), + released(T0 + 8_000, "l1"), + // A probe that was declined is nothing of its requester's. + probe(T0 + 9_000, "p2", "gw:g1:b"), + declined(T0 + 9_100, { requestId: "p2" }), + requested(T0 + 10_000, "local", "agent-b"), + granted(T0 + 10_400, "local", "l2", "warm", "agent-b"), + released(T0 + 12_400, "l2"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.turnaround).toEqual({ count: 1, max: 2_400, p50: 2_400, p95: 2_400 }); + expect(usage.requesters).toEqual([ + { granted: 1, heldTotalMs: 2_000, id: "agent-b", rejected: 0, requests: 1 }, + { granted: 1, heldTotalMs: 6_000, id: "gw:g1:a", rejected: 0, requests: 0 }, + ]); + }); + + it("on a worker counts no rejection for a probe, which a worker declines and never rejects", () => { + const usage = computeUsage( + [probe(T0 + 1_000, "p1"), rejected(T0 + 1_500, "p1", "no-wait", "gw:g1:a")], + WINDOW, + WORKER, + ); + + expect(usage.totals).toMatchObject({ probes: 1, rejected: { byReason: {}, total: 0 } }); + expect(usage.requesters).toEqual([]); + }); + + it("lists workers by id whichever order the events name them in", () => { + const usage = computeUsage( + [ + capacity(T0 + 1_000, figures(0, 0, 2), undefined, { workerId: "w2" }), + capacity(T0 + 2_000, figures(0, 0, 2), undefined, { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.workers.map((worker) => worker.id)).toEqual(["w1", "w2"]); + }); + + it("on a worker leaves a probe between its lease.requested and its grant out of series[].waiting", () => { + const usage = computeUsage( + [ + // A probe made before the window and still open at its start is not waiting either. + probe(T0 - 30_000, "old"), + probe(T0 + 10_000, "p1"), + granted(T0 + 100_000, "p1", "l1", "warm", "gw:g1:a", { fleetRequestId: "f-p1" }), + requested(T0 + 20_000, "local", "agent-b"), + granted(T0 + 130_000, "local", "l2", "warm", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 1, 0]); + }); + + it("sets partial and coversFrom when the oldest event is inside the window", () => { + const partial = computeUsage( + [requested(T0 + 5 * MINUTE, "r1"), requested(T0 + 9 * MINUTE, "r2", "b")], + WINDOW, + WORKER, + ); + const whole = computeUsage( + [capacity(T0 - 1_000, figures(0, 0, 2)), requested(T0 + 5 * MINUTE, "r1")], + WINDOW, + WORKER, + ); + const stated = computeUsage([requested(T0 + 5 * MINUTE, "r1")], WINDOW, { + ...WORKER, + oldestTs: T0 - DAY, + }); + const empty = computeUsage([], WINDOW, WORKER); + + expect(partial).toMatchObject({ coversFrom: T0 + 5 * MINUTE, partial: true }); + expect(whole).toMatchObject({ coversFrom: T0, partial: false }); + expect(stated).toMatchObject({ coversFrom: T0, partial: false }); + expect(empty).toMatchObject({ coversFrom: T0, partial: false }); + }); + + it("picks 1-minute buckets for one hour, 15-minute buckets for one day and 1-hour buckets for seven days, and never more than 200 points for a 90-day window", () => { + expect(seriesBucketMs(HOUR)).toBe(MINUTE); + expect(seriesBucketMs(DAY)).toBe(15 * MINUTE); + expect(seriesBucketMs(7 * DAY)).toBe(HOUR); + + const ninety = computeUsage([], { from: T0, to: T0 + 90 * DAY }, WORKER); + expect(ninety.bucketMs).toBe(DAY); + expect(ninety.series).toHaveLength(90); + // The edges: 200 minutes is the last window one minute serves, 201 moves to five. + expect(seriesBucketMs(200 * MINUTE)).toBe(MINUTE); + expect(seriesBucketMs(200 * MINUTE + 1)).toBe(5 * MINUTE); + expect(seriesBucketMs(1_000 * MINUTE)).toBe(5 * MINUTE); + expect(seriesBucketMs(1_000 * MINUTE + 1)).toBe(15 * MINUTE); + expect(seriesBucketMs(3_000 * MINUTE)).toBe(15 * MINUTE); + expect(seriesBucketMs(3_000 * MINUTE + 1)).toBe(HOUR); + expect(seriesBucketMs(200 * HOUR)).toBe(HOUR); + expect(seriesBucketMs(200 * HOUR + 1)).toBe(6 * HOUR); + expect(seriesBucketMs(1_200 * HOUR)).toBe(6 * HOUR); + expect(seriesBucketMs(1_200 * HOUR + 1)).toBe(DAY); + }); + + it("lists requesters by request count with the label the caller supplied", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "tok_b"), + granted(T0 + 2_000, "r1", "l1", "warm", "tok_b"), + released(T0 + 7_000, "l1"), + requested(T0 + 3_000, "r2", "tok_a"), + requested(T0 + 4_000, "r3", "tok_a"), + rejected(T0 + 5_000, "r3", "timeout", "tok_a"), + requested(T0 + 6_000, "r4", "tok_a"), + ], + WINDOW, + { ...WORKER, labels: { tok_b: "ci-bot" } }, + ); + + expect(usage.requesters).toStrictEqual([ + { granted: 0, heldTotalMs: 0, id: "tok_a", rejected: 1, requests: 3 }, + { granted: 1, heldTotalMs: 5_000, id: "tok_b", label: "ci-bot", rejected: 0, requests: 1 }, + ]); + }); + + it("gives a worker one entry for itself, named by the workerId it is given", () => { + const usage = computeUsage( + [requested(T0 + 1_000, "r1"), granted(T0 + 2_000, "r1", "l1")], + WINDOW, + { fleet: false, workers: [{ id: "wrk_me", label: "my-mac" }] }, + ); + + expect(usage.workers).toHaveLength(1); + expect(usage.workers[0]).toMatchObject({ + granted: 1, + id: "wrk_me", + label: "my-mac", + requests: 1, + }); + }); + + it("counts every `*-failed` event in the window by its event name under failures.byEvent, by the event's timestamp", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.purge-failed", { + attemptedStrategy: "erase", + deviceId: "d1", + duration: 1, + error: "x", + leaseId: "l", + }), + at(T0 + 2_000, "device.purge-failed", { + attemptedStrategy: "erase", + deviceId: "d2", + duration: 1, + error: "x", + leaseId: "l", + }), + at(T0 + 3_000, "component.install-failed", { + componentId: "26.4", + durationMs: 1, + error: "x", + platform: "ios", + }), + at(T0 + 4_000, "device.recovery-failed", { deviceId: "d3", leaseId: "l" }), + at(T0 + 5_000, "device.some-new-failed" as EventName, { deviceId: "d3" }), + at(WINDOW.from - 1, "device.purge-failed", { deviceId: "d9" }), + at(WINDOW.to + 1, "device.purge-failed", { deviceId: "d9" }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.failures.byEvent).toEqual({ + "component.install-failed": 1, + "device.purge-failed": 2, + "device.recovery-failed": 1, + "device.some-new-failed": 1, + }); + expect(usage.totals.incidents.lost).toBe(1); + }); + + it("sorts events supplied out of order and leaves the caller's array exactly as it was", () => { + const events = [ + released(T0 + 13_000, "l1"), + granted(T0 + 3_000, "r1", "l1"), + requested(T0 + 1_000, "r1"), + ]; + const before = events.map((event) => event.id); + + const usage = computeUsage(events, WINDOW, WORKER); + + expect(events.map((event) => event.id)).toEqual(before); + expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + expect(usage.totals.held).toEqual({ count: 1, max: 10_000, p50: 10_000, p95: 10_000 }); + // The oldest event is the earliest one by time, not the first one given. + expect(usage.coversFrom).toBe(T0 + 1_000); + }); + + it("names the one worker `local` when it is given no workers", () => { + const usage = computeUsage([capacity(T0 + MINUTE, figures(1, 0, 2))], WINDOW, { + fleet: false, + }); + + expect(usage.workers.map((worker) => worker.id)).toEqual(["local"]); + }); + + it("counts a grant whose source is none of warm, booted and provisioned under no source", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1"), + granted(T0 + 2_000, "r1", "l1", "teleported"), + requested(T0 + 3_000, "r2"), + granted(T0 + 4_000, "r2", "l2", "warm"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.granted).toBe(2); + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 0, warm: 1 }); + }); + + it("keeps durations and incidents out of failures.byEvent when device events of every kind are present", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.provisioned", { + deviceId: "d1", + driver: "fake", + duration: 90_000, + spec: { platform: "ios" }, + }), + at(T0 + 2_000, "device.ready", { bootDuration: 20_000, deviceId: "d1" }), + at(T0 + 3_000, "device.quarantined", { deviceId: "d1", reason: "x" }), + at(T0 + 4_000, "device.purge-failed", { + attemptedStrategy: "erase", + deviceId: "d1", + duration: 1, + error: "x", + leaseId: "l", + }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.failures.byEvent).toEqual({ "device.purge-failed": 1 }); + expect(usage.totals.incidents.quarantined).toBe(1); + expect(usage.totals.boot.count).toBe(1); + expect(usage.totals.provisioning.count).toBe(1); + }); + + it("gives each worker only its own capacity steps and each platform only its own entry in them", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(1, 0, 2), undefined, { + android: figures(0, 0, 1), + ios: figures(1, 0, 2), + workerId: "w1", + }), + capacity(T0 - 1_000, figures(3, 0, 6), undefined, { + android: figures(3, 0, 5), + ios: figures(0, 0, 1), + workerId: "w2", + }), + ], + WINDOW, + FLEET, + ); + const byId = Object.fromEntries(usage.workers.map((worker) => [worker.id, worker])); + + expect(byId.w1?.utilisation.slots).toEqual({ max: 2, mean: 1, peak: 1 }); + expect(byId.w2?.utilisation.slots).toEqual({ max: 6, mean: 3, peak: 3 }); + expect(usage.totals.utilisation.slots).toEqual({ max: 8, mean: 4, peak: 4 }); + expect(usage.platforms.ios.utilisation.slots).toEqual({ max: 3, mean: 1, peak: 1 }); + expect(usage.platforms.android.utilisation.slots).toEqual({ max: 6, mean: 3, peak: 3 }); + }); + + it("reports slots.max as the highest total across the window, from stretches that last, and null when no step is known", () => { + const usage = computeUsage( + [ + capacity(T0 + 10 * MINUTE, figures(0, 0, 2)), + capacity(T0 + 20 * MINUTE, figures(0, 0, 9)), + // Two steps at one instant: the 9 holds for no time at all. + capacity(T0 + 30 * MINUTE, figures(0, 0, 4)), + capacity(T0 + 30 * MINUTE, figures(0, 0, 4)), + capacity(T0 + 30 * MINUTE, figures(0, 0, 3)), + ], + { from: T0, to: T0 + 40 * MINUTE }, + WORKER, + ); + const none = computeUsage([], WINDOW, WORKER); + + expect(usage.totals.utilisation.slots.max).toBe(9); + expect(none.totals.utilisation.slots).toEqual({ max: null, mean: null, peak: null }); + }); + + it("does not let a zero-length stretch between two steps at one instant set slots.max", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(0, 0, 2)), + capacity(T0 + 10 * MINUTE, figures(0, 0, 50)), + capacity(T0 + 10 * MINUTE, figures(0, 0, 3)), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.utilisation.slots.max).toBe(3); + }); + + it("orders requesters by requests descending, then by id ascending whichever order they arrive in", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "b"), + requested(T0 + 2_000, "r2", "a"), + requested(T0 + 3_000, "r3", "c"), + requested(T0 + 4_000, "r4", "c"), + requested(T0 + 5_000, "r5", "d"), + requested(T0 + 6_000, "r6", "d"), + requested(T0 + 7_000, "r7", "e"), + ], + WINDOW, + WORKER, + ); + + expect(usage.requesters.map((entry) => [entry.id, entry.requests])).toEqual([ + ["c", 2], + ["d", 2], + ["a", 1], + ["b", 1], + ["e", 1], + ]); + }); + + it("counts a rejection refused before admission under rejected for its requester and not under requests", () => { + const usage = computeUsage( + [ + rejected(T0 + 1_000, "r1", "already-leased", "gw-tok"), + requested(T0 + 2_000, "r2", "gw-tok"), + ], + WINDOW, + { ...WORKER, labels: { "gw-tok": "ci" } }, + ); + + expect(usage.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: "gw-tok", label: "ci", rejected: 1, requests: 1 }, + ]); + expect(usage.totals.requests).toBe(1); + expect(usage.totals.rejected).toEqual({ byReason: { "already-leased": 1 }, total: 1 }); + }); + + it("carries queue depth on a worker's own row and none on a gateway's worker rows", () => { + const events = [ + queueChanged(T0 - 1_000, 2), + capacity(T0 - 1_000, figures(0, 0, 2), undefined, { workerId: "w1" }), + ]; + + const worker = computeUsage(events, WINDOW, WORKER); + const fleet = computeUsage(events, WINDOW, { ...FLEET, workers: [{ id: "w1" }] }); + + expect(worker.workers[0]?.queue).toEqual({ meanDepth: 2, peakDepth: 2 }); + expect(fleet.workers[0]?.queue).toEqual({ meanDepth: null, peakDepth: null }); + }); + + it("counts a request made before the window and still waiting at its start in the series' waiting, though in no total", () => { + const usage = computeUsage( + [ + requested(T0 - 30_000, "old", "agent-a"), + // Answered before the window opened (the history says so): not waiting at its start. + requested(T0 - 40_000, "done", "agent-b"), + queueChanged(T0 - 30_000, 1), + granted(T0 + 90_000, "old", "l1"), + ], + WINDOW, + { ...WORKER, answeredBefore: new Set(["done"]) }, + ); + + expect(usage.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 0, 0]); + expect(usage.series[0]?.queueDepth).toBe(1); + expect(usage.totals.requests).toBe(0); + expect(usage.totals.granted).toBe(0); + }); + + it("with fleet: true counts a request made before the window until the gateway's request.granted for it", () => { + const usage = computeUsage( + [ + requested(T0 - 30_000, "gw-old", "agent-a"), + // A relayed grant for the same requester is not the gateway's answer. + granted(T0 + 30_000, "w-0", "l0", "warm", "gw:g1:agent-a", { workerId: "w1" }), + handed(T0 + 90_000, "gw-old", "w1", "l1"), + ], + WINDOW, + FLEET, + ); + + expect(usage.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 0, 0]); + expect(usage.totals.granted).toBe(0); + }); + + it("with fleet: true counts a request made before the window as waiting at its start unless the history says it was answered by then", () => { + const events = [ + requested(T0 - 30_000, "gw-waits", "agent-a"), + requested(T0 - 20_000, "gw-done", "agent-b"), + ]; + + const usage = computeUsage(events, WINDOW, { ...FLEET, answeredBefore: new Set(["gw-done"]) }); + + expect(usage.series[0]?.waiting).toBe(1); + }); + + it("with fleet: true stops counting a request the gateway lost at a stop as waiting from the start that followed it, whether before the window or inside it", () => { + const before = computeUsage( + [requested(T0 - 60_000, "gw-lost"), restarted(T0 - 30_000)], + WINDOW, + FLEET, + ); + const inside = computeUsage( + [requested(T0 - 60_000, "gw-lost"), restarted(T0 + 90_000)], + WINDOW, + FLEET, + ); + + expect(before.series.slice(0, 3).map((point) => point.waiting)).toEqual([0, 0, 0]); + expect(inside.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 0, 0]); + expect(inside.totals.rejected.total).toBe(0); + }); + + it("counts a request as waiting from the instant it is made until the instant it settles, per series point", () => { + const usage = computeUsage( + [ + // Refused before admission: no request, so it never waits. + rejected(T0 + 10_000, "x", "killed", "z"), + // Made first of all, settled last. + requested(T0 + 20_000, "c", "c"), + // Made, never settled. + requested(T0 + 30_000, "open", "o"), + // Made on a bucket end, settled on a bucket end. + requested(T0 + MINUTE, "a", "a"), + granted(T0 + 2 * MINUTE, "a", "la", "warm", "a"), + // Made after a, settled before it. + requested(T0 + 70_000, "b", "b"), + granted(T0 + 100_000, "b", "lb", "warm", "b"), + rejected(T0 + 5 * MINUTE + 30_000, "c", "timeout", "c"), + ], + WINDOW, + WORKER, + ); + const waiting = usage.series.slice(0, 7).map((point) => point.waiting); + + // Ends at 1m, 2m, ... 7m: c, open and a (made at exactly 1m) wait at 1m; b is made, and b and + // a settle (a at exactly 2m), by 2m; c settles at 5m30s. + expect(waiting).toEqual([3, 2, 2, 2, 2, 1, 1]); + expect(usage.series.at(-1)?.waiting).toBe(1); + }); +}); diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts new file mode 100644 index 00000000..3dc7394f --- /dev/null +++ b/src/core/usage/compute-usage.ts @@ -0,0 +1,357 @@ +import { byTimeThenSeq, type EventEnvelope } from "../../bus/index.js"; +import type { UsageFigures, UsageOutput } from "../../contract/index.js"; +import { + type CapacityStep, + type CapacityValue, + type DeviceFact, + type Platform, + type ReadEvents, + readEvents, + type RequestFact, + type UsageWindow, +} from "./read-events.js"; +import { peakAndMean, segmentsOf, type Step, valuesAt } from "./timeline.js"; + +export type { UsageWindow } from "./read-events.js"; + +export interface UsageOptions { + /** The series bucket, picked once from the window asked for; the window's own span when absent. */ + readonly bucketMs?: number; + /** The `requestId` of every request made before the window; a rejection of one is in no count. */ + readonly requestedBefore?: ReadonlySet; + /** The `requestId` of every request answered, granted or rejected, before the window; a request + * made before it and not in this set was still waiting when it began. */ + readonly answeredBefore?: ReadonlySet; + /** The gateway rule (ADR 0016 §6 as ADR 0021 §5 amends it): request facts from the gateway's + * own events, device facts from the events its workers relayed. */ + readonly fleet: boolean; + /** Requester id to label, for the requesters the caller found a label for (ADR 0016 §7). */ + readonly labels?: Readonly>; + /** The workers there are to report: a worker's own entry, a gateway's registry. A worker the + * events name and this list does not is reported too. */ + readonly workers?: readonly { readonly id: string; readonly label?: string | undefined }[]; + /** The oldest timestamp the history holds, when the events passed start at the window's start + * and so cannot say. */ + readonly oldestTs?: number | undefined; +} + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +/** The widths a series may have, narrowest first (ADR 0016 §8). */ +const BUCKET_WIDTHS_MS = [MINUTE, 5 * MINUTE, 15 * MINUTE, HOUR, 6 * HOUR, 24 * HOUR]; +const MAX_SERIES_POINTS = 200; + +/** The series bucket width for a window of `spanMs`: the narrowest that keeps 200 points. */ +export function seriesBucketMs(spanMs: number): number { + return ( + BUCKET_WIDTHS_MS.find((width) => Math.ceil(spanMs / width) <= MAX_SERIES_POINTS) ?? + (BUCKET_WIDTHS_MS.at(-1) as number) + ); +} + +/** + * The usage figures for `window`, from the events of a history (ADR 0016). Pure: events in, numbers + * out. `events` are the envelopes the history holds for the window, with the latest envelope at or + * before its start of `capacity.changed` and `queue.changed` (the step in force), of + * `lease.requested` for each requester (a request still waiting when the window began) and, on a + * gateway, the latest of its own `daemon.started`; any order, and any envelope outside the window + * is left out. + */ +export function computeUsage( + events: readonly EventEnvelope[], + window: UsageWindow, + options: UsageOptions, +): UsageOutput { + const sorted = [...events].sort(byTimeThenSeq); + const workers = options.workers ?? []; + const read = readEvents(sorted, window, { + answeredBefore: options.answeredBefore ?? new Set(), + fleet: options.fleet, + requestedBefore: options.requestedBefore ?? new Set(), + self: workers[0]?.id ?? "local", + }); + const bucketMs = options.bucketMs ?? seriesBucketMs(window.to - window.from); + const oldest = options.oldestTs ?? sorted[0]?.timestamp; + const coversFrom = oldest === undefined ? window.from : Math.max(window.from, oldest); + const figures = (scope: Scope) => figuresFor(read, window, scope, options.fleet); + const labelOf = new Map(workers.map((worker) => [worker.id, worker.label])); + const ids = new Set([...labelOf.keys(), ...read.workers]); + return { + bucketMs, + coversFrom, + partial: coversFrom > window.from, + platforms: { android: figures({ platform: "android" }), ios: figures({ platform: "ios" }) }, + requesters: requestersOf(read.requests, options.labels ?? {}), + series: seriesOf(read, window, bucketMs), + totals: figures({ queue: true }), + window: { from: window.from, to: window.to }, + workers: [...ids].sort().map((id) => { + const label = labelOf.get(id); + return { + ...figures({ queue: !options.fleet, worker: id }), + id, + ...(label === undefined ? {} : { label }), + }; + }), + }; +} + +interface Scope { + readonly platform?: Platform; + readonly worker?: string; + /** Whether the figures carry the queue's depth, which is the host's or the fleet's, not a + * platform's or a worker's. */ + readonly queue?: boolean; +} + +function inScope( + item: { readonly platform?: Platform | undefined; readonly worker?: string | undefined }, + scope: Scope, +): boolean { + return ( + (scope.platform === undefined || item.platform === scope.platform) && + (scope.worker === undefined || item.worker === scope.worker) + ); +} + +/** + * One scope's figures. On a worker a probe (a gateway's dispatch, ADR 0021 §5) is not a request: + * it is counted under `probes`, gives no wait, turnaround or rejection, and still counts as a grant + * with its source and held time. A gateway's answer has no `probes`, and its sources include + * `unknown`. + */ +function figuresFor( + read: ReadEvents, + window: UsageWindow, + scope: Scope, + fleet: boolean, +): UsageFigures { + const requests = read.requests.filter((fact) => inScope(fact, scope)); + const devices = read.devices.filter((fact) => inScope(fact, scope)); + const asked = requests.filter((fact) => fact.requestedAt !== undefined && fact.probe !== true); + const probes = requests.filter((fact) => fact.probe === true); + const grants = requests.flatMap((fact) => + fact.outcome?.kind === "granted" ? [{ ...fact, outcome: fact.outcome }] : [], + ); + const turnarounds = grants.filter((fact) => fact.probe !== true).flatMap(turnaroundOf); + const rejected = requests.flatMap((fact) => + fact.outcome?.kind === "rejected" && fact.probe !== true ? [fact.outcome.reason] : [], + ); + const sources = sourcesOf(grants.map((fact) => fact.outcome.source)); + return { + ...deviceFigures(devices), + bySource: { + booted: sources.booted, + provisioned: sources.provisioned, + ...(fleet ? { unknown: sources.unknown } : {}), + warm: sources.warm, + }, + declined: read.declines.filter((fact) => inScope(fact, scope)).length, + granted: grants.length, + held: summarise(grants.flatMap((fact) => fact.heldMs ?? [])), + queue: + scope.queue === true ? queueOf(read.queue, window) : { meanDepth: null, peakDepth: null }, + rejected: { byReason: countBy(rejected), total: rejected.length }, + requests: asked.length, + turnaround: summarise(turnarounds), + utilisation: utilisationOf(read.capacity, window, scope), + wait: summarise(asked.flatMap((fact) => spanOf(fact.requestedAt, fact.outcome?.at))), + ...(fleet ? {} : { probes: probes.length }), + }; +} + +/** Wait plus held time: each by the clock of the host that saw it, so the two never mix. */ +function turnaroundOf(fact: RequestFact): number[] { + const wait = spanOf(fact.requestedAt, fact.outcome?.at)[0]; + return wait === undefined || fact.heldMs === undefined ? [] : [wait + fact.heldMs]; +} + +/** The milliseconds from `start` to `end`, when both are known. */ +function spanOf(start: number | undefined, end: number | undefined): number[] { + return start === undefined || end === undefined ? [] : [end - start]; +} + +type GrantSource = "booted" | "provisioned" | "unknown" | "warm"; + +function sourcesOf(grantedSources: readonly string[]): Record { + const counts = countBy(grantedSources); + return { + booted: counts.booted ?? 0, + provisioned: counts.provisioned ?? 0, + unknown: counts.unknown ?? 0, + warm: counts.warm ?? 0, + }; +} + +/** The figures a device event brings: durations, incidents, and failures by name. */ +function deviceFigures( + devices: readonly DeviceFact[], +): Pick { + const durations = (kind: DeviceFact["kind"]) => + summarise(devices.filter((fact) => fact.kind === kind).map((fact) => fact.value as number)); + const named = (kind: DeviceFact["kind"]) => + countBy(devices.filter((fact) => fact.kind === kind).map((fact) => fact.value as string)); + const incidents = named("incident"); + return { + boot: durations("boot"), + failures: { byEvent: named("failure") }, + incidents: { + crashRecovered: incidents.crashRecovered ?? 0, + lost: incidents.lost ?? 0, + quarantineRecovered: incidents.quarantineRecovered ?? 0, + quarantined: incidents.quarantined ?? 0, + }, + provisioning: durations("provisioning"), + }; +} + +function countBy(values: readonly string[]): Record { + const counts: Record = {}; + for (const value of values) counts[value] = (counts[value] ?? 0) + 1; + return counts; +} + +/** p50, p95 and max by nearest rank: the smallest sample at or above that share of the samples. */ +function summarise(values: readonly number[]): UsageFigures["wait"] { + if (values.length === 0) return { count: 0, max: null, p50: null, p95: null }; + const sorted = [...values].sort((left, right) => left - right); + const rank = (share: number) => + sorted[Math.max(0, Math.ceil(share * sorted.length) - 1)] as number; + return { count: sorted.length, max: sorted.at(-1) as number, p50: rank(0.5), p95: rank(0.95) }; +} + +function queueOf(steps: readonly Step[], window: UsageWindow): UsageFigures["queue"] { + const figures = peakAndMean(segmentsOf(steps, window.from, window.to), (values) => values[0]); + return figures === undefined + ? { meanDepth: null, peakDepth: null } + : { meanDepth: figures.mean, peakDepth: figures.peak }; +} + +const sumOf = (values: readonly number[]): number | undefined => + values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0); + +/** The RAM figure summed over the workers that report a budget; `undefined` when none does. */ +const ramOf = (values: readonly CapacityValue[], key: "used" | "limit"): number | undefined => + sumOf(values.flatMap((value) => (value.ram === undefined ? [] : [value.ram[key]]))); + +/** What one scope's capacity steps say: a platform's own entry, or the global one. */ +function scopedSteps(steps: readonly CapacityStep[], scope: Scope): Step[] { + const platform = scope.platform; + return steps + .filter((step) => scope.worker === undefined || step.key === scope.worker) + .map((step) => ({ + at: step.at, + key: step.key, + value: platform === undefined ? step.value : step.value.platforms[platform], + })); +} + +function utilisationOf( + steps: readonly CapacityStep[], + window: UsageWindow, + scope: Scope, +): UsageFigures["utilisation"] { + const segments = segmentsOf(scopedSteps(steps, scope), window.from, window.to); + const slots = peakAndMean(segments, (values) => sumOf(values.map((value) => value.used))); + const ram = peakAndMean(segments, (values) => ramOf(values, "used")); + const maxima = segments.flatMap((segment) => + segment.end > segment.start ? (sumOf(segment.values.map((value) => value.max)) ?? []) : [], + ); + const limits = segments.flatMap((segment) => ramOf(segment.values, "limit") ?? []); + return { + ...(ram === undefined + ? {} + : { ram: { limitBytes: limits.at(-1) as number, meanBytes: ram.mean, peakBytes: ram.peak } }), + slots: { + max: maxima.length === 0 ? null : Math.max(...maxima), + mean: slots?.mean ?? null, + peak: slots?.peak ?? null, + }, + }; +} + +function requestersOf( + requests: readonly RequestFact[], + labels: Readonly>, +): UsageOutput["requesters"] { + const byId = new Map(); + for (const fact of requests) { + // A probe that was not granted is nothing of its requester's: it was never its request. + if (fact.probe === true && fact.outcome?.kind !== "granted") continue; + const entry = byId.get(fact.requester) ?? { + granted: 0, + heldTotalMs: 0, + id: fact.requester, + rejected: 0, + requests: 0, + }; + byId.set(fact.requester, countRequest(entry, fact)); + } + return [...byId.values()] + .sort((left, right) => right.requests - left.requests || (left.id < right.id ? -1 : 1)) + .map((entry) => { + const label = Object.hasOwn(labels, entry.id) ? labels[entry.id] : undefined; + return label === undefined ? entry : { ...entry, label }; + }); +} + +type Requester = UsageOutput["requesters"][number]; + +/** `entry` with `fact` counted in it: a request made, a grant and the time it was held, a rejection. + * A probe counts only as a grant and its held time. */ +function countRequest(entry: Requester, fact: RequestFact): Requester { + const probe = fact.probe === true; + const kind = fact.outcome?.kind; + return { + ...entry, + granted: entry.granted + Number(kind === "granted"), + // Only a granted request has a held time. + heldTotalMs: entry.heldTotalMs + (fact.heldMs ?? 0), + rejected: entry.rejected + Number(kind === "rejected" && !probe), + requests: entry.requests + Number(fact.requestedAt !== undefined && !probe), + }; +} + +function seriesOf(read: ReadEvents, window: UsageWindow, bucketMs: number): UsageOutput["series"] { + const points = Math.ceil((window.to - window.from) / bucketMs); + const ends = Array.from({ length: points }, (_, index) => + Math.min(window.from + (index + 1) * bucketMs, window.to), + ); + const capacity = valuesAt(scopedSteps(read.capacity, {}), ends); + const depths = valuesAt(read.queue, ends); + const waiting = waitingAt( + [...read.requests, ...read.carried].filter((fact) => fact.probe !== true), + ends, + ); + return ends.map((at, index) => { + const values = capacity[index] as readonly CapacityValue[]; + const ram = ramOf(values, "used"); + return { + at, + queueDepth: (depths[index] as readonly number[])[0] ?? null, + ...(ram === undefined ? {} : { ramUsedBytes: ram }), + slotsMax: sumOf(values.map((value) => value.max)) ?? null, + slotsUsed: sumOf(values.map((value) => value.used)) ?? null, + waiting: waiting[index] as number, + }; + }); +} + +/** + * How many requests have been made and not yet settled at each of `times`, which ascend: every + * request open at that time, those made before the window and still open when it began included. + */ +function waitingAt(requests: readonly RequestFact[], times: readonly number[]): number[] { + const ascending = (values: number[]) => values.sort((left, right) => left - right); + const made = ascending(requests.flatMap((fact) => fact.requestedAt ?? [])); + const settled = ascending( + requests.flatMap((fact) => (fact.requestedAt === undefined ? [] : (fact.outcome?.at ?? []))), + ); + let madeBy = 0; + let settledBy = 0; + return times.map((time) => { + while (madeBy < made.length && (made[madeBy] as number) <= time) madeBy += 1; + while (settledBy < settled.length && (settled[settledBy] as number) <= time) settledBy += 1; + return madeBy - settledBy; + }); +} diff --git a/src/core/usage/index.ts b/src/core/usage/index.ts new file mode 100644 index 00000000..530547ce --- /dev/null +++ b/src/core/usage/index.ts @@ -0,0 +1 @@ +export { tokenLabelMap, UsageReader } from "./usage-reader.js"; diff --git a/src/core/usage/read-events.test.ts b/src/core/usage/read-events.test.ts new file mode 100644 index 00000000..a6d81d8c --- /dev/null +++ b/src/core/usage/read-events.test.ts @@ -0,0 +1,1118 @@ +import { describe, expect, it } from "vitest"; + +import type { EventEnvelope } from "../../bus/index.js"; +import { readEvents, type ReadOptions } from "./read-events.js"; + +const WINDOW = { from: 100, to: 200 }; +const WORKER: ReadOptions = { + answeredBefore: new Set(), + fleet: false, + requestedBefore: new Set(), + self: "self", +}; +const FLEET: ReadOptions = { + answeredBefore: new Set(), + fleet: true, + requestedBefore: new Set(), + self: "gateway", +}; + +let sequence = 0; + +function at(timestamp: number, event: string, payload: Record): EventEnvelope { + sequence += 1; + return { + event, + id: `evt_${sequence}`, + module: "test", + payload, + seq: sequence, + timestamp, + } as unknown as EventEnvelope; +} + +const read = (events: EventEnvelope[], options: ReadOptions = WORKER) => + readEvents(events, WINDOW, options); + +const entry = (running: number, reserved: number, maxRunning: number) => ({ + maxRunning, + reserved, + running, +}); +const capacityPayload = (extra: Record = {}) => ({ + android: entry(0, 0, 2), + global: entry(1, 2, 5), + ios: entry(1, 0, 3), + ...extra, +}); +const request = (ts: number, requestId: string, requester = "a", extra = {}) => + at(ts, "lease.requested", { requestId, requestSpec: { platform: "ios" }, requester, ...extra }); +/** What a worker relays of the grant it made for a gateway's dispatch. */ +const relayedGrant = (ts: number, extra: Record = {}) => + at(ts, "lease.granted", { + leaseId: "L", + requestId: "worker-side", + requester: "gw:a", + source: "warm", + workerId: "w1", + ...extra, + }); +/** The gateway's own record that it handed a fleet request's grant to its caller. */ +const handed = (ts: number, requestId: string, extra: Record = {}) => + at(ts, "request.granted", { + leaseId: "gw-lease", + requestId, + worker: "w1", + workerLeaseId: "L", + ...extra, + }); +/** The gateway's own rejection of a fleet request. */ +const gatewayReject = ( + ts: number, + requestId: string, + reason: string, + extra: Record = {}, +) => + at(ts, "lease.rejected", { + reason, + requestId, + requestSpec: { platform: "ios" }, + requester: "a", + ...extra, + }); +const restarted = (ts: number, extra: Record = {}) => + at(ts, "daemon.started", { configSnapshot: {}, version: "0", ...extra }); +/** What a worker relays of its refusal of a gateway's dispatch. */ +const declined = (ts: number, extra: Record = {}) => + at(ts, "lease.declined", { + fleetRequestId: "r", + reason: "no-wait", + requestId: "worker-side", + requestSpec: { platform: "ios" }, + requester: "gw:a", + ...extra, + }); + +describe("payload guards", () => { + it("ignores a deviceId, spec platform and duration of the wrong types when provisioning", () => { + const result = read([ + at(150, "device.provisioned", { deviceId: 5, duration: "12", spec: { platform: "ios" } }), + at(151, "device.provisioned", { duration: Number.NaN, spec: { platform: "ios" } }), + at(152, "device.provisioned", { duration: Infinity, spec: { platform: "ios" } }), + at(153, "device.provisioned", { duration: -Infinity, spec: { platform: "ios" } }), + at(154, "device.provisioned", { duration: 7, spec: ["ios"] }), + at(155, "device.provisioned", { duration: 8, spec: null }), + at(156, "device.provisioned", { duration: 9, spec: { platform: "windows" } }), + at(157, "device.provisioned", { duration: 0, spec: { platform: "android" } }), + ]); + expect(result.devices).toEqual([ + { kind: "provisioning", platform: undefined, value: 7, worker: "self" }, + { kind: "provisioning", platform: undefined, value: 8, worker: "self" }, + { kind: "provisioning", platform: undefined, value: 9, worker: "self" }, + { kind: "provisioning", platform: "android", value: 0, worker: "self" }, + ]); + }); + + it("does not count a boot whose duration is not a finite number", () => { + const result = read([ + at(150, "device.ready", { bootDuration: "5", deviceId: "d" }), + at(151, "device.ready", { bootDuration: Number.NaN, deviceId: "d" }), + at(152, "device.ready", { bootDuration: Infinity, deviceId: "d" }), + at(153, "device.ready", { deviceId: "d" }), + at(154, "device.ready", { bootDuration: 4, deviceId: "d" }), + ]); + expect(result.devices).toEqual([ + { kind: "boot", platform: undefined, value: 4, worker: "self" }, + ]); + }); +}); + +describe("capacity steps", () => { + const stepFor = (payload: Record) => + read([at(150, "capacity.changed", payload)]).capacity; + + it("turns a complete capacity.changed into a step with used = running + reserved", () => { + expect(stepFor(capacityPayload())).toEqual([ + { + at: 150, + key: "self", + value: { + max: 5, + platforms: { + android: { max: 2, ram: undefined, used: 0 }, + ios: { max: 3, ram: undefined, used: 1 }, + }, + ram: undefined, + used: 3, + }, + }, + ]); + }); + + it.each(["global", "ios", "android"])("ignores a step when %s is missing", (missing) => { + const payload: Record = capacityPayload(); + delete payload[missing]; + expect(stepFor(payload)).toEqual([]); + }); + + it.each(["global", "ios", "android"])("ignores a step when %s is not an object", (name) => { + expect(stepFor(capacityPayload({ [name]: null }))).toEqual([]); + expect(stepFor(capacityPayload({ [name]: 3 }))).toEqual([]); + }); + + it.each(["running", "reserved", "maxRunning"])( + "ignores a step when %s is missing in an entry", + (field) => { + const bad: Record = entry(1, 1, 1); + delete bad[field]; + for (const name of ["global", "ios", "android"]) { + expect(stepFor(capacityPayload({ [name]: bad }))).toEqual([]); + } + }, + ); + + it("ignores a step whose entry figure is not a finite number", () => { + expect(stepFor(capacityPayload({ global: { ...entry(1, 1, 1), running: "1" } }))).toEqual([]); + expect( + stepFor(capacityPayload({ global: { ...entry(1, 1, 1), reserved: Number.NaN } })), + ).toEqual([]); + expect( + stepFor(capacityPayload({ global: { ...entry(1, 1, 1), maxRunning: Infinity } })), + ).toEqual([]); + }); + + it("reads the ram figure from ramBudget and drops it when either side is missing", () => { + const ramOf = (budget: unknown) => stepFor(capacityPayload({ ramBudget: budget }))[0]?.value; + expect(ramOf({ limitBytes: 100, usedBytes: 40 })?.ram).toEqual({ limit: 100, used: 40 }); + expect(ramOf({ limitBytes: 100, usedBytes: 40 })?.platforms.ios.ram).toEqual({ + limit: 100, + used: 40, + }); + expect(ramOf({ limitBytes: 100 })).toMatchObject({ ram: undefined, used: 3 }); + expect(ramOf({ usedBytes: 40 })).toMatchObject({ ram: undefined, used: 3 }); + expect(ramOf({ limitBytes: "100", usedBytes: 40 })).toMatchObject({ ram: undefined }); + expect(ramOf({ limitBytes: 100, usedBytes: Number.NaN })).toMatchObject({ ram: undefined }); + expect(ramOf(null)).toMatchObject({ ram: undefined }); + expect(ramOf([1])).toMatchObject({ ram: undefined, used: 3 }); + }); + + it("keeps a step at exactly the end of the window, ignores one after it", () => { + const result = read([ + at(200, "capacity.changed", capacityPayload()), + at(201, "capacity.changed", capacityPayload()), + ]); + expect(result.capacity.map((s) => s.at)).toEqual([200]); + }); + + it("keeps a step from before the window start so the opening level is known", () => { + expect(read([at(50, "capacity.changed", capacityPayload())]).capacity).toHaveLength(1); + }); + + it("keys a relayed step by its worker and records the worker", () => { + const result = read([at(150, "capacity.changed", capacityPayload({ workerId: "w1" }))], FLEET); + expect(result.capacity.map((s) => s.key)).toEqual(["w1"]); + expect([...result.workers]).toEqual(["w1"]); + }); + + it("ignores a gateway capacity.changed with no workerId", () => { + const result = read([at(150, "capacity.changed", capacityPayload())], FLEET); + expect(result.capacity).toEqual([]); + expect([...result.workers]).toEqual([]); + }); + + it("records the worker of a malformed capacity.changed without adding a step", () => { + const result = read([at(150, "capacity.changed", { workerId: "w1" })], FLEET); + expect(result.capacity).toEqual([]); + expect([...result.workers]).toEqual(["w1"]); + }); + + it("records the worker of a worker's own malformed capacity.changed", () => { + const result = read([at(150, "capacity.changed", {})]); + expect([...result.workers]).toEqual(["self"]); + }); +}); + +describe("queue steps", () => { + it("records a depth at exactly the window end and ignores one after it", () => { + const result = read([ + at(50, "queue.changed", { depth: 1 }), + at(200, "queue.changed", { depth: 2 }), + at(201, "queue.changed", { depth: 3 }), + ]); + expect(result.queue).toEqual([ + { at: 50, key: "queue", value: 1 }, + { at: 200, key: "queue", value: 2 }, + ]); + }); + + it("ignores a depth that is not a finite number", () => { + const result = read([ + at(150, "queue.changed", { depth: "3" }), + at(151, "queue.changed", { depth: Number.NaN }), + at(152, "queue.changed", { depth: Infinity }), + at(153, "queue.changed", {}), + at(154, "queue.changed", { depth: 0 }), + ]); + expect(result.queue).toEqual([{ at: 154, key: "queue", value: 0 }]); + }); + + it("ignores a queue.changed a worker relayed to the gateway", () => { + const result = read( + [ + at(150, "queue.changed", { depth: 4, workerId: "w1" }), + at(151, "queue.changed", { depth: 5 }), + ], + FLEET, + ); + expect(result.queue).toEqual([{ at: 151, key: "queue", value: 5 }]); + }); +}); + +describe("requests on a worker", () => { + it("counts a grant with no source as an empty source, and takes no grant without a leaseId", () => { + const result = read([ + request(110, "r1"), + at(120, "lease.granted", { leaseId: "L1", requestId: "r1", requester: "a" }), + request(130, "r2", "b"), + at(140, "lease.granted", { requestId: "r2", requester: "b", source: "warm" }), + ]); + expect(result.requests[0]?.outcome).toEqual({ at: 120, kind: "granted", source: "" }); + expect(result.requests[1]).not.toHaveProperty("outcome"); + }); + + it("makes no request of an event at exactly the window start and one of an event at its end", () => { + const result = read([request(100, "r0"), request(200, "r1"), request(201, "r2")]); + expect(result.requests.map((r) => r.requestedAt)).toEqual([200]); + }); + + it("is not a request when requestId or requester is missing or not a string", () => { + const result = read([ + at(150, "lease.requested", { requester: "a" }), + at(151, "lease.requested", { requestId: "r" }), + at(152, "lease.requested", { requestId: 1, requester: "a" }), + at(153, "lease.requested", { requestId: "r", requester: 2 }), + request(154, "ok"), + ]); + expect(result.requests).toEqual([ + { platform: "ios", requestedAt: 154, requester: "a", worker: "self" }, + ]); + }); + + it("reads the platform from requestSpec and leaves it undefined otherwise", () => { + const result = read([ + at(150, "lease.requested", { + requestId: "a", + requestSpec: { platform: "android" }, + requester: "x", + }), + at(151, "lease.requested", { requestId: "b", requestSpec: ["ios"], requester: "x" }), + at(152, "lease.requested", { requestId: "c", requestSpec: null, requester: "x" }), + at(153, "lease.requested", { + requestId: "d", + requestSpec: { platform: "tv" }, + requester: "x", + }), + ]); + expect(result.requests.map((r) => r.platform)).toEqual([ + "android", + undefined, + undefined, + undefined, + ]); + }); + + it("joins a grant with its end by lease id and defaults a missing source to empty", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(130, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests).toEqual([ + { + heldMs: 10, + outcome: { at: 120, kind: "granted", source: "" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: "self", + }, + ]); + }); + + it("keeps the source of a grant and joins an expiry as an end", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r", source: "warm" }), + at(130, "lease.expired", { leaseId: "L" }), + ]); + expect(result.requests[0]).toMatchObject({ heldMs: 10, outcome: { source: "warm" } }); + }); + + it("does not join an end of another lease id", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(130, "lease.released", { leaseId: "other" }), + ]); + expect(result.requests[0]).not.toHaveProperty("heldMs"); + }); + + it("ignores a grant missing requestId or leaseId, and one outside the window", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L" }), + at(121, "lease.granted", { requestId: "r" }), + at(100, "lease.granted", { leaseId: "L", requestId: "r" }), + at(201, "lease.granted", { leaseId: "L", requestId: "r" }), + ]); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("takes a grant at exactly the window end", () => { + const result = read([ + request(110, "r"), + at(200, "lease.granted", { leaseId: "L", requestId: "r", source: "cold" }), + ]); + expect(result.requests[0]?.outcome).toEqual({ at: 200, kind: "granted", source: "cold" }); + }); + + it("ignores an end with no lease id, and one outside the window", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(130, "lease.released", {}), + at(100, "lease.released", { leaseId: "L" }), + at(201, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests[0]).not.toHaveProperty("heldMs"); + }); + + it("takes an end at exactly the window end", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(200, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests[0]?.heldMs).toBe(80); + }); + + it("outcome of a rejection of a seen request is the rejection with its reason", () => { + const result = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "no-capacity", requestId: "r", requester: "a" }), + ]); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason: "no-capacity" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: "self", + }, + ]); + }); + + it("ignores a rejection missing requestId, reason or requester, or outside the window", () => { + const result = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "killed", requester: "a" }), + at(121, "lease.rejected", { requestId: "r", requester: "a" }), + at(122, "lease.rejected", { reason: "killed", requestId: "r" }), + at(123, "lease.rejected", { reason: 4, requestId: "r", requester: "a" }), + at(100, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + at(201, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(result.requests).toHaveLength(1); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("takes a rejection at exactly the window end", () => { + const result = read([ + request(110, "r"), + at(200, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(result.requests[0]?.outcome).toEqual({ at: 200, kind: "rejected", reason: "killed" }); + }); + + it("counts a refused-at-admission rejection of an unseen request, with its platform", () => { + const result = read([ + at(150, "lease.rejected", { + reason: "already-leased", + requestId: "x", + requestSpec: { platform: "android" }, + requester: "b", + }), + at(151, "lease.rejected", { reason: "killed", requestId: "y", requester: "b" }), + at(152, "lease.rejected", { reason: "no-capacity", requestId: "z", requester: "b" }), + ]); + expect(result.requests).toEqual([ + { + outcome: { at: 150, kind: "rejected", reason: "already-leased" }, + platform: "android", + requestedAt: undefined, + requester: "b", + worker: "self", + }, + { + outcome: { at: 151, kind: "rejected", reason: "killed" }, + platform: undefined, + requestedAt: undefined, + requester: "b", + worker: "self", + }, + ]); + }); + + it("does not double count a refused-at-admission rejection whose request was seen", () => { + const result = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(result.requests).toHaveLength(1); + }); + + it("counts a killed rejection of a request made before the window in no count, and one made inside it with that request", () => { + const before = read( + [at(120, "lease.rejected", { reason: "killed", requestId: "old", requester: "a" })], + { ...WORKER, requestedBefore: new Set(["old"]) }, + ); + expect(before.requests).toEqual([]); + + const inside = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(inside.requests).toHaveLength(1); + expect(inside.requests[0]).toMatchObject({ requestedAt: 110 }); + }); + + it("an empty payload is read as no facts instead of throwing", () => { + const envelope = { event: "lease.requested", id: "e", module: "t", seq: 1, timestamp: 150 }; + expect(read([envelope as unknown as EventEnvelope]).requests).toEqual([]); + }); +}); + +describe("device facts", () => { + it("counts a provisioning at exactly the window end and not at its start or after", () => { + const result = read([ + at(100, "device.provisioned", { duration: 1 }), + at(200, "device.provisioned", { duration: 2 }), + at(201, "device.provisioned", { duration: 3 }), + ]); + expect(result.devices.map((d) => d.value)).toEqual([2]); + }); + + it("takes the boot platform from the earlier provisioning of the same device on the same worker", () => { + const result = read([ + at(110, "device.provisioned", { deviceId: "d", duration: 1, spec: { platform: "android" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d" }), + at(121, "device.ready", { bootDuration: 6, deviceId: "other" }), + at(122, "device.ready", { bootDuration: 7 }), + ]); + expect(result.devices.map((d) => [d.kind, d.platform, d.value])).toEqual([ + ["provisioning", "android", 1], + ["boot", "android", 5], + ["boot", undefined, 6], + ["boot", undefined, 7], + ]); + }); + + it("remembers the platform even when the provisioning is before the window or has no duration", () => { + const result = read([ + at(50, "device.provisioned", { deviceId: "d", duration: 1, spec: { platform: "ios" } }), + at(60, "device.provisioned", { deviceId: "e", spec: { platform: "android" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d" }), + at(121, "device.ready", { bootDuration: 6, deviceId: "e" }), + ]); + expect(result.devices.map((d) => d.platform)).toEqual(["ios", "android"]); + }); + + it("does not remember a platform for a provisioning with no device id or no platform", () => { + const result = read([ + at(110, "device.provisioned", { duration: 1, spec: { platform: "ios" } }), + at(111, "device.provisioned", { deviceId: "d", duration: 1 }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d" }), + ]); + expect(result.devices.at(-1)).toEqual({ + kind: "boot", + platform: undefined, + value: 5, + worker: "self", + }); + }); + + it("counts each incident event under the name the figure uses, and each `*-failed` event under its event name, a recovery failure under both", () => { + const names: Record = { + "device.quarantine-abandoned": "lost", + "device.quarantine-recovered": "quarantineRecovered", + "device.quarantine-stranded": "lost", + "device.quarantined": "quarantined", + "device.recovered": "crashRecovered", + "device.recovery-failed": "lost", + }; + const incidents = Object.keys(names).map((name, i) => at(110 + i, name, {})); + const failures = [ + at(130, "device.purge-failed", {}), + at(131, "component.install-failed", {}), + at(132, "device.some-new-failed", {}), + ]; + const result = read([ + ...incidents, + ...failures, + at(133, "device.created", {}), + at(134, "lease.other", {}), + ]); + expect(result.devices.map((d) => [d.kind, d.value])).toEqual([ + ...Object.values(names).map((v) => ["incident", v]), + ["failure", "device.recovery-failed"], + ["failure", "device.purge-failed"], + ["failure", "component.install-failed"], + ["failure", "device.some-new-failed"], + ]); + }); + + it("does not count incidents outside the window", () => { + const result = read([at(100, "device.quarantined", {}), at(201, "device.quarantined", {})]); + expect(result.devices).toEqual([]); + }); + + it("takes an incident's platform from its payload, else from the provisioned device", () => { + const result = read([ + at(110, "device.provisioned", { deviceId: "d", spec: { platform: "android" } }), + at(120, "device.quarantined", { deviceId: "d", platform: "ios" }), + at(121, "device.quarantined", { deviceId: "d" }), + at(122, "device.quarantined", { deviceId: "d", platform: "tv" }), + at(123, "device.quarantined", {}), + ]); + expect(result.devices.map((d) => d.platform)).toEqual(["ios", "android", "android", undefined]); + }); + + it("adds the worker to the workers set from a device fact", () => { + expect([...read([at(110, "device.quarantined", {})]).workers]).toEqual(["self"]); + }); +}); + +describe("requests carried into the window", () => { + const answered = (...ids: string[]): ReadOptions => ({ + ...WORKER, + answeredBefore: new Set(ids), + }); + const grant = (ts: number, requestId: string, requester = "a") => + at(ts, "lease.granted", { leaseId: `L${requestId}`, requestId, requester, source: "warm" }); + + it("carries a request made exactly at the window's start, and treats an answer the history reports as answering it", () => { + expect(read([request(100, "edge")]).carried).toHaveLength(1); + expect(read([request(90, "old")], answered("old")).carried).toEqual([]); + }); + + it("carries a request made before the window that nothing had answered, with its outcome inside it", () => { + const result = read([request(90, "old"), grant(150, "old")]); + expect(result.requests).toEqual([]); + expect(result.carried).toHaveLength(1); + expect(result.carried[0]).toMatchObject({ + outcome: { at: 150, kind: "granted", source: "warm" }, + requestedAt: 90, + requester: "a", + worker: "self", + }); + }); + + it("does not carry a request answered before the window, or one made inside it", () => { + const result = read( + [request(80, "done", "b"), request(85, "refused", "c"), request(110, "inside")], + answered("done", "refused"), + ); + expect(result.carried).toEqual([]); + expect(result.requests).toHaveLength(1); + }); + + it("carries only a requester's latest request made before the window", () => { + const result = read([request(80, "first"), request(90, "second")], answered("first")); + expect(result.carried.map((fact) => fact.requestedAt)).toEqual([90]); + }); + + it("matches a carried request to its answer by request id, not by requester", () => { + // Another request of the same requester was answered; this one was not. + const result = read([request(90, "waits")], answered("some-other-request")); + expect(result.carried.map((fact) => fact.requestedAt)).toEqual([90]); + }); + + it("does not carry a worker's probe, which never waits there", () => { + const result = read([request(90, "probe", "gw:a", { fleetRequestId: "f" })]); + expect(result.carried).toEqual([]); + }); + + it("carries a gateway request that nothing, on the gateway's own clock, had answered, with its outcome inside the window", () => { + const result = read([request(90, "old"), handed(150, "old")], FLEET); + expect(result.carried).toEqual([ + expect.objectContaining({ outcome: expect.objectContaining({ at: 150 }), worker: "w1" }), + ]); + expect(read([request(90, "old")], FLEET).carried).toHaveLength(1); + expect( + read([request(90, "old")], { ...FLEET, answeredBefore: new Set(["old"]) }).carried, + ).toEqual([]); + }); + + it("settles a carried gateway request at the gateway's next start, whether that is before the window or inside it, and leaves one a start preceded open", () => { + const before = read([request(80, "old"), restarted(90)], FLEET); + expect(before.carried).toEqual([ + expect.objectContaining({ + outcome: { at: 90, kind: "rejected", reason: "daemon-restarted" }, + }), + ]); + const later = read([request(80, "old"), restarted(150)], FLEET); + expect(later.carried).toEqual([ + expect.objectContaining({ + outcome: { at: 150, kind: "rejected", reason: "daemon-restarted" }, + requestedAt: 80, + }), + ]); + const after = read([restarted(70), request(80, "old")], FLEET); + expect(after.carried[0]).not.toHaveProperty("outcome"); + }); +}); + +describe("a worker's probes and declines", () => { + it("marks a request carrying fleetRequestId as a probe and a request without one as no probe", () => { + const result = read([ + request(110, "p", "gw:a", { fleetRequestId: "f" }), + request(111, "l", "b"), + ]); + expect(result.requests[0]).toHaveProperty("probe", true); + expect(result.requests[1]).not.toHaveProperty("probe"); + }); + + it("counts a probe's grant and held time like any grant", () => { + const result = read([ + request(110, "p", "gw:a", { fleetRequestId: "f" }), + at(120, "lease.granted", { leaseId: "L", requestId: "p", requester: "gw:a", source: "warm" }), + at(150, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests[0]).toMatchObject({ + heldMs: 30, + outcome: { at: 120, kind: "granted", source: "warm" }, + probe: true, + }); + }); + + it("counts the worker's own lease.declined under itself with the platform of requestSpec", () => { + const result = read([ + declined(110, { requestSpec: { platform: "android" } }), + declined(111, { requestSpec: {} }), + ]); + expect(result.declines).toEqual([ + { platform: "android", worker: "self" }, + { platform: undefined, worker: "self" }, + ]); + expect(result.requests).toEqual([]); + expect([...result.workers]).toEqual(["self"]); + }); + + it("counts a decline at exactly the window end and not at its start or after", () => { + const result = read([declined(100), declined(200), declined(201)]); + expect(result.declines).toHaveLength(1); + }); +}); + +describe("a gateway", () => { + it("counts a relayed device fact against its worker and ignores one with no worker", () => { + const result = read( + [ + at(110, "device.provisioned", { + deviceId: "d", + duration: 1, + spec: { platform: "ios" }, + workerId: "w1", + }), + at(111, "device.provisioned", { + deviceId: "d", + duration: 2, + spec: { platform: "android" }, + workerId: "w2", + }), + at(112, "device.provisioned", { deviceId: "d", duration: 3, spec: { platform: "ios" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d", workerId: "w2" }), + at(121, "device.ready", { bootDuration: 6, deviceId: "d", workerId: "w3" }), + at(122, "device.ready", { bootDuration: 7, deviceId: "d" }), + at(123, "device.quarantined", { workerId: "w1" }), + at(124, "device.quarantined", {}), + ], + FLEET, + ); + expect(result.devices.map((d) => [d.kind, d.worker, d.platform, d.value])).toEqual([ + ["provisioning", "w1", "ios", 1], + ["provisioning", "w2", "android", 2], + ["boot", "w2", "android", 5], + ["boot", "w3", undefined, 6], + ["incident", "w1", undefined, "quarantined"], + ]); + expect([...result.workers].sort()).toEqual(["w1", "w2", "w3"]); + }); + + it("gives a request with no outcome no worker and leaves it open", () => { + const result = read([request(110, "r")], FLEET); + expect(result.requests).toEqual([ + { platform: "ios", requestedAt: 110, requester: "a", worker: undefined }, + ]); + expect([...result.workers]).toEqual([]); + }); + + it("does not take a relayed lease.requested as the gateway's own request", () => { + const result = read([request(110, "r", "a", { workerId: "w1" })], FLEET); + expect(result.requests).toEqual([]); + }); + + describe("the outcome of a fleet request, by its request id", () => { + it("is the gateway's request.granted, by the gateway's clock, naming the worker it handed the grant from", () => { + const result = read( + [relayedGrant(105), request(110, "r"), handed(150, "r", { worker: "w1" })], + FLEET, + ); + expect(result.requests[0]).toMatchObject({ + outcome: { at: 150, kind: "granted" }, + requestedAt: 110, + worker: "w1", + }); + expect([...result.workers]).toEqual(["w1"]); + }); + + it("is a request.granted at exactly the window end, and none missing a field or outside the window", () => { + expect(read([request(110, "r"), handed(200, "r")], FLEET).requests[0]).toHaveProperty( + "outcome", + ); + const none = read( + [ + request(110, "r"), + handed(120, "r", { requestId: undefined }), + handed(121, "r", { worker: undefined }), + handed(122, "r", { workerLeaseId: undefined }), + handed(100, "r"), + handed(201, "r"), + ], + FLEET, + ); + expect(none.requests[0]).not.toHaveProperty("outcome"); + }); + + it("is not a request.granted that carries a workerId, which only a worker's relayed event does", () => { + const result = read([request(110, "r"), handed(120, "r", { workerId: "w1" })], FLEET); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("is not a request.granted on a worker", () => { + const result = read([request(110, "r"), handed(120, "r")]); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("is the gateway's own lease.rejected, under no worker, and a worker-failed one under its worker", () => { + const result = read( + [ + request(110, "timed-out"), + gatewayReject(120, "timed-out", "timeout", { worker: "w1" }), + request(111, "failed", "b"), + gatewayReject(121, "failed", "worker-failed", { + code: "INTERNAL", + requester: "b", + worker: "w2", + }), + ], + FLEET, + ); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason: "timeout" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: undefined, + }, + { + outcome: { at: 121, kind: "rejected", reason: "worker-failed" }, + platform: "ios", + requestedAt: 111, + requester: "b", + worker: "w2", + }, + ]); + expect([...result.workers]).toEqual(["w2"]); + }); + + it("is never a rejection a worker relayed, nor a grant it relayed, nor a decline", () => { + const result = read( + [ + request(110, "r"), + gatewayReject(120, "r", "no-wait", { workerId: "w1" }), + relayedGrant(121, { requestId: "r" }), + declined(122, { requestId: "r" }), + request(130, "r2", "b"), + relayedGrant(131, { requester: "gw:b" }), + ], + FLEET, + ); + expect(result.requests.map((fact) => fact.outcome)).toEqual([undefined, undefined]); + }); + + it("follows the request id, so another request of the same requester neither takes nor loses the outcome", () => { + const result = read( + [request(110, "r1"), request(112, "r2"), handed(120, "r2"), request(130, "r3")], + FLEET, + ); + expect(result.requests.map((fact) => fact.outcome?.kind)).toEqual([ + undefined, + "granted", + undefined, + ]); + }); + + it("is, with neither, the gateway's next own daemon.started, as a rejection daemon-restarted under no worker", () => { + const result = read([request(110, "r"), restarted(150), restarted(160)], FLEET); + expect(result.requests[0]).toEqual({ + outcome: { at: 150, kind: "rejected", reason: "daemon-restarted" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: undefined, + }); + }); + + it("is a daemon.started at exactly the window end", () => { + const result = read([request(110, "r"), restarted(200)], FLEET); + expect(result.requests[0]?.outcome).toEqual({ + at: 200, + kind: "rejected", + reason: "daemon-restarted", + }); + }); + + it("stays open past a daemon.started before the request, a relayed one, one after the window, and one on a worker", () => { + const open = (events: EventEnvelope[], options: ReadOptions = FLEET) => + read(events, options).requests[0]?.outcome; + expect(open([restarted(105), request(110, "r")])).toBeUndefined(); + expect(open([request(110, "r"), restarted(150, { workerId: "w1" })])).toBeUndefined(); + expect(open([request(110, "r"), restarted(201)])).toBeUndefined(); + expect(open([request(110, "r"), restarted(150)], WORKER)).toBeUndefined(); + }); + + it("is the request.granted or lease.rejected, not the restart that follows it", () => { + const result = read( + [ + request(110, "g"), + request(111, "j", "b"), + restarted(150), + handed(160, "g"), + gatewayReject(161, "j", "timeout", { requester: "b" }), + ], + FLEET, + ); + expect(result.requests.map((fact) => fact.outcome?.kind)).toEqual(["granted", "rejected"]); + expect(result.requests[1]?.outcome).toMatchObject({ reason: "timeout" }); + }); + }); + + describe("the device facts of a fleet grant, from the worker it names", () => { + const events = (endWorker: string, endLease: string) => [ + request(110, "r"), + relayedGrant(112), + handed(115, "r"), + at(130, "lease.released", { leaseId: endLease, workerId: endWorker }), + ]; + + it("take the source and the held time, by the worker's clock, from the relayed lease.granted and end of the worker and lease it names", () => { + const result = read(events("w1", "L"), FLEET); + expect(result.requests[0]).toMatchObject({ + heldMs: 18, + outcome: { at: 115, kind: "granted", source: "warm" }, + }); + }); + + it("have no held time from an end of another worker or another lease, or one no worker relayed", () => { + expect(read(events("w2", "L"), FLEET).requests[0]).not.toHaveProperty("heldMs"); + expect(read(events("w1", "M"), FLEET).requests[0]).not.toHaveProperty("heldMs"); + const unrelayed = read( + [ + request(110, "r"), + relayedGrant(112), + handed(115, "r"), + at(130, "lease.released", { leaseId: "L" }), + ], + FLEET, + ); + expect(unrelayed.requests[0]).not.toHaveProperty("heldMs"); + }); + + it("look up each grant's end under the worker its own request.granted names, when two workers use one lease id", () => { + const result = read( + [ + request(110, "r1"), + request(111, "r2", "b"), + relayedGrant(112, { workerId: "w1" }), + relayedGrant(115, { requester: "gw:b", workerId: "w2" }), + handed(116, "r1", { worker: "w1" }), + handed(117, "r2", { worker: "w2" }), + at(130, "lease.released", { leaseId: "L", workerId: "w1" }), + at(150, "lease.released", { leaseId: "L", workerId: "w2" }), + ], + FLEET, + ); + expect(result.requests.map((fact) => [fact.worker, fact.heldMs])).toEqual([ + ["w1", 18], + ["w2", 35], + ]); + }); + + it("are the source unknown and no held time when no worker relayed the grant, whichever of its lease id or worker is missing", () => { + const missing = read([request(110, "r"), handed(115, "r")], FLEET); + expect(missing.requests[0]).toMatchObject({ + outcome: { at: 115, kind: "granted", source: "unknown" }, + worker: "w1", + }); + expect(missing.requests[0]).not.toHaveProperty("heldMs"); + const otherLease = read( + [request(110, "r"), relayedGrant(112, { leaseId: "M" }), handed(115, "r")], + FLEET, + ); + expect(otherLease.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + const otherWorker = read( + [request(110, "r"), relayedGrant(112, { workerId: "w2" }), handed(115, "r")], + FLEET, + ); + expect(otherWorker.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + // An end of the lease with no relayed grant to start from gives no held time either. + const endOnly = read( + [ + request(110, "r"), + handed(115, "r"), + at(130, "lease.released", { leaseId: "L", workerId: "w1" }), + ], + FLEET, + ); + expect(endOnly.requests[0]).not.toHaveProperty("heldMs"); + }); + + it("default a relayed grant's missing source to empty, and ignore a relayed grant with no lease id or worker", () => { + const noSource = read( + [request(110, "r"), relayedGrant(112, { source: undefined }), handed(115, "r")], + FLEET, + ); + expect(noSource.requests[0]?.outcome).toMatchObject({ source: "" }); + const noLease = read( + [ + request(110, "r"), + relayedGrant(112, { leaseId: undefined }), + handed(115, "r", { workerLeaseId: "undefined" }), + ], + FLEET, + ); + expect(noLease.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + const noWorker = read( + [ + request(110, "r"), + relayedGrant(112, { workerId: undefined }), + handed(115, "r", { worker: "undefined" }), + ], + FLEET, + ); + expect(noWorker.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + }); + + it("ignore a relayed grant outside the window", () => { + const result = read( + [request(110, "r"), relayedGrant(100), relayedGrant(201), handed(115, "r")], + FLEET, + ); + expect(result.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + }); + }); + + it("counts a relayed lease.declined under its worker and platform, one for each, and ignores a gateway's own and one outside the window", () => { + const result = read( + [ + declined(110, { workerId: "w1" }), + declined(111, { workerId: "w1" }), + declined(112, { requestSpec: { platform: "android" }, workerId: "w2" }), + declined(113), + declined(100, { workerId: "w1" }), + declined(201, { workerId: "w1" }), + ], + FLEET, + ); + expect(result.declines).toEqual([ + { platform: "ios", worker: "w1" }, + { platform: "ios", worker: "w1" }, + { platform: "android", worker: "w2" }, + ]); + expect(result.requests).toEqual([]); + expect([...result.workers].sort()).toEqual(["w1", "w2"]); + }); + + it("counts a gateway's own refused-at-admission rejection with no worker, including lease-id-taken", () => { + for (const reason of ["killed", "already-leased", "lease-id-taken"]) { + const result = read( + [at(120, "lease.rejected", { reason, requestId: "x", requester: "a" })], + FLEET, + ); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason }, + platform: undefined, + requestedAt: undefined, + requester: "a", + worker: undefined, + }, + ]); + } + }); + + it("does not count a relayed rejection with a refused-at-admission reason as the gateway's own", () => { + const result = read([gatewayReject(120, "x", "killed", { workerId: "w1" })], FLEET); + expect(result.requests).toEqual([]); + }); +}); + +describe("keys that look like missing ids", () => { + it("does not join an end with no lease id to a grant of a lease named undefined", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "undefined", requestId: "r" }), + at(130, "lease.released", {}), + ]); + expect(result.requests[0]).not.toHaveProperty("heldMs"); + }); + + it("does not remember a platform for a provisioning with no device id under the name undefined", () => { + const result = read([ + at(110, "device.provisioned", { duration: 1, spec: { platform: "android" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "undefined" }), + ]); + expect(result.devices.at(-1)?.platform).toBeUndefined(); + }); + + it("gives a boot with no device id no platform even if a device was named undefined", () => { + const result = read([ + at(110, "device.provisioned", { + deviceId: "undefined", + duration: 1, + spec: { platform: "android" }, + }), + at(120, "device.ready", { bootDuration: 5 }), + ]); + expect(result.devices.at(-1)?.platform).toBeUndefined(); + }); + + it("does not share a platform between a worker-less provisioning and a worker named undefined", () => { + const result = read( + [ + at(110, "device.provisioned", { + deviceId: "d", + duration: 1, + spec: { platform: "android" }, + }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d", workerId: "undefined" }), + ], + FLEET, + ); + expect(result.devices).toEqual([ + { kind: "boot", platform: undefined, value: 5, worker: "undefined" }, + ]); + }); +}); diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts new file mode 100644 index 00000000..5fb8d3bb --- /dev/null +++ b/src/core/usage/read-events.ts @@ -0,0 +1,595 @@ +import type { EventEnvelope } from "../../bus/index.js"; +import type { UsageFigures } from "../../contract/index.js"; +import type { Step } from "./timeline.js"; + +/** + * Reads a window's events into the facts the figures count (ADR 0016 §2, ADR 0021 §5): one fact + * for each request, one for each device event that counts, one for each decline, and the capacity + * and queue steps. Nothing here adds a number up. + */ + +export type Platform = "ios" | "android"; + +export interface RequestFact { + readonly requester: string; + readonly platform: Platform | undefined; + /** The worker that served it: a worker's own, or the one a gateway dispatched it to. */ + readonly worker: string | undefined; + /** `undefined` for a rejection refused before the request was stored, which is no request. */ + readonly requestedAt: number | undefined; + /** On a worker: a gateway's dispatch (ADR 0021 §1), which gives no wait, turnaround, request or + * rejection. Absent otherwise. */ + readonly probe?: true; + readonly outcome?: Granted | Rejected; + /** How long the lease was held, when the window saw both its grant and its end. Both are the + * clock of the host that granted it, so on a gateway it is not `outcome.at` to an end. */ + readonly heldMs?: number; +} + +type RequestFactBase = Pick; + +interface Granted { + readonly kind: "granted"; + readonly at: number; + /** `unknown` on a gateway for a grant whose worker's own `lease.granted` was not relayed. */ + readonly source: string; +} + +interface Rejected { + readonly kind: "rejected"; + readonly at: number; + readonly reason: string; +} + +export interface DeviceFact { + readonly kind: "provisioning" | "boot" | "incident" | "failure"; + readonly worker: string | undefined; + readonly platform: Platform | undefined; + /** The duration for the first two, the incident name or the failure event name for the others. */ + readonly value: number | string; +} + +/** One `lease.declined` (ADR 0021 §2): a worker's refusal of a gateway dispatch. Not a request. */ +export interface DeclineFact { + readonly worker: string; + readonly platform: Platform | undefined; +} + +export interface CapacityValue { + readonly used: number; + readonly max: number; + readonly ram: { readonly used: number; readonly limit: number } | undefined; +} + +export type CapacityStep = Step< + CapacityValue & { readonly platforms: Readonly> } +>; + +export interface ReadEvents { + readonly requests: readonly RequestFact[]; + /** + * Requests with no outcome by the window's start, which only the series counts: ones made + * before the window and still open when it began, and a gateway's requests that a restart + * before the window's end ended (outcome `daemon-restarted`). + */ + readonly carried: readonly RequestFact[]; + readonly devices: readonly DeviceFact[]; + readonly declines: readonly DeclineFact[]; + readonly capacity: readonly CapacityStep[]; + readonly queue: readonly Step[]; + readonly workers: ReadonlySet; +} + +export interface ReadOptions { + readonly fleet: boolean; + /** The `requestId` of every request made before the window. */ + readonly requestedBefore: ReadonlySet; + /** The `requestId` of every request answered before the window: granted or rejected. */ + readonly answeredBefore: ReadonlySet; + /** The id a worker's own events are attributed to. */ + readonly self: string; +} + +export interface UsageWindow { + readonly from: number; + readonly to: number; +} + +type Payload = Readonly>; +type Incident = keyof UsageFigures["incidents"]; + +/** A rejection with no `lease.requested` anywhere in the history read was refused before the + * request was stored; only these reasons are ever given before admission. One whose request + * is in the history follows that request: counted with it, or in no count when it was made before + * the window. */ +const REFUSED_AT_ADMISSION = new Set(["already-leased", "killed", "lease-id-taken"]); +/** The device events that count as an incident, by what the figure calls them. */ +const INCIDENT_OF: Readonly> = { + "device.quarantine-abandoned": "lost", + "device.quarantine-recovered": "quarantineRecovered", + "device.quarantine-stranded": "lost", + "device.quarantined": "quarantined", + "device.recovered": "crashRecovered", + "device.recovery-failed": "lost", +}; +/** A failure is any event named `*-failed`, counted by its name. */ +const isFailure = (event: string): boolean => event.endsWith("-failed"); + +const text = (payload: Payload, key: string): string | undefined => { + const value = payload[key]; + return typeof value === "string" ? value : undefined; +}; + +const number = (payload: Payload, key: string): number | undefined => { + const value = payload[key]; + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +}; + +const platformOf = (value: unknown): Platform | undefined => + value === "ios" || value === "android" ? value : undefined; + +const objectOf = (value: unknown): Payload | undefined => + typeof value === "object" && value !== null ? (value as Payload) : undefined; + +interface RelayedGrant { + readonly at: number; + readonly index: number; + readonly source: string; +} + +interface Request { + readonly index: number; + readonly at: number; + readonly requester: string; + readonly platform: Platform | undefined; + readonly probe: boolean; +} + +/** What one event brings to a handler. */ +interface Seen { + readonly event: EventEnvelope; + readonly index: number; + readonly payload: Payload; + /** The worker the fact is about: a worker's own id, or on a gateway the one that relayed it. */ + readonly worker: string | undefined; + /** A gateway's own fact, as against one a worker relayed. Always true on a worker. */ + readonly own: boolean; + readonly within: boolean; +} + +type OwnRejection = Omit & { + readonly requester: string; + readonly platform: Platform | undefined; + /** The worker a `worker-failed` rejection names. */ + readonly worker: string | undefined; +}; + +export function readEvents( + sorted: readonly EventEnvelope[], + window: UsageWindow, + options: ReadOptions, +): ReadEvents { + const reader = new Reader(window, options); + sorted.forEach((event, index) => reader.see(event, index)); + return reader.finish(); +} + +class Reader { + readonly #devices: DeviceFact[] = []; + readonly #declines: DeclineFact[] = []; + readonly #capacity: CapacityStep[] = []; + readonly #queue: Step[] = []; + readonly #workers = new Set(); + readonly #devicePlatform = new Map(); + readonly #requested = new Map(); + /** The latest request of each requester made before the window. */ + readonly #carriedRequests = new Map(); + readonly #ownRejections = new Map(); + /** A worker's own grants by request id: the lease each served. */ + readonly #grants = new Map< + string, + { + readonly at: number; + readonly index: number; + readonly leaseId: string; + readonly source: string; + } + >(); + /** A gateway's `request.granted` by request id (ADR 0021 §4). */ + readonly #handed = new Map< + string, + { readonly at: number; readonly worker: string; readonly workerLeaseId: string } + >(); + /** + * The grants workers relayed to a gateway, by worker and the worker's lease id, in event order. + * A lease id is unique only while its lease is active (ADR 0020), so one key can hold several. + */ + readonly #relayedGrants = new Map(); + /** The ends of leases, by worker and lease id, in event order: a key can hold several. */ + readonly #ends = new Map(); + /** Where the gateway's own `daemon.started` events fall, in event order (ADR 0021 §5). */ + readonly #restarts: { readonly index: number; readonly at: number }[] = []; + readonly #handlers: Record void> = { + "capacity.changed": (seen) => this.#capacityChanged(seen), + "daemon.started": (seen) => this.#daemonStarted(seen), + "device.provisioned": (seen) => this.#provisioned(seen), + "device.ready": (seen) => this.#ready(seen), + "lease.declined": (seen) => this.#declined(seen), + "lease.expired": (seen) => this.#ended(seen), + "lease.granted": (seen) => this.#granted(seen), + "lease.rejected": (seen) => this.#rejected(seen), + "lease.released": (seen) => this.#ended(seen), + "lease.requested": (seen) => this.#leaseRequested(seen), + "queue.changed": (seen) => this.#queueChanged(seen), + "request.granted": (seen) => this.#requestGranted(seen), + }; + + constructor( + private readonly window: UsageWindow, + private readonly options: ReadOptions, + ) {} + + see(event: EventEnvelope, index: number): void { + const payload = (event.payload ?? {}) as Payload; + const relayedFrom = text(payload, "workerId"); + const seen: Seen = { + event, + index, + own: !this.options.fleet || relayedFrom === undefined, + payload, + within: event.timestamp > this.window.from && event.timestamp <= this.window.to, + worker: this.options.fleet ? relayedFrom : this.options.self, + }; + (this.#handlers[event.event] ?? ((other) => this.#deviceEvent(other)))(seen); + } + + finish(): ReadEvents { + return { + capacity: this.#capacity, + carried: this.#carriedOpen(), + declines: this.#declines, + devices: this.#devices, + queue: this.#queue, + requests: [...this.#requests(), ...this.#refusedAtAdmission()], + workers: this.#workers, + }; + } + + #key(worker: string, id: string): string { + return `${worker}\u0000${id}`; + } + + /** On a gateway a device fact needs a worker behind it; on a worker it is always its own. */ + #counts(seen: Seen): seen is Seen & { readonly worker: string } { + return seen.within && (!this.options.fleet || seen.worker !== undefined); + } + + #capacityChanged(seen: Seen): void { + const { event, payload, worker } = seen; + if (event.timestamp > this.window.to || worker === undefined) return; + const step = capacityStep(payload, event.timestamp, worker); + if (step !== undefined) this.#capacity.push(step); + this.#workers.add(worker); + } + + #queueChanged(seen: Seen): void { + const depth = number(seen.payload, "depth"); + if (seen.event.timestamp > this.window.to || !seen.own || depth === undefined) return; + this.#queue.push({ at: seen.event.timestamp, key: "queue", value: depth }); + } + + #leaseRequested(seen: Seen): void { + if (!seen.own) return; + const requestId = text(seen.payload, "requestId"); + const requester = text(seen.payload, "requester"); + if (requestId === undefined || requester === undefined) return; + const request: Request = { + at: seen.event.timestamp, + index: seen.index, + platform: requestPlatform(seen.payload), + probe: !this.options.fleet && text(seen.payload, "fleetRequestId") !== undefined, + requester, + }; + if (seen.event.timestamp <= this.window.from) { + this.#carriedRequests.set(requester, { ...request, requestId }); + } + if (seen.within) this.#requested.set(requestId, request); + } + + /** A gateway's restart ends every fleet request still without an outcome (ADR 0021 §5). */ + #daemonStarted(seen: Seen): void { + if (!this.options.fleet || !seen.own || seen.event.timestamp > this.window.to) return; + this.#restarts.push({ at: seen.event.timestamp, index: seen.index }); + } + + /** A gateway's own `request.granted` is its fleet request's grant. */ + #requestGranted(seen: Seen): void { + const requestId = text(seen.payload, "requestId"); + const worker = text(seen.payload, "worker"); + const workerLeaseId = text(seen.payload, "workerLeaseId"); + if (!this.options.fleet || !seen.own || !seen.within) return; + if (requestId === undefined || worker === undefined || workerLeaseId === undefined) return; + this.#handed.set(requestId, { at: seen.event.timestamp, worker, workerLeaseId }); + } + + /** A worker's `lease.declined`, or one a gateway's worker relayed: an event, never a request. */ + #declined(seen: Seen): void { + if (!seen.within || seen.worker === undefined) return; + this.#declines.push({ platform: requestPlatform(seen.payload), worker: seen.worker }); + this.#workers.add(seen.worker); + } + + #rejected(seen: Seen): void { + const requestId = text(seen.payload, "requestId"); + const reason = text(seen.payload, "reason"); + const requester = text(seen.payload, "requester"); + if (!seen.within || !seen.own) return; + if (requestId === undefined || reason === undefined || requester === undefined) return; + this.#ownRejections.set(requestId, { + at: seen.event.timestamp, + platform: requestPlatform(seen.payload), + reason, + requester, + worker: reason === "worker-failed" ? text(seen.payload, "worker") : undefined, + }); + } + + #granted(seen: Seen): void { + if (!seen.within) return; + const leaseId = text(seen.payload, "leaseId"); + if (leaseId === undefined) return; + const source = text(seen.payload, "source") ?? ""; + if (this.options.fleet) { + if (seen.worker !== undefined) { + const key = this.#key(seen.worker, leaseId); + const earlier = this.#relayedGrants.get(key) ?? []; + earlier.push({ at: seen.event.timestamp, index: seen.index, source }); + this.#relayedGrants.set(key, earlier); + } + return; + } + const requestId = text(seen.payload, "requestId"); + if (requestId === undefined) return; + this.#grants.set(requestId, { at: seen.event.timestamp, index: seen.index, leaseId, source }); + } + + #ended(seen: Seen): void { + const leaseId = text(seen.payload, "leaseId"); + if (!this.#counts(seen) || leaseId === undefined) return; + const key = this.#key(seen.worker, leaseId); + const earlier = this.#ends.get(key) ?? []; + earlier.push({ at: seen.event.timestamp, index: seen.index }); + this.#ends.set(key, earlier); + } + + #provisioned(seen: Seen): void { + const deviceId = text(seen.payload, "deviceId"); + const platform = platformOf(objectOf(seen.payload.spec)?.platform); + const duration = number(seen.payload, "duration"); + if (seen.worker !== undefined && deviceId !== undefined && platform !== undefined) { + this.#devicePlatform.set(this.#key(seen.worker, deviceId), platform); + } + if (this.#counts(seen) && duration !== undefined) { + this.#addDevice(seen, { kind: "provisioning", platform, value: duration }); + } + } + + #ready(seen: Seen): void { + const deviceId = text(seen.payload, "deviceId"); + const duration = number(seen.payload, "bootDuration"); + if (!this.#counts(seen) || duration === undefined) return; + this.#addDevice(seen, { + kind: "boot", + platform: this.#platformOfDevice(seen.worker, deviceId), + value: duration, + }); + } + + #deviceEvent(seen: Seen): void { + const incident = INCIDENT_OF[seen.event.event]; + const failure = isFailure(seen.event.event); + if (!this.#counts(seen) || (incident === undefined && !failure)) return; + const platform = + platformOf(seen.payload.platform) ?? + this.#platformOfDevice(seen.worker, text(seen.payload, "deviceId")); + if (incident !== undefined) + this.#addDevice(seen, { kind: "incident", platform, value: incident }); + if (failure) this.#addDevice(seen, { kind: "failure", platform, value: seen.event.event }); + } + + #platformOfDevice(worker: string, deviceId: string | undefined): Platform | undefined { + return deviceId === undefined + ? undefined + : this.#devicePlatform.get(this.#key(worker, deviceId)); + } + + #addDevice(seen: Seen, fact: Omit): void { + this.#devices.push({ ...fact, worker: seen.worker }); + if (seen.worker !== undefined) this.#workers.add(seen.worker); + } + + // ---- joining a request with what became of it ------------------------------------------- + + /** One fact for each request the window saw made, joined with its outcome and its end. */ + #requests(): RequestFact[] { + const facts: RequestFact[] = []; + for (const [requestId, request] of this.#requested) { + facts.push(this.#factFor(requestId, request)); + } + return facts; + } + + /** + * The requests made before the window that the history does not report answered by its start. + * One a gateway restart ended may carry an outcome from before the window, which settles it for + * every time in it. A worker's probe never waits there (ADR 0021 §5). + */ + #carriedOpen(): RequestFact[] { + const facts: RequestFact[] = []; + for (const request of this.#carriedRequests.values()) { + if (request.probe || this.options.answeredBefore.has(request.requestId)) continue; + facts.push(this.#factFor(request.requestId, request)); + } + return facts; + } + + #factFor(requestId: string, request: Request): RequestFact { + const base = { + platform: request.platform, + requestedAt: request.at, + requester: request.requester, + ...(request.probe ? { probe: true as const } : {}), + }; + return this.options.fleet + ? this.#fleetFact(base, requestId, request) + : this.#workerFact(base, requestId, this.options.self); + } + + #workerFact(base: RequestFactBase, requestId: string, own: string): RequestFact { + const rejection = this.#ownRejections.get(requestId); + if (rejection !== undefined) { + return { ...base, outcome: rejectedOf(rejection), worker: own }; + } + const grant = this.#grants.get(requestId); + if (grant === undefined) return { ...base, worker: own }; + const outcome: Granted = { at: grant.at, kind: "granted", source: grant.source }; + return this.#withHeld( + { ...base, outcome, worker: own }, + this.#heldFor(own, grant.leaseId, grant), + ); + } + + /** + * A fleet request's outcome is the gateway's own `request.granted` or `lease.rejected` for its + * request id; with neither, the gateway's next start ends it; else it is open (ADR 0021 §5). + */ + #fleetFact(base: RequestFactBase, requestId: string, request: Request): RequestFact { + const rejection = this.#ownRejections.get(requestId); + if (rejection !== undefined) { + if (rejection.worker !== undefined) this.#workers.add(rejection.worker); + return { ...base, outcome: rejectedOf(rejection), worker: rejection.worker }; + } + const handed = this.#handed.get(requestId); + if (handed !== undefined) return this.#grantedFleetFact(base, handed); + const restart = this.#restarts.find((candidate) => candidate.index > request.index); + if (restart === undefined) return { ...base, worker: undefined }; + const outcome: Rejected = { at: restart.at, kind: "rejected", reason: "daemon-restarted" }; + return { ...base, outcome, worker: undefined }; + } + + /** The grant source and the held time are the worker's own facts, by the worker's clock. */ + #grantedFleetFact( + base: RequestFactBase, + handed: { readonly at: number; readonly worker: string; readonly workerLeaseId: string }, + ): RequestFact { + this.#workers.add(handed.worker); + const relayed = this.#relayedGrantFor(handed); + const outcome: Granted = { + at: handed.at, + kind: "granted", + source: relayed === undefined ? "unknown" : relayed.source, + }; + const fact = { ...base, outcome, worker: handed.worker }; + return relayed === undefined + ? fact + : this.#withHeld(fact, this.#heldFor(handed.worker, handed.workerLeaseId, relayed)); + } + + /** Of the grants a worker relayed under one lease id, the one nearest in time to the handover. */ + #relayedGrantFor(handed: { + readonly at: number; + readonly worker: string; + readonly workerLeaseId: string; + }): RelayedGrant | undefined { + const candidates = this.#relayedGrants.get(this.#key(handed.worker, handed.workerLeaseId)); + let nearest: RelayedGrant | undefined; + for (const candidate of candidates ?? []) { + if ( + nearest === undefined || + Math.abs(candidate.at - handed.at) < Math.abs(nearest.at - handed.at) + ) { + nearest = candidate; + } + } + return nearest; + } + + /** How long a lease was held, from its grant to the first end after it the window saw. */ + #heldFor( + worker: string, + leaseId: string, + grant: { readonly at: number; readonly index: number }, + ): number | undefined { + const end = this.#ends.get(this.#key(worker, leaseId))?.find((e) => e.index > grant.index); + return end === undefined ? undefined : end.at - grant.at; + } + + #withHeld(fact: RequestFact, heldMs: number | undefined): RequestFact { + return heldMs === undefined ? fact : { ...fact, heldMs }; + } + + /** A gateway's or worker's own rejection of a request the window did not see made, when its + * reason is one given before admission. */ + #refusedAtAdmission(): RequestFact[] { + const facts: RequestFact[] = []; + for (const [requestId, rejection] of this.#ownRejections) { + if (this.#requested.has(requestId) || this.options.requestedBefore.has(requestId)) continue; + if (!REFUSED_AT_ADMISSION.has(rejection.reason)) continue; + facts.push({ + outcome: rejectedOf(rejection), + platform: rejection.platform, + requestedAt: undefined, + requester: rejection.requester, + worker: this.options.fleet ? undefined : this.options.self, + }); + } + return facts; + } +} + +const rejectedOf = (rejection: OwnRejection): Rejected => ({ + at: rejection.at, + kind: "rejected", + reason: rejection.reason, +}); + +function requestPlatform(payload: Payload): Platform | undefined { + return platformOf(objectOf(payload.requestSpec)?.platform); +} + +function capacityEntry( + entry: unknown, +): + | { readonly running: number; readonly reserved: number; readonly maxRunning: number } + | undefined { + const figures = objectOf(entry); + if (figures === undefined) return undefined; + const running = number(figures, "running"); + const reserved = number(figures, "reserved"); + const maxRunning = number(figures, "maxRunning"); + return running === undefined || reserved === undefined || maxRunning === undefined + ? undefined + : { maxRunning, reserved, running }; +} + +/** A `capacity.changed` as a step, or `undefined` for a payload that is not one. */ +function capacityStep(payload: Payload, at: number, key: string): CapacityStep | undefined { + const global = capacityEntry(payload.global); + const ios = capacityEntry(payload.ios); + const android = capacityEntry(payload.android); + if (global === undefined || ios === undefined || android === undefined) return undefined; + const budget = objectOf(payload.ramBudget); + const used = budget === undefined ? undefined : number(budget, "usedBytes"); + const limit = budget === undefined ? undefined : number(budget, "limitBytes"); + const ram = used === undefined || limit === undefined ? undefined : { limit, used }; + const valueOf = (entry: NonNullable>): CapacityValue => ({ + max: entry.maxRunning, + ram, + used: entry.running + entry.reserved, + }); + return { + at, + key, + value: { ...valueOf(global), platforms: { android: valueOf(android), ios: valueOf(ios) } }, + }; +} diff --git a/src/core/usage/timeline.test.ts b/src/core/usage/timeline.test.ts new file mode 100644 index 00000000..3dd621d1 --- /dev/null +++ b/src/core/usage/timeline.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; + +import { peakAndMean, segmentsOf, type Step, valuesAt } from "./timeline.js"; + +const step = (at: number, key: string, value: number): Step => ({ at, key, value }); +const sum = (values: readonly number[]): number | undefined => + values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0); + +describe("segmentsOf", () => { + it("starts with the step in force at the start of the window and cuts a segment at each later step", () => { + const segments = segmentsOf([step(5, "a", 1), step(20, "a", 3), step(40, "a", 0)], 10, 50); + + expect(segments).toEqual([ + { end: 20, start: 10, values: [1] }, + { end: 40, start: 20, values: [3] }, + { end: 50, start: 40, values: [0] }, + ]); + }); + + it("holds no value before the first step, and reads a step at the very start of the window as in force", () => { + const segments = segmentsOf([step(30, "a", 2)], 10, 50); + const atStart = segmentsOf([step(10, "a", 2)], 10, 50); + + expect(segments).toEqual([ + { end: 30, start: 10, values: [] }, + { end: 50, start: 30, values: [2] }, + ]); + expect(atStart).toEqual([{ end: 50, start: 10, values: [2] }]); + }); + + it("leaves out steps after the end of the window, and keeps one value for each key", () => { + const segments = segmentsOf( + [step(5, "a", 1), step(6, "b", 10), step(30, "a", 2), step(80, "b", 99)], + 10, + 50, + ); + + expect(segments).toEqual([ + { end: 30, start: 10, values: [1, 10] }, + { end: 50, start: 30, values: [2, 10] }, + ]); + }); +}); + +describe("segmentsOf at the end of the window", () => { + it("cuts no segment for a step at the very end of the window, which has no time in the window", () => { + const segments = segmentsOf([step(5, "a", 1), step(50, "a", 9)], 10, 50); + + expect(segments).toEqual([{ end: 50, start: 10, values: [1] }]); + }); +}); + +describe("valuesAt", () => { + it("answers the values in force at each time, counting a step at that time", () => { + const steps = [step(5, "a", 1), step(20, "a", 3), step(20, "b", 7)]; + + expect(valuesAt(steps, [4, 5, 19, 20, 99])).toEqual([[], [1], [1], [3, 7], [3, 7]]); + }); +}); + +describe("peakAndMean", () => { + it("weighs each segment by its time and leaves out the time before the first step", () => { + const segments = segmentsOf([step(30, "a", 4), step(40, "a", 0)], 0, 60); + + expect(peakAndMean(segments, sum)).toEqual({ mean: 40 / 30, peak: 4 }); + expect(peakAndMean(segmentsOf([], 0, 60), sum)).toBeUndefined(); + }); + + it("gives no weight to a segment with no time in it", () => { + const segments = segmentsOf([step(10, "a", 100), step(10, "a", 2)], 0, 20); + + expect(peakAndMean(segments, sum)).toEqual({ mean: 2, peak: 2 }); + }); +}); diff --git a/src/core/usage/timeline.ts b/src/core/usage/timeline.ts new file mode 100644 index 00000000..b635bb89 --- /dev/null +++ b/src/core/usage/timeline.ts @@ -0,0 +1,81 @@ +/** + * Step functions over time (ADR 0016 §3): each key (a worker, or the one queue) holds its latest + * value until it steps. The window's start is told by the step in force there, which is the last + * one at or before it; where there is none, the time before the first step is unknown, not zero. + */ +export interface Step { + readonly at: number; + readonly key: string; + readonly value: Value; +} + +export interface Segment { + readonly start: number; + readonly end: number; + /** The value each key holds throughout the segment; empty before any key has stepped. */ + readonly values: readonly Value[]; +} + +/** The segments of `[from, to]`, one for each stretch between steps. `steps` is in time order. */ +export function segmentsOf( + steps: readonly Step[], + from: number, + to: number, +): Segment[] { + const current = new Map(); + let index = 0; + for (; index < steps.length && (steps[index] as Step).at <= from; index += 1) { + const step = steps[index] as Step; + current.set(step.key, step.value); + } + const segments: Segment[] = []; + let start = from; + for (; index < steps.length; index += 1) { + const step = steps[index] as Step; + if (step.at >= to) break; + segments.push({ end: step.at, start, values: [...current.values()] }); + current.set(step.key, step.value); + start = step.at; + } + segments.push({ end: to, start, values: [...current.values()] }); + return segments; +} + +/** The values in force at each of `times`, which ascend. A step at a time is already in force. */ +export function valuesAt( + steps: readonly Step[], + times: readonly number[], +): (readonly Value[])[] { + const current = new Map(); + let index = 0; + return times.map((time) => { + for (; index < steps.length && (steps[index] as Step).at <= time; index += 1) { + const step = steps[index] as Step; + current.set(step.key, step.value); + } + return [...current.values()]; + }); +} + +/** + * The peak and the time-weighted mean of what `read` takes from each segment, over the stretches + * where it is known. `undefined` when no stretch with time in it is known. + */ +export function peakAndMean( + segments: readonly Segment[], + read: (values: readonly Value[]) => number | undefined, +): { readonly peak: number; readonly mean: number } | undefined { + let peak: number | undefined; + let weighted = 0; + let known = 0; + for (const segment of segments) { + const duration = segment.end - segment.start; + if (duration <= 0) continue; + const value = read(segment.values); + if (value === undefined) continue; + peak = Math.max(peak ?? value, value); + weighted += value * duration; + known += duration; + } + return peak === undefined ? undefined : { mean: weighted / known, peak }; +} diff --git a/src/core/usage/usage-reader.test.ts b/src/core/usage/usage-reader.test.ts new file mode 100644 index 00000000..59903b83 --- /dev/null +++ b/src/core/usage/usage-reader.test.ts @@ -0,0 +1,316 @@ +import { describe, expect, it } from "vitest"; + +import type { EventEnvelope, EventName } from "../../bus/index.js"; +import { tokenLabelMap, UsageReader, type UsageHistory } from "./usage-reader.js"; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const T0 = 10 * HOUR; + +function requested( + timestamp: number, + requester: string, + requestId = `req_${timestamp}`, +): EventEnvelope { + return { + event: "lease.requested", + id: `evt_${requestId}`, + module: "test", + payload: { requestId, requestSpec: { platform: "ios" }, requester, waitPolicy: "wait" }, + seq: timestamp, + timestamp, + } as unknown as EventEnvelope; +} + +/** A history that records what it was asked for and answers what the test sets. */ +function history( + events: readonly EventEnvelope[] = [], + oldestTs: number | undefined = undefined, + before: { requested?: readonly string[]; answered?: readonly string[] } = {}, +) { + const state = { + asked: [] as { sinceTs: number; carry: readonly EventName[] }[], + events, + newest: "evt_1" as string | undefined, + oldestTs, + }; + const source: UsageHistory = { + latestId: () => state.newest, + read: async (input) => { + state.asked.push(input); + return { + answeredBefore: new Set(before.answered), + events: state.events, + oldestTs: state.oldestTs, + requestedBefore: new Set(before.requested), + }; + }, + }; + return { source, state }; +} + +function reader( + source: UsageHistory, + extra: Partial[0]> = {}, +) { + const calls = { labels: 0 }; + const usage = new UsageReader({ + history: source, + tokenLabels: async () => { + calls.labels += 1; + return tokenLabelMap([{ id: "tok_a", label: "ci-bot" }, { id: "tok_plain" }]); + }, + workers: () => [{ id: "self" }], + ...extra, + }); + return { calls, usage }; +} + +const answer = (result: Awaited>) => { + if (!("usage" in result)) throw new Error("expected an answer"); + return result.usage; +}; + +describe("UsageReader", () => { + it("rounds the window down to the bucket on both sides, so neither end moves later than asked, and reads the history from its start, with the steps, the requests and a gateway start carried", async () => { + const { source, state } = history(); + const { usage } = reader(source); + + const result = answer(await usage.get({ from: T0 + 10_000, to: T0 + HOUR + 10_000 })); + + expect(result.window).toEqual({ from: T0, to: T0 + HOUR }); + expect(result.bucketMs).toBe(MINUTE); + expect(state.asked).toEqual([ + { + carry: ["capacity.changed", "daemon.started", "lease.requested", "queue.changed"], + sinceTs: T0, + }, + ]); + }); + + it("computes with the series bucket picked from the span asked, not from the rounded window", async () => { + const { usage } = reader(history().source); + + const result = answer(await usage.get({ from: T0 + 1_000, to: T0 + 200 * MINUTE + 31_000 })); + + expect(result.window).toEqual({ from: T0, to: T0 + 200 * MINUTE }); + expect(result.bucketMs).toBe(5 * MINUTE); + expect(result.series).toHaveLength(40); + }); + + it("counts no rejection of a request the history says was made before the window, and one of a request it does not say so about", async () => { + const refused = { + event: "lease.rejected", + id: "evt_rej", + module: "test", + payload: { reason: "already-leased", requestId: "old", requester: "a" }, + seq: 1, + timestamp: T0 + 1_000, + } as unknown as EventEnvelope; + const known = reader(history([refused], undefined, { requested: ["old"] }).source).usage; + const unknown = reader(history([refused]).source).usage; + + expect(answer(await known.get({ from: T0, to: T0 + HOUR })).totals.rejected.total).toBe(0); + expect(answer(await unknown.get({ from: T0, to: T0 + HOUR })).totals.rejected.total).toBe(1); + }); + + it("counts a request made before the window as waiting at its start unless the history says it was answered by then", async () => { + const events = [requested(T0 - 30_000, "a", "waits"), requested(T0 - 20_000, "b", "done")]; + const { usage } = reader(history(events, undefined, { answered: ["done"] }).source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result.series[0]?.waiting).toBe(1); + }); + + it("leaves a window that is already on bucket edges as it is", async () => { + const { source } = history(); + const { usage } = reader(source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result.window).toEqual({ from: T0, to: T0 + HOUR }); + }); + + it("refuses a window that ends before the oldest held event, with that time, and answers one that ends exactly at it", async () => { + const { source } = history([], T0 + HOUR + 1); + const { usage } = reader(source); + const edge = reader(history([], T0 + HOUR).source).usage; + + await expect(usage.get({ from: T0, to: T0 + HOUR })).resolves.toEqual({ + oldestTs: T0 + HOUR + 1, + }); + answer(await edge.get({ from: T0, to: T0 + HOUR })); + }); + + it("answers an empty history for any window, as covering all of it", async () => { + const { usage } = reader(history().source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result).toMatchObject({ coversFrom: T0, partial: false }); + }); + + it("says the figures are partial when the history it read starts inside the window", async () => { + const { usage } = reader(history([requested(T0 + 5 * MINUTE, "a")], T0 + 5 * MINUTE).source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result).toMatchObject({ coversFrom: T0 + 5 * MINUTE, partial: true }); + expect(result.totals.requests).toBe(1); + }); + + it("says the figures cover the window when the history reaches before it, though no event falls in its first minutes", async () => { + const { usage } = reader(history([requested(T0 + 30 * MINUTE, "a")], T0 - HOUR).source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result).toMatchObject({ coversFrom: T0, partial: false }); + }); + + it("keeps its answer while the window and the newest event are the same, and reads again when either changes", async () => { + const { source, state } = history(); + const { usage } = reader(source); + const ask = (shift: number) => usage.get({ from: T0 + shift, to: T0 + HOUR + shift }); + + const first = await ask(1_000); + const again = await ask(2_000); + expect(state.asked).toHaveLength(1); + expect(answer(again)).toBe(answer(first)); + + state.newest = undefined; + await ask(1_000); + expect(state.asked).toHaveLength(2); + state.newest = "evt_9"; + await ask(1_000); + expect(state.asked).toHaveLength(3); + await ask(1_000 + MINUTE); + expect(state.asked).toHaveLength(4); + // Each end of the window is part of the key. + await usage.get({ from: T0 + 1_000 + MINUTE, to: T0 + HOUR + 1_000 + 2 * MINUTE }); + expect(state.asked).toHaveLength(5); + }); + + it("does not keep a refusal: a history that has grown to cover the window answers it", async () => { + const { source, state } = history([], T0 + 2 * HOUR); + const { usage } = reader(source); + + await expect(usage.get({ from: T0, to: T0 + HOUR })).resolves.toEqual({ + oldestTs: T0 + 2 * HOUR, + }); + state.oldestTs = T0 - HOUR; + answer(await usage.get({ from: T0, to: T0 + HOUR })); + expect(state.asked).toHaveLength(2); + }); + + it("puts the token label beside each requester the token store knows, and no label beside one it does not", async () => { + const { source } = history([ + requested(T0 + 1_000, "tok_a"), + requested(T0 + 2_000, "tok_plain"), + requested(T0 + 3_000, "stranger"), + ]); + const { usage } = reader(source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(Object.fromEntries(result.requesters.map((entry) => [entry.id, entry.label]))).toEqual({ + stranger: undefined, + tok_a: "ci-bot", + tok_plain: undefined, + }); + }); + + it("reads no labels when the events name no requester", async () => { + const { source } = history([ + { + event: "daemon.stopping", + id: "evt_x", + module: "d", + payload: { reason: "x" }, + seq: 1, + timestamp: T0 + 1, + } as EventEnvelope, + ]); + const { calls, usage } = reader(source); + + answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(calls.labels).toBe(0); + }); + + it("strips its own prefix before the label lookup on a gateway, and leaves another prefix and a worker's own ids alone", async () => { + const events = [ + requested(T0 + 1_000, "gw:g1:tok_a"), + requested(T0 + 2_000, "gw:g2:tok_a"), + requested(T0 + 3_000, "tok_a"), + ]; + const gateway = reader(history(events).source, { fleet: { requesterPrefix: "gw:g1:" } }); + const worker = reader(history(events).source); + + const onGateway = answer(await gateway.usage.get({ from: T0, to: T0 + HOUR })); + const onWorker = answer(await worker.usage.get({ from: T0, to: T0 + HOUR })); + + const labels = (usage: typeof onGateway) => + Object.fromEntries(usage.requesters.map((entry) => [entry.id, entry.label])); + expect(labels(onGateway)).toEqual({ + "gw:g1:tok_a": "ci-bot", + "gw:g2:tok_a": undefined, + tok_a: "ci-bot", + }); + expect(labels(onWorker)).toEqual({ + "gw:g1:tok_a": undefined, + "gw:g2:tok_a": undefined, + tok_a: "ci-bot", + }); + }); + + it("counts as the fleet when it is given a prefix: a worker's relayed request is not a request", async () => { + const relayed = { + ...requested(T0 + 1_000, "gw:g1:tok_a"), + payload: { + requestId: "w_1", + requestSpec: { platform: "ios" }, + requester: "gw:g1:tok_a", + waitPolicy: "no-wait", + workerId: "wrk_1", + }, + } as unknown as EventEnvelope; + const events = [requested(T0 + 500, "tok_a", "own_1"), relayed]; + const gateway = reader(history(events).source, { fleet: { requesterPrefix: "gw:g1:" } }); + const worker = reader(history(events).source); + + const onGateway = answer(await gateway.usage.get({ from: T0, to: T0 + HOUR })); + const onWorker = answer(await worker.usage.get({ from: T0, to: T0 + HOUR })); + + expect(onGateway.totals.requests).toBe(1); + expect(onWorker.totals.requests).toBe(2); + }); + + it("reports the workers it is given, with their labels, read afresh for each answer", async () => { + const { source, state } = history(); + let workers = [{ id: "wrk_1", label: "mac" }]; + const { usage } = reader(source, { workers: () => workers }); + + const first = answer(await usage.get({ from: T0, to: T0 + HOUR })); + workers = [ + { id: "wrk_1", label: "mac" }, + { id: "wrk_2", label: "mini" }, + ]; + state.newest = "evt_2"; + const second = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(first.workers.map((worker) => [worker.id, worker.label])).toEqual([["wrk_1", "mac"]]); + expect(second.workers.map((worker) => worker.id)).toEqual(["wrk_1", "wrk_2"]); + }); +}); + +describe("tokenLabelMap", () => { + it("maps each labelled token's id to its label and leaves out one with none", () => { + expect(tokenLabelMap([{ id: "a", label: "x" }, { id: "b" }, { id: "c", label: "z" }])).toEqual( + new Map([ + ["a", "x"], + ["c", "z"], + ]), + ); + }); +}); diff --git a/src/core/usage/usage-reader.ts b/src/core/usage/usage-reader.ts new file mode 100644 index 00000000..69091ca5 --- /dev/null +++ b/src/core/usage/usage-reader.ts @@ -0,0 +1,123 @@ +import type { EventEnvelope, EventName } from "../../bus/index.js"; +import type { UsageOutput } from "../../contract/index.js"; +import { computeUsage, seriesBucketMs, type UsageWindow } from "./compute-usage.js"; + +/** The events whose latest envelope at or before the window's start is read with it: the two + * steps in force (ADR 0016 §3), the latest request of each requester, which with the ids of the + * requests answered by then tells what was still waiting at the start, and a gateway's latest + * start, which ends the fleet requests it lost (ADR 0021 §5). */ +const CARRIED_EVENTS: readonly EventName[] = [ + "capacity.changed", + "daemon.started", + "lease.requested", + "queue.changed", +]; + +/** The part of the event history `usage.get` reads (`EventHistory` satisfies it). */ +export interface UsageHistory { + read(input: { readonly sinceTs: number; readonly carry: readonly EventName[] }): Promise<{ + readonly events: readonly EventEnvelope[]; + readonly oldestTs: number | undefined; + /** The `requestId` of every `lease.requested` at or before `sinceTs`. */ + readonly requestedBefore: ReadonlySet; + /** The `requestId` of every request answered at or before `sinceTs`. */ + readonly answeredBefore: ReadonlySet; + }>; + /** The id of the event published last; the memo is good while it has not moved. */ + latestId(): string | undefined; +} + +export interface UsageReaderOptions { + readonly history: UsageHistory; + /** Token id to label, from the store the answering daemon owns (ADR 0016 §7). */ + readonly tokenLabels: () => Promise>; + /** The workers to report: a worker's own entry, a gateway's registry. Read on each answer. */ + readonly workers: () => readonly { readonly id: string; readonly label?: string | undefined }[]; + /** Set on a gateway: the figures are the fleet's (ADR 0016 §6, ADR 0021 §5), and this is the + * prefix it stamps on the requesters it forwards (`gw::`), which is taken off before a + * token label is looked up. */ + readonly fleet?: { readonly requesterPrefix: string }; +} + +/** What `get` answers: the figures, or how far back the history reaches when it is not far enough. */ +export type UsageResult = { readonly usage: UsageOutput } | { readonly oldestTs: number }; + +/** + * Answers `usage.get` for a worker or a gateway from the same code (ADR 0016): reads the history + * for the window, hands it to `computeUsage`, joins token labels. The window is rounded down to the + * series bucket, both ends, so neither end moves later than asked, and the last answer is kept by + * that window and the newest event, so a caller that asks again within the bucket while nothing + * happened reads nothing. + */ +export class UsageReader { + #memo: { readonly key: string; readonly usage: UsageOutput } | undefined; + + constructor(private readonly options: UsageReaderOptions) {} + + async get(asked: UsageWindow): Promise { + const bucketMs = seriesBucketMs(asked.to - asked.from); + // Down to the bucket on both sides (ADR 0016 §8): neither end moves later than asked. + const window = { + from: Math.floor(asked.from / bucketMs) * bucketMs, + to: Math.floor(asked.to / bucketMs) * bucketMs, + }; + const key = `${window.from}:${window.to}:${String(this.options.history.latestId())}`; + if (this.#memo?.key === key) return { usage: this.#memo.usage }; + + const { answeredBefore, events, oldestTs, requestedBefore } = await this.options.history.read({ + carry: CARRIED_EVENTS, + sinceTs: window.from, + }); + if (oldestTs !== undefined && oldestTs > asked.to) return { oldestTs }; + const usage = computeUsage(events, window, { + answeredBefore, + bucketMs, + fleet: this.options.fleet !== undefined, + labels: await this.#labelsFor(events), + oldestTs, + requestedBefore, + workers: this.options.workers(), + }); + this.#memo = { key, usage }; + return { usage }; + } + + /** The label of each requester the events name that the token store knows (ADR 0016 §7). */ + async #labelsFor(events: readonly EventEnvelope[]): Promise> { + const prefix = this.options.fleet?.requesterPrefix; + const requesters = requestersIn(events); + if (requesters.size === 0) return {}; + const tokens = await this.options.tokenLabels(); + const labels: Record = {}; + for (const requester of requesters) { + const own = prefix !== undefined && requester.startsWith(prefix); + const label = tokens.get(own ? requester.slice(prefix.length) : requester); + if (label !== undefined) labels[requester] = label; + } + return labels; + } +} + +/** Every requester id the events name, as a request, a rejection or a dispatch has it. */ +function requestersIn(events: readonly EventEnvelope[]): Set { + const requesters = new Set(); + for (const event of events) { + const payload = event.payload as { + readonly requester?: unknown; + readonly requesterId?: unknown; + }; + for (const requester of [payload.requester, payload.requesterId]) { + if (typeof requester === "string") requesters.add(requester); + } + } + return requesters; +} + +/** The labelled tokens of a store as the reader takes them: token id to label. */ +export function tokenLabelMap( + tokens: readonly { readonly id: string; readonly label?: string | undefined }[], +): Map { + return new Map( + tokens.flatMap((token) => (token.label === undefined ? [] : [[token.id, token.label]])), + ); +} diff --git a/src/daemon/dispatch.ts b/src/daemon/dispatch.ts index 106703a5..e6b27d2f 100644 --- a/src/daemon/dispatch.ts +++ b/src/daemon/dispatch.ts @@ -26,6 +26,7 @@ import { type OperationDefinition, type OperationName, type Role, + type UsageOutput, type leaseProgressSchema, } from "../contract/index.js"; @@ -390,3 +391,21 @@ function parseDispatchOutput( `Internal: ${operationName} produced a response that does not match its contract output schema`, ); } + +/** + * What a `usage.get` handler answers with: the figures, or `HISTORY_NOT_KEPT` carrying the oldest + * time the history reaches when the window ends before it (ADR 0016 §5). Both dispatchers read + * their answer through here, so the one message and the one `details` shape exist once. + */ +export function usageAnswer( + result: { readonly usage: UsageOutput } | { readonly oldestTs: number }, +): UsageOutput { + if ("usage" in result) return result.usage; + const oldest = new Date(result.oldestTs); + const when = Number.isNaN(oldest.getTime()) ? String(result.oldestTs) : oldest.toISOString(); + throw new DispatchError( + "HISTORY_NOT_KEPT", + `The event history does not reach back to the end of that window; its oldest event is from ${when}.`, + { oldestTs: result.oldestTs }, + ); +} diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 4c7897f2..601a95e4 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -94,8 +94,8 @@ function stallOptions( function resolveEventHistoryOverride( eventBus: EventBus, filesystem: MemoryFilesystem, - override: Pick | undefined, -): Pick { + override: Pick | undefined, +): Pick { return ( override ?? new EventHistory({ bus: eventBus, filesystem, logger: new NoopLogger(), path: "/events.jsonl" }) @@ -192,7 +192,7 @@ async function buildDispatcher( * takes no `--set`), rather than `ScriptedProcessRunner`'s scripted chunks. */ readonly passthroughOverride?: PassthroughResolver; /** Stands in for the event history, so `events.replay` can be checked against it. */ - readonly eventHistory?: Pick; + readonly eventHistory?: Pick; /** `status.get`'s host block; a fixed machine with no tools by default. */ readonly hostFacts?: () => HostFacts; /** Replaces the capacity block, for a test about one strategy's options. */ @@ -205,6 +205,8 @@ async function buildDispatcher( ComponentInstaller, "claimProvision" | "inProgress" | "install" | "list" | "remove" >; + /** Leaves the token store out, as a daemon started without one is. */ + readonly withoutTokens?: boolean; /** The daemon's health, as `status.get` reports it; `running` by default. */ readonly health?: "starting" | "running" | "failed"; /** `gateway.label` in this daemon's config; unset by default. */ @@ -322,7 +324,7 @@ async function buildDispatcher( reaper, registry, stalls: stallOptions(engine, driver, overrides), - tokens, + ...tokenOption(overrides.withoutTokens, tokens), version: "1.2.3", }); return { @@ -338,6 +340,11 @@ async function buildDispatcher( }; } +/** The `tokens` option, left out for a daemon started without a store. */ +function tokenOption(without: boolean | undefined, tokens: TokenStore): { tokens?: TokenStore } { + return without === true ? {} : { tokens }; +} + function session(overrides: Partial = {}): DispatchSession { return { manageEventSubscription: () => undefined, @@ -1188,6 +1195,7 @@ describe("Dispatcher: events.replay", () => { const asked: unknown[] = []; const { dispatcher } = await buildDispatcher({ eventHistory: { + ...noUsageHistory, replay: async (input) => { asked.push(input); return fromHistory; @@ -1202,6 +1210,183 @@ describe("Dispatcher: events.replay", () => { }); }); +/** The parts of the history `usage.get` reads, for a test that is not about it. */ +const noUsageHistory = { + latestId: () => undefined, + read: async () => ({ + events: [], + oldestTs: undefined, + answeredBefore: new Set(), + requestedBefore: new Set(), + }), +}; + +describe("Dispatcher: usage.get", () => { + const HOUR = 3_600_000; + const WINDOW = { from: 10 * HOUR, to: 11 * HOUR }; + + /** A history that counts its reads and whose newest event the test moves. */ + function countingHistory(oldestTs: number | undefined = undefined) { + const state = { newest: "evt_1" as string | undefined, reads: 0 }; + const history = { + latestId: () => state.newest, + read: async () => { + state.reads += 1; + return { + events: [], + oldestTs, + answeredBefore: new Set(), + requestedBefore: new Set(), + }; + }, + replay: async () => [], + }; + return { history, state }; + } + + it("the worker handler answers a second call inside the same bucket from the memo without reading the history, and reads again after an event arrives or the bucket moves", async () => { + const { history, state } = countingHistory(); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + const call = (window: { from: number; to: number }) => + dispatcher.dispatch("usage.get", window, session({ role: "admin" })); + const minute = 60_000; + const asked = { from: WINDOW.from + 10_000, to: WINDOW.to + 10_000 }; + + const first = await call(asked); + const second = await call({ from: asked.from + 20_000, to: asked.to + 20_000 }); + + expect(state.reads).toBe(1); + expect(second).toEqual(first); + // The answer's window is the window asked for rounded down to the bucket, so never later than asked. + expect(second.window).toEqual({ from: WINDOW.from, to: WINDOW.to }); + + state.newest = "evt_2"; + await call(asked); + expect(state.reads).toBe(2); + + await call({ from: asked.from + minute, to: asked.to + minute }); + expect(state.reads).toBe(3); + }); + + it("the worker handler answers HISTORY_NOT_KEPT with the oldest held timestamp when the window ends before it", async () => { + const { history } = countingHistory(WINDOW.to + 5 * HOUR); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch("usage.get", WINDOW, session({ role: "admin" })), + ).rejects.toMatchObject({ + code: "HISTORY_NOT_KEPT", + details: { oldestTs: WINDOW.to + 5 * HOUR }, + }); + }); + + it("the worker handler names an oldest timestamp no date can hold as the number in HISTORY_NOT_KEPT's message", async () => { + const unreadable = 9e15; + const { history } = countingHistory(unreadable); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch("usage.get", WINDOW, session({ role: "admin" })), + ).rejects.toMatchObject({ + code: "HISTORY_NOT_KEPT", + details: { oldestTs: unreadable }, + message: expect.stringContaining("its oldest event is from 9000000000000000."), + }); + }); + + it("the worker handler answers for itself as one worker, with the token label beside a requester it knows", async () => { + const { dispatcher, eventBus, tokens } = await buildDispatcher(); + const { record } = await tokens.create("agent", "ci-bot"); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: record.id, + waitPolicy: "wait", + }, + "test", + ); + + const usage = await dispatcher.dispatch( + "usage.get", + { from: 0, to: 600_000 }, + session({ role: "admin" }), + ); + + expect(usage.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: record.id, label: "ci-bot", rejected: 0, requests: 1 }, + ]); + expect(usage.workers.map((worker) => worker.id)).toEqual(["instance-1"]); + expect(usage.totals.requests).toBe(1); + }); + + it("the worker handler answers with no label for a requester when the daemon has no token store", async () => { + const { dispatcher, eventBus } = await buildDispatcher({ withoutTokens: true }); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: "tok_x", + waitPolicy: "wait", + }, + "test", + ); + + const usage = await dispatcher.dispatch( + "usage.get", + { from: 0, to: 600_000 }, + session({ role: "admin" }), + ); + + expect(usage.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: "tok_x", rejected: 0, requests: 1 }, + ]); + }); + + it("the worker handler's HISTORY_NOT_KEPT message names when the history begins", async () => { + const oldestTs = Date.parse("2026-10-05T10:00:00.000Z"); + const { history } = countingHistory(oldestTs); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch( + "usage.get", + { from: oldestTs - 2 * HOUR, to: oldestTs - HOUR }, + session({ role: "admin" }), + ), + ).rejects.toMatchObject({ + message: + "The event history does not reach back to the end of that window; its oldest event is from 2026-10-05T10:00:00.000Z.", + }); + }); + + it("usage.get is an admin operation an agent token cannot call", async () => { + const { history, state } = countingHistory(); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch("usage.get", WINDOW, session({ role: "agent" })), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(state.reads).toBe(0); + }); + + it("refuses a window that is not ordered or is longer than ninety days as a bad request", async () => { + const { dispatcher } = await buildDispatcher(); + const call = (window: unknown) => + dispatcher.dispatch("usage.get", window, session({ role: "admin" })); + + await expect(call({ from: 5, to: 5 })).rejects.toMatchObject({ code: "BAD_REQUEST" }); + await expect(call({ from: 0, to: 91 * 24 * HOUR })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + await expect(call({ from: 0, to: 90 * 24 * HOUR })).resolves.toMatchObject({ + window: { from: 0 }, + }); + }); +}); + describe("Dispatcher: nuke.run", () => { it("rejects an agent session with FORBIDDEN, without deleting anything", async () => { const { dispatcher, registry } = await buildDispatcher({ includeNuke: true }); diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index 3a0d5920..6e971b77 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -21,6 +21,8 @@ import { RuntimeMissingError, transitionEnteredAt, UnknownLeaseError, + tokenLabelMap, + UsageReader, type CapacityReader, type WarmPoolReader, type CatalogReader, @@ -57,6 +59,7 @@ import { runDispatch, type DispatchSession, type ErasedHandler, + usageAnswer, } from "./dispatch.js"; export { DispatchError, type ContractDispatcher, type DispatchSession } from "./dispatch.js"; @@ -108,7 +111,7 @@ export interface DispatcherOptions { readonly config: Config; readonly doctor?: Doctor; /** Answers `events.replay`: the ring, or the event file for a `sinceTs`. */ - readonly eventHistory: Pick; + readonly eventHistory: Pick; /** Answers whether a device's pool is the one a request naming no mode draws from. */ readonly deviceModes: DeviceModeReader; readonly leases: LeaseCommands; @@ -239,9 +242,16 @@ export class Dispatcher { #viewCatalog: { readonly readAt: number; readonly catalog: Promise } | undefined; /** Ends the bus subscriptions that drop `#viewCatalog`; see `dispose`. */ readonly #unsubscribe: (() => void)[] = []; + /** Answers `usage.get` from the event history, as this host's own entry (ADR 0016, ADR 0012). */ + readonly #usage: UsageReader; constructor(private readonly options: DispatcherOptions) { this.#logger = options.logger ?? new NoopLogger(); + this.#usage = new UsageReader({ + history: options.eventHistory, + tokenLabels: async () => tokenLabelMap((await options.tokens?.list()) ?? []), + workers: () => [{ id: options.instanceId, label: options.config.gateway.label }], + }); // Observers only (architecture rule 5): dropping a kept read decides nothing. for (const event of WORKER_VIEW_CATALOG_EVENTS) { const unsubscribe = options.eventBus?.subscribe(event, () => { @@ -267,6 +277,7 @@ export class Dispatcher { "nuke.run": this.#nukeRun, "config.get": this.#configGet, "events.replay": this.#eventsReplay, + "usage.get": this.#usageGet, "events.subscribe": this.#eventsSubscribe, "events.unsubscribe": this.#eventsUnsubscribe, "token.create": this.#tokenCreate, @@ -665,6 +676,8 @@ export class Dispatcher { #eventsReplay: Handler<"events.replay"> = (input) => this.options.eventHistory.replay(input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }); + #usageGet: Handler<"usage.get"> = async (input) => usageAnswer(await this.#usage.get(input)); + #eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { const subscriptionId = session.manageEventSubscription(true); if (subscriptionId === undefined) { diff --git a/src/daemon/server.test.ts b/src/daemon/server.test.ts index f1169a5b..eab2d47b 100644 --- a/src/daemon/server.test.ts +++ b/src/daemon/server.test.ts @@ -438,6 +438,19 @@ describe("DaemonServer", () => { expect(harness.registry.snapshot.leases).toEqual([]); }); + it("routes usage.get to the dispatcher with the window the frame carries", async () => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + const answer = await client.request("usage.get", { from: 0, to: 3_600_000 }); + + expect(answer).toMatchObject({ + ok: true, + payload: { totals: { requests: 0 }, window: { from: 0, to: 3_600_000 } }, + }); + }); + it("serves the device catalog, omitting platforms with no registered driver", async () => { const harness = await createHarness(); const client = await createClient(harness.socketPath); diff --git a/src/daemon/server.ts b/src/daemon/server.ts index fa338e94..fd9e3242 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -150,7 +150,7 @@ export interface DaemonServerEngineOptions { readonly components: Pick; readonly doctor?: Doctor; /** What `events.replay` answers from; see `EventHistory`. */ - readonly eventHistory: Pick; + readonly eventHistory: Pick; readonly leases: LeaseCommands; /** `status.get`'s host block (ADR 0008 §5); see `DispatcherOptions.hostFacts`. */ readonly hostFacts: () => HostFacts; @@ -1105,6 +1105,12 @@ export class DaemonServer { frame.payload ?? {}, this.#session(connection), ); + case "usage.get": + return this.#dispatcher.dispatch( + "usage.get", + frame.payload ?? {}, + this.#session(connection), + ); case "events.subscribe": return this.#dispatcher.dispatch( "events.subscribe", diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 2b3e86cd..f74450ac 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -79,6 +79,8 @@ const gatewayConfig = { }; class FakeTokens implements GatewayTokenStore { + constructor(private readonly records: readonly { id: string; label?: string }[] = []) {} + readonly created: string[] = []; readonly revoked: string[] = []; @@ -91,7 +93,10 @@ class FakeTokens implements GatewayTokenStore { } async list() { - return [{ createdAt: 1, id: "tok_1", role: "worker" as const }]; + return [ + { createdAt: 1, id: "tok_1", role: "worker" as const }, + ...this.records.map((record) => ({ createdAt: 1, role: "agent" as const, ...record })), + ]; } async revoke(id: string) { @@ -127,7 +132,11 @@ class FakeDirectory implements WorkerDirectory { function harness( options: { - readonly eventHistory?: Pick; + readonly eventHistory?: Pick; + /** The tokens the gateway's store lists besides its worker token; none by default. */ + readonly tokenRecords?: readonly { id: string; label?: string }[]; + /** Leaves the token store out, as a gateway started without one is. */ + readonly withoutTokens?: boolean; /** The gateway's `http` block; enabled on 127.0.0.1:4700 by default. */ readonly http?: (typeof gatewayConfig)["http"]; /** The gateway's own health; `running` by default. */ @@ -144,7 +153,7 @@ function harness( leaseMaxTtlMs: gatewayConfig.lease.maxTtlMs, retentionMs: 24 * 60 * 60_000, }); - const tokens = new FakeTokens(); + const tokens = new FakeTokens(options.tokenRecords); /** C-2: every id `closeUplinksForToken` was actually called with, in call order. */ const closedUplinkTokens: string[] = []; const directory = new FakeDirectory(); @@ -193,7 +202,7 @@ function harness( // `classifyError`'s answer for the errors this dispatcher throws itself. errorCode: (error) => (error instanceof DispatchError ? error.code : undefined), logger: new JsonLinesLogger({ clock, module: "gateway", sink: logSink }), - tokens, + ...(options.withoutTokens === true ? {} : { tokens }), workers, }); return { @@ -685,6 +694,13 @@ describe("GatewayDispatcher", () => { const asked: unknown[] = []; const { dispatcher } = harness({ eventHistory: { + latestId: () => undefined, + read: async () => ({ + events: [], + oldestTs: undefined, + answeredBefore: new Set(), + requestedBefore: new Set(), + }), replay: async (input) => { asked.push(input); return fromHistory; @@ -698,6 +714,124 @@ describe("GatewayDispatcher", () => { expect(asked).toEqual([{ sinceTs: 10 }]); }); + it("the gateway handler strips its own gw: prefix before the label lookup and leaves another prefix alone", async () => { + const { dispatcher, eventBus } = harness({ + tokenRecords: [{ id: "tok_a", label: "ci-bot" }], + }); + for (const [index, requester] of [ + `${GATEWAY_REQUESTER_PREFIX}tok_a`, + "gw:other-instance:tok_a", + "tok_a", + ].entries()) { + eventBus.emit( + "lease.requested", + { + requestId: `req_${index}`, + requestSpec: { platform: "ios" }, + requester, + waitPolicy: "wait", + }, + "gateway", + ); + } + + const usage = await dispatcher.dispatch("usage.get", { from: 0, to: 600_000 }, session()); + + expect( + Object.fromEntries(usage.requesters.map((requester) => [requester.id, requester.label])), + ).toEqual({ + "gw:instance-1:tok_a": "ci-bot", + "gw:other-instance:tok_a": undefined, + tok_a: "ci-bot", + }); + }); + + it("the gateway handler answers with no label for a requester when it has no token store", async () => { + const { dispatcher, eventBus } = harness({ withoutTokens: true }); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: "tok_x", + waitPolicy: "wait", + }, + "gateway", + ); + + const usage = await dispatcher.dispatch("usage.get", { from: 0, to: 600_000 }, session()); + + expect(usage.requesters.map((requester) => requester.label)).toEqual([undefined]); + expect(usage.requesters).toHaveLength(1); + }); + + it("answers usage.get with fleet totals and one entry per worker, labelled from the registry", async () => { + const { dispatcher, eventBus, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: "agent-1", + waitPolicy: "wait", + }, + "gateway", + ); + eventBus.republish({ + event: "lease.granted", + id: "evt_worker_1", + module: "lease-lifecycle", + payload: { + deviceId: "dev_1", + leaseId: "l1", + requestId: "wreq_1", + requester: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + source: "warm", + workerId: "wrk_1", + } as never, + timestamp: 1_000, + }); + + eventBus.emit( + "request.granted", + { leaseId: "gwl_1", requestId: "req_1", worker: "wrk_1", workerLeaseId: "l1" }, + "gateway", + ); + + const usage = await dispatcher.dispatch("usage.get", { from: 0, to: 600_000 }, session()); + + expect(usage.totals).toMatchObject({ bySource: { warm: 1 }, granted: 1, requests: 1 }); + expect(usage.workers).toMatchObject([{ granted: 1, id: "wrk_1", label: "mac-mini-1" }]); + }); + + it("the gateway handler answers HISTORY_NOT_KEPT with the oldest held timestamp, and refuses an agent token before it reads any history", async () => { + let reads = 0; + const { dispatcher } = harness({ + eventHistory: { + latestId: () => "evt_1", + read: async () => { + reads += 1; + return { + events: [], + oldestTs: 99 * 3_600_000, + answeredBefore: new Set(), + requestedBefore: new Set(), + }; + }, + replay: async () => [], + }, + }); + + await expect( + dispatcher.dispatch("usage.get", { from: 0, to: 3_600_000 }, session()), + ).rejects.toMatchObject({ code: "HISTORY_NOT_KEPT", details: { oldestTs: 99 * 3_600_000 } }); + await expect( + dispatcher.dispatch("usage.get", { from: 0, to: 3_600_000 }, session({ role: "agent" })), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(reads).toBe(1); + }); + it("mints and revokes its own tokens, worker join tokens included", async () => { const { dispatcher, tokens } = harness(); diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 06a5e161..3f33336c 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -46,9 +46,11 @@ import { runDispatch, type DispatchSession, type ErasedHandler, + usageAnswer, } from "../daemon/dispatch.js"; import type { Clock, Logger } from "../ports/index.js"; import { NoopLogger } from "../ports/index.js"; +import { tokenLabelMap, UsageReader } from "../core/index.js"; import { aggregateCatalog, aggregateStatus, type AggregateStatusOptions } from "./aggregate.js"; import { relayComponentInstall } from "./component-relay.js"; import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; @@ -96,7 +98,7 @@ export interface GatewayDispatcherOptions { /** The gateway's own config -- what `config.get` returns (ADR 0005 §34). */ readonly config: GatewayConfig; /** Answers `events.replay`: the ring, or the event file for a `sinceTs`. */ - readonly eventHistory: Pick; + readonly eventHistory: Pick; readonly workers: WorkerRegistry; /** Each worker's live link, for `worker.install-component` (ADR 0010 §7). `GatewayService` * satisfies it. */ @@ -148,22 +150,34 @@ export interface GatewayDispatcherOptions { * (`FleetLeaseIndex#project`/`#all`) -- kept separate from `coordinator` because this * dispatcher only ever *reads* it, never mutates it. `isGatewayRequester` marks the requests * this gateway sent a worker, which `list.get`'s `requests` already lists as its own. */ - readonly leaseIndex: Pick; + readonly leaseIndex: Pick< + FleetLeaseIndex, + "project" | "all" | "isGatewayRequester" | "requesterPrefix" + >; } export class GatewayDispatcher { readonly #logger: Logger; readonly #dispatchLogger: Logger; readonly #handlers: Record, ErasedHandler>; + /** Answers `usage.get` as the fleet, from the gateway's own merged history (ADR 0016 §6). */ + readonly #usage: UsageReader; constructor(private readonly options: GatewayDispatcherOptions) { this.#logger = options.logger ?? new NoopLogger(); + this.#usage = new UsageReader({ + fleet: { requesterPrefix: options.leaseIndex.requesterPrefix }, + history: options.eventHistory, + tokenLabels: async () => tokenLabelMap((await options.tokens?.list()) ?? []), + workers: () => options.workers.views().map((view) => ({ id: view.id, label: view.label })), + }); this.#dispatchLogger = this.#logger.child("dispatch"); this.#handlers = { "catalog.get": this.#catalogGet, "status.get": this.#statusGet, "config.get": this.#configGet, "events.replay": this.#eventsReplay, + "usage.get": this.#usageGet, "events.subscribe": this.#eventsSubscribe, "events.unsubscribe": this.#eventsUnsubscribe, "token.create": this.#tokenCreate, @@ -268,6 +282,8 @@ export class GatewayDispatcher { #eventsReplay: Handler<"events.replay"> = (input) => this.options.eventHistory.replay(input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }); + #usageGet: Handler<"usage.get"> = async (input) => usageAnswer(await this.#usage.get(input)); + #eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { const subscriptionId = session.manageEventSubscription(true); if (subscriptionId === undefined) { diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index d8c937f4..2d074f3e 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -70,6 +70,9 @@ describe("connectSimlock: handshake", () => { await expect(client.runCleanup()).rejects.toMatchObject({ code: "PROTOCOL_VERSION_UNSUPPORTED", }); + await expect(client.usage({ from: 0, to: 60_000 })).rejects.toMatchObject({ + code: "PROTOCOL_VERSION_UNSUPPORTED", + }); expect(connection.sent).toHaveLength(before); const stopPromise = client.stopDaemon(); @@ -219,6 +222,54 @@ describe("connectSimlock: handshake", () => { await expect(callPromise).resolves.toEqual(answer); }); + it("asks usage.get for a window from an admin client and returns the daemon's answer, refusing a window the contract bounds before sending it", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const client = await connectPromise; + const none = { count: 0, max: null, p50: null, p95: null }; + const figures = { + boot: none, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, + failures: { byEvent: {} }, + granted: 0, + held: none, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: none, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: none, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: none, + }; + const answer = { + bucketMs: 60_000, + coversFrom: 1_000, + partial: false, + platforms: { android: figures, ios: figures }, + requesters: [], + series: [], + totals: figures, + window: { from: 1_000, to: 3_601_000 }, + workers: [], + }; + + const callPromise = client.usage({ from: 1_000, to: 3_601_000 }); + await flushMicrotasks(); + const call = connection.lastSentOf("usage.get")!; + expect(call.payload).toEqual({ from: 1_000, to: 3_601_000 }); + connection.reply(call.id, answer); + + await expect(callPromise).resolves.toEqual(answer); + + const sentBefore = connection.sent.length; + await expect(client.usage({ from: 5, to: 5 })).rejects.toMatchObject({ code: "BAD_REQUEST" }); + expect(connection.sent).toHaveLength(sentBefore); + }); + it("wraps a malformed daemon response instead of throwing a raw parse failure", async () => { const connection = new ScriptedConnection(); const connectPromise = connectSimlock({ connection }); diff --git a/src/simlock-client/client.ts b/src/simlock-client/client.ts index 3001d961..9dc94a81 100644 --- a/src/simlock-client/client.ts +++ b/src/simlock-client/client.ts @@ -49,6 +49,8 @@ import type { ExecOptions, EventsReplayInput, EventsReplayOutput, + UsageGetInput, + UsageGetOutput, InstallComponentOnWorkersOptions, InstallComponentOptions, LeaseCancelInput, @@ -109,6 +111,8 @@ export type { ExecOptions, EventsReplayInput, EventsReplayOutput, + UsageGetInput, + UsageGetOutput, EventsSubscribeOutput, EventsUnsubscribeOutput, InstallComponentOnWorkersOptions, @@ -237,6 +241,8 @@ export interface SimlockAdminClient extends SimlockClient { * does not gate this operation by role at all -- see the caveat in the PR report). */ stopDaemon(): Promise; replayEvents(input?: EventsReplayInput): Promise; + /** ADR 0016: the usage figures for a window, computed by the daemon from its event history. */ + usage(window: UsageGetInput): Promise; subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise>; createToken(input: TokenCreateInput): Promise; listTokens(): Promise; @@ -396,6 +402,7 @@ function buildDegradedClient( }, ), replayEvents: () => rejected(), + usage: () => rejected(), subscribeEvents: () => rejected(), createToken: () => rejected(), listTokens: () => rejected(), @@ -575,6 +582,10 @@ class SimlockClientImpl { return this.#call("events.replay", input); } + usage(window: UsageGetInput): Promise { + return this.#call("usage.get", window); + } + async subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise> { const unsubscribePush = this.#wire.onEvent((payload) => listener(payload as EventPush)); const result = await this.#call("events.subscribe", {}); diff --git a/src/simlock-client/types.ts b/src/simlock-client/types.ts index d290c3e6..42f332d9 100644 --- a/src/simlock-client/types.ts +++ b/src/simlock-client/types.ts @@ -55,6 +55,8 @@ export type SimlockConfig = OpOutput<"config.get">; export type DaemonStopOutput = OpOutput<"daemon.stop">; export type EventsReplayInput = OpInput<"events.replay">; export type EventsReplayOutput = OpOutput<"events.replay">; +export type UsageGetInput = OpInput<"usage.get">; +export type UsageGetOutput = OpOutput<"usage.get">; export type EventsSubscribeOutput = OpOutput<"events.subscribe">; export type EventsUnsubscribeOutput = OpOutput<"events.unsubscribe">; export type TokenCreateInput = OpInput<"token.create">;