From 38fd0661bcd57e92ae87fd279c6c09ab1d670fd1 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 16:28:13 +0200 Subject: [PATCH 01/23] test: usage.get and simlock stats: the figures from the event history, on a worker and a gateway (#347) Red: 50 failing. The contract (usage.get, HISTORY_NOT_KEPT, the output schema) is in place; the figures module, the history reads, the handlers, the client method and the CLI command are stubs. --- e2e/usage-stats.test.ts | 235 +++++++++ src/admin/index.ts | 2 + src/bus/event-file.test.ts | 115 +++++ src/bus/event-file.ts | 21 +- src/cli/index.test.ts | 212 +++++++- src/cli/stats.test.ts | 155 ++++++ src/cli/stats.ts | 6 + src/contract/errors.test.ts | 23 +- src/contract/errors.ts | 8 + src/contract/operations.test.ts | 24 + src/contract/operations.ts | 19 + src/contract/schemas.ts | 109 ++++ src/core/usage/compute-usage.test.ts | 735 +++++++++++++++++++++++++++ src/core/usage/compute-usage.ts | 59 +++ src/daemon/dispatcher.test.ts | 117 ++++- src/daemon/dispatcher.ts | 7 +- src/daemon/server.ts | 8 +- src/gateway/dispatcher.test.ts | 117 ++++- src/gateway/dispatcher.ts | 7 +- src/simlock-client/client.test.ts | 47 ++ src/simlock-client/client.ts | 11 + src/simlock-client/types.ts | 2 + 22 files changed, 2024 insertions(+), 15 deletions(-) create mode 100644 e2e/usage-stats.test.ts create mode 100644 src/cli/stats.test.ts create mode 100644 src/cli/stats.ts create mode 100644 src/core/usage/compute-usage.test.ts create mode 100644 src/core/usage/compute-usage.ts diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts new file mode 100644 index 00000000..1f7e5e03 --- /dev/null +++ b/e2e/usage-stats.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; + +import { freeLoopbackPort, waitFor, withDaemon } from "./helpers/index.js"; +import type { RecordedEvent } from "./helpers/events.js"; + +/** + * ADR 0016 with real processes: `simlock stats` reads the figures from the event history, on a + * worker and on a gateway, and the figures are what a reader counts by hand from `simlock + * events` for the same window. + */ + +const LEASE_ARGS = ["lease", "--platform", "ios", "--device", "iPhone 16", "--os", "18.4"] as const; + +interface Samples { + readonly count: number; + readonly max: number | null; + readonly p50: number | null; +} + +interface Figures { + readonly requests: number; + readonly granted: number; + readonly bySource: Record; + readonly rejected: { readonly total: number; readonly byReason: Record }; + readonly wait: Samples; + readonly held: Samples; + readonly turnaround: Samples; + readonly provisioning: Samples; + readonly boot: Samples; + readonly incidents: Record; +} + +interface Usage { + readonly window: { readonly from: number; readonly to: number }; + readonly partial: boolean; + readonly totals: Figures; + readonly workers: readonly (Figures & { readonly id: string; readonly label?: string })[]; + readonly requesters: readonly { readonly id: string; readonly requests: number }[]; +} + +async function stats( + env: { cli: (args: string[]) => Promise<{ code: number | null; json?: unknown }> }, + since = "1h", +): Promise { + const result = await env.cli(["stats", "--since", since, "--json"]); + expect(result.code).toBe(0); + return result.json as Usage; +} + +function countBy(values: readonly string[]): Record { + const counts: Record = {}; + for (const value of values) counts[value] = (counts[value] ?? 0) + 1; + return counts; +} + +function inWindow(events: readonly RecordedEvent[], window: Usage["window"]): RecordedEvent[] { + return events.filter((event) => event.timestamp > window.from && event.timestamp <= window.to); +} + +function payload(event: RecordedEvent): Record { + return event.payload as Record; +} + +describe("simlock stats", () => { + it("after a scripted run of leases against the fake driver, simlock stats --since 1h --json reports the counts a test derives independently from simlock events --since 1h", async () => { + const env = await withDaemon({ + configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }, + }); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + const first = await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"]); + expect(first.code).toBe(0); + const firstLeaseId = (first.json as { lease: { id: string } }).lease.id; + // Refused for holding a lease already, refused for want of capacity, and timed out waiting. + expect((await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"])).code).toBe(13); + expect( + (await env.cli([...LEASE_ARGS, "--agent-id", "agent-b", "--no-wait", "--detach"])).code, + ).toBe(11); + expect( + ( + await env.cli([...LEASE_ARGS, "--agent-id", "agent-c", "--timeout", "300ms", "--detach"], { + timeout: 15_000, + }) + ).code, + ).toBe(10); + expect((await env.cli(["release", firstLeaseId])).code).toBe(0); + // A second lease on the device the first one left warm. + const second = await env.cli([...LEASE_ARGS, "--agent-id", "agent-d", "--detach"]); + expect(second.code).toBe(0); + expect( + (await env.cli(["release", (second.json as { lease: { id: string } }).lease.id])).code, + ).toBe(0); + + const usage = await stats(env); + const history = inWindow(await env.events(), usage.window); + const named = (name: string) => history.filter((event) => event.event === name); + + expect(usage.partial).toBe(false); + expect(usage.totals.requests).toBe(named("lease.requested").length); + expect(usage.totals.granted).toBe(2); + expect(usage.totals.bySource).toMatchObject( + countBy(named("lease.granted").map((event) => payload(event).source as string)), + ); + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 1, warm: 1 }); + expect(usage.totals.rejected.total).toBe(named("lease.rejected").length); + expect(usage.totals.rejected.byReason).toEqual( + countBy(named("lease.rejected").map((event) => payload(event).reason as string)), + ); + expect(Object.keys(usage.totals.rejected.byReason).sort()).toEqual([ + "already-leased", + "no-wait", + "timeout", + ]); + // Held and turnaround are the two leases' own spans, as the events state them. + const heldByHand = named("lease.granted") + .map((granted) => { + const end = named("lease.released").find( + (event) => payload(event).leaseId === payload(granted).leaseId, + ); + return (end?.timestamp ?? NaN) - granted.timestamp; + }) + .sort((a, b) => a - b); + expect(usage.totals.held.count).toBe(2); + expect(usage.totals.held.max).toBe(heldByHand[1]); + expect(usage.totals.turnaround.count).toBe(2); + expect(usage.totals.wait.count).toBeGreaterThanOrEqual(3); + expect(usage.totals.provisioning.count).toBe(named("device.provisioned").length); + expect(usage.totals.boot.count).toBe(named("device.ready").length); + expect(usage.totals.incidents).toEqual({ + crashRecovered: 0, + lost: 0, + quarantineRecovered: 0, + quarantined: 0, + }); + expect(usage.workers).toHaveLength(1); + expect(usage.requesters.map((requester) => requester.id).sort()).toEqual([ + "agent-a", + "agent-b", + "agent-c", + "agent-d", + ]); + + const table = await env.cli(["stats", "--since", "1h"]); + expect(table.code).toBe(0); + expect(table.stdout).toContain("Totals"); + expect(table.stdout).toContain("Requests:"); + }); + + it("simlock stats --since 1h returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { + const env = await withDaemon(); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + const lease = await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"]); + expect( + (await env.cli(["release", (lease.json as { lease: { id: string } }).lease.id])).code, + ).toBe(0); + const before = await stats(env); + expect(before.totals.granted).toBe(1); + + await env.restartDaemon(); + const after = await stats(env); + + expect(after.totals).toMatchObject({ + boot: before.totals.boot, + bySource: before.totals.bySource, + granted: before.totals.granted, + held: before.totals.held, + incidents: before.totals.incidents, + provisioning: before.totals.provisioning, + rejected: before.totals.rejected, + requests: before.totals.requests, + turnaround: before.totals.turnaround, + wait: before.totals.wait, + }); + expect(after.requesters).toEqual(before.requesters); + }); + + it("on a gateway with two workers, simlock stats has fleet totals and one row per worker, and each worker's own simlock stats covers itself only", async () => { + const port = await freeLoopbackPort(); + const gateway = await withDaemon({ + configOverrides: { http: { host: "127.0.0.1", port }, mode: "gateway" }, + driver: "none", + }); + const minted = await gateway.cli(["token", "create", "--role", "worker"]); + const { secret } = minted.json as { secret: string }; + const oneAtATime = { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }; + const joinWith = (label: string) => + withDaemon({ + configOverrides: { + ...oneAtATime, + gateway: { label, token: secret, url: `ws://127.0.0.1:${port}` }, + }, + driverScript: { ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] } }, + }); + const workerA = await joinWith("worker-a"); + const workerB = await joinWith("worker-b"); + await waitFor( + async () => { + const list = await gateway.cli(["worker", "list", "--json"]); + const workers = (list.json as { workers: { connection: string }[] }).workers; + return workers.length === 2 && workers.every((worker) => worker.connection === "connected"); + }, + { label: "both workers joined", timeout: 30_000 }, + ); + + // One worker is full after its lease, so the second lease lands on the other. + for (const agent of ["agent-1", "agent-2"]) { + const leased = await gateway.cli([...LEASE_ARGS, "--agent-id", agent, "--detach"], { + timeout: 60_000, + }); + expect(leased.code).toBe(0); + } + + let fleet: Usage | undefined; + await waitFor( + async () => { + fleet = await stats(gateway); + return fleet.totals.granted === 2 && fleet.workers.length === 2; + }, + { label: "the gateway's figures show both grants", timeout: 30_000 }, + ); + expect(fleet?.totals.requests).toBe(2); + expect(fleet?.workers.map((worker) => worker.label).sort()).toEqual(["worker-a", "worker-b"]); + expect(fleet?.workers.map((worker) => worker.granted)).toEqual([1, 1]); + + for (const worker of [workerA, workerB]) { + const own = await stats(worker); + expect(own.workers).toHaveLength(1); + expect(own.totals.granted).toBe(1); + expect(own.workers[0]?.granted).toBe(1); + } + }); +}); diff --git a/src/admin/index.ts b/src/admin/index.ts index 0b74a486..00f4f32b 100644 --- a/src/admin/index.ts +++ b/src/admin/index.ts @@ -66,6 +66,8 @@ export type { TokenListOutput, TokenRevokeInput, TokenRevokeOutput, + UsageGetInput, + UsageGetOutput, WorkerComponentProgress, WorkerDrainInput, WorkerDrainOutput, diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index 8007b23b..20d0d9e9 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -198,6 +198,53 @@ describe("EventHistory", () => { { reason: "memory only" }, ]); }); + + it("reads the events and the oldest timestamp the file holds in one call, carrying the step in force", async () => { + const path = await eventFilePath(); + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const sink = new NodeFileLogSink({ path }); + const events = history({ bus, path, sink }); + const early = bus.emit("queue.changed", { depth: 2 }, "wait-queue"); + clock.advance(1_000); + bus.emit("daemon.stopping", { reason: "a" }, "daemon"); + clock.advance(1_000); + const late = bus.emit("daemon.stopping", { reason: "b" }, "daemon"); + sink.close(); + + const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_500 }); + + expect(read.oldestTs).toBe(1_000); + expect(read.events).toEqual([early, late]); + }); + + it("reads from the ring, with the carried step and its oldest timestamp, when there is no sink", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + const early = bus.emit("queue.changed", { depth: 2 }, "wait-queue"); + clock.advance(2_000); + const late = bus.emit("daemon.stopping", { reason: "b" }, "daemon"); + + const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_000 }); + + expect(read).toEqual({ events: [early, late], oldestTs: 1_000 }); + expect(await events.replay({ carry: ["queue.changed"], sinceTs: 2_000 })).toEqual([ + early, + late, + ]); + }); + + it("names the newest event's id, and none before the first event", () => { + const bus = new EventBus(new FakeClock(1_000)); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + expect(events.latestId()).toBeUndefined(); + + bus.emit("daemon.stopping", { reason: "a" }, "daemon"); + const last = bus.emit("daemon.stopping", { reason: "b" }, "daemon"); + + expect(events.latestId()).toBe(last.id); + }); }); describe("readEventFile", () => { @@ -442,4 +489,72 @@ describe("readEventFile", () => { expect(read.map((entry) => entry.seq)).toEqual([1, 3]); }); + + it("readEventFile with carry returns the latest capacity.changed and queue.changed at or before sinceTs, one per worker id, and none when there is none", async () => { + const step = ( + seq: number, + timestamp: number, + event: "capacity.changed" | "queue.changed", + workerId?: string, + ): EventEnvelope => + ({ + event, + id: `evt_${seq}`, + module: "test", + payload: { + ...(event === "queue.changed" ? { depth: seq } : {}), + ...(workerId === undefined ? {} : { workerId }), + }, + seq, + timestamp, + }) as unknown as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + step(1, 100, "capacity.changed"), + step(2, 200, "capacity.changed"), + step(3, 150, "queue.changed", "w1"), + step(4, 180, "queue.changed", "w2"), + step(5, 190, "queue.changed", "w1"), + envelope(6, 250), + envelope(7, 400), + step(8, 500, "capacity.changed"), + ), + }); + const carry = ["capacity.changed", "queue.changed"] as const; + + const read = await readEventFile(filesystem, "/data/events.jsonl", { carry, sinceTs: 300 }); + const none = await readEventFile(filesystem, "/data/events.jsonl", { carry, sinceTs: 50 }); + const plain = await readEventFile(filesystem, "/data/events.jsonl", { sinceTs: 300 }); + + // The newest step of each kind and worker at or before 300, then everything after it. + expect(read.map((entry) => entry.id)).toEqual(["evt_4", "evt_5", "evt_2", "evt_7", "evt_8"]); + expect(plain.map((entry) => entry.id)).toEqual(["evt_7", "evt_8"]); + // Nothing is at or before 50, so nothing is carried and nothing is repeated. + expect(none.map((entry) => entry.id)).toEqual([ + "evt_1", + "evt_2", + "evt_3", + "evt_4", + "evt_5", + "evt_6", + "evt_7", + "evt_8", + ]); + }); + + it("carries a step stamped exactly sinceTs, and counts it once", async () => { + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines({ + ...envelope(1, 300), + event: "capacity.changed", + } as unknown as EventEnvelope), + }); + + const read = await readEventFile(filesystem, "/data/events.jsonl", { + carry: ["capacity.changed"], + sinceTs: 300, + }); + + expect(read.map((entry) => entry.id)).toEqual(["evt_1"]); + }); }); diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index 8f4be2ce..66d429ac 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -1,6 +1,6 @@ import { type Filesystem, isMissingPathError, type Logger, type LogSink } from "../ports/index.js"; import { EVENT_ID_PATTERN } from "../contract/schemas.js"; -import type { EventBus, EventEnvelope } from "./index.js"; +import type { EventBus, EventEnvelope, EventName } from "./index.js"; import { byTimeThenSeq } from "./order.js"; /** The durable record of every business event, in the data directory (ADR 0006). */ @@ -20,7 +20,7 @@ export const EVENT_FILE_NAME = "events.jsonl"; export async function readEventFile( filesystem: Filesystem, path: string, - options: { readonly sinceTs: number }, + options: { readonly sinceTs: number; readonly carry?: readonly EventName[] }, ): Promise { return (await readEventHistory(filesystem, path, options)).events; } @@ -33,7 +33,7 @@ export async function readEventFile( export async function readEventHistory( filesystem: Filesystem, path: string, - { sinceTs }: { readonly sinceTs: number }, + { sinceTs }: { readonly sinceTs: number; readonly carry?: readonly EventName[] }, ): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { const generations = await readGenerations(filesystem, path); const seen = new Set(); @@ -144,7 +144,9 @@ export class EventHistory { * Without `sinceTs`, the ring, as `simlock events` has always answered. With it, the event * file while the writer is writing; the ring when there is no file to trust. */ - async replay(input: { readonly sinceTs?: number } = {}): Promise { + async replay( + input: { readonly sinceTs?: number; readonly carry?: readonly EventName[] } = {}, + ): Promise { const { bus, filesystem, logger, path } = this.#options; if (input.sinceTs === undefined) return bus.replay(); if (!this.#writing) return bus.replay({ sinceTs: input.sinceTs }); @@ -158,4 +160,15 @@ export class EventHistory { return bus.replay({ sinceTs: input.sinceTs }); } } + + async read(_input: { + readonly sinceTs: number; + readonly carry: readonly EventName[]; + }): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { + return { events: [], oldestTs: undefined }; + } + + latestId(): string | undefined { + return undefined; + } } diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index c8f2be66..18dfadbb 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -28,7 +28,7 @@ import { DaemonEndpointHost } from "../daemon/connection-host.js"; import { DaemonServer } from "../daemon/server.js"; import { AdminSecretManager } from "../daemon/admin-secret.js"; import { createCredentialRoleResolver } from "../daemon/session.js"; -import { SimlockError, type AnySimlockError } from "../contract/index.js"; +import { SimlockError, type AnySimlockError, type UsageOutput } from "../contract/index.js"; import type { CatalogGetOutput, DeviceRecoveredPush, @@ -4753,6 +4753,7 @@ function fakeClient(overrides: Partial = {}): SimlockAdminCl getConfig: () => Promise.resolve({} as SimlockConfig), stopDaemon: () => Promise.resolve({ stopping: true }), replayEvents: () => Promise.resolve([]), + usage: (window) => Promise.resolve(usageAnswer(window)), subscribeEvents: () => Promise.resolve(() => Promise.resolve()), createToken: (input) => Promise.resolve({ @@ -5338,3 +5339,212 @@ describe("simlock lease: a request names a model, a class, or nothing", () => { ); }); }); + +/** A `usage.get` answer with nothing counted, for the window it was asked for. */ +function usageAnswer( + window: { readonly from: number; readonly to: number }, + overrides: Partial = {}, +): UsageOutput { + const samples = { count: 0, max: null, p50: null, p95: null }; + const figures = { + boot: samples, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + errors: { byCode: {} }, + granted: 0, + held: samples, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: samples, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: samples, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: samples, + }; + return { + bucketMs: 60_000, + coversFrom: window.from, + partial: false, + platforms: { android: figures, ios: figures }, + requesters: [], + series: [], + totals: figures, + window, + workers: [], + ...overrides, + }; +} + +describe("CLI: stats", () => { + const NOW = Date.parse("2026-10-05T12:00:00.000Z"); + const HOUR = 3_600_000; + + function statsRun(argv: readonly string[], usage: SimlockAdminClient["usage"]) { + const output = outputCapture(); + const asked: { from: number; to: number }[] = []; + const run = runCli( + ["stats", ...argv], + output.environmentWith({ + clock: new FakeClock(NOW), + connectAdmin: async () => + fakeClient({ + usage: async (window) => { + asked.push(window); + return usage(window); + }, + }), + }), + ); + return { asked, output, run }; + } + + it("simlock stats rejects --since with --from, and --from later than --to, as usage errors", async () => { + // Each flag on its own is accepted, so the refusals below are the combinations'. + for (const argv of [["--since", "1h"], ["--from", "2026-10-05T10:00:00Z"], ["--json"]]) { + const { output, run } = statsRun(argv, async (window) => usageAnswer(window)); + await expect(run, argv.join(" ")).resolves.toBe(0); + expect(output.stderr, argv.join(" ")).toBe(""); + } + for (const argv of [ + ["--since", "1h", "--from", "2026-10-05T10:00:00Z"], + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-03T10:00:00Z"], + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T10:00:00Z"], + ["--to", "2026-10-04T10:00:00Z"], + ["--since", "1h", "--to", "2026-10-04T10:00:00Z"], + ["--since", "soon"], + ["--from", "last tuesday"], + ]) { + const { asked, output, run } = statsRun(argv, async (window) => usageAnswer(window)); + + await expect(run, argv.join(" ")).resolves.toBe(2); + expect(output.stderr, argv.join(" ")).toContain('"code":"USAGE"'); + expect(asked, argv.join(" ")).toEqual([]); + expect(output.stdout).toBe(""); + } + }); + + it("simlock stats --json prints the operation's output unchanged", async () => { + const answer = usageAnswer( + { from: NOW - HOUR, to: NOW }, + { + partial: true, + requesters: [ + { granted: 1, heldTotalMs: 5, id: "tok_a", label: "ci", rejected: 0, requests: 1 }, + ], + }, + ); + const { output, run } = statsRun(["--since", "1h", "--json"], async () => answer); + + await expect(run).resolves.toBe(0); + + expect(JSON.parse(output.stdout)).toEqual(answer); + expect(output.stdout.endsWith("\n")).toBe(true); + }); + + it("asks for the last 24 hours by default, and for --since up to now", async () => { + const byDefault = statsRun([], async (window) => usageAnswer(window)); + const since = statsRun(["--since", "90m"], async (window) => usageAnswer(window)); + + await byDefault.run; + await since.run; + + expect(byDefault.asked).toEqual([{ from: NOW - 24 * HOUR, to: NOW }]); + expect(since.asked).toEqual([{ from: NOW - 90 * 60_000, to: NOW }]); + }); + + it("asks for the ISO window --from and --to name, and for --from up to now", async () => { + const both = statsRun( + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T12:00:00Z"], + async (window) => usageAnswer(window), + ); + const open = statsRun(["--from", "2026-10-05T10:00:00Z"], async (window) => + usageAnswer(window), + ); + + await both.run; + await open.run; + + expect(both.asked).toEqual([ + { from: Date.parse("2026-10-04T10:00:00Z"), to: Date.parse("2026-10-04T12:00:00Z") }, + ]); + expect(open.asked).toEqual([{ from: Date.parse("2026-10-05T10:00:00Z"), to: NOW }]); + }); + + it("prints the figures as a table, with the partial note above them", async () => { + const { output, run } = statsRun(["--since", "30d"], async (window) => + usageAnswer(window, { coversFrom: NOW - 60_000, partial: true }), + ); + + await expect(run).resolves.toBe(0); + + const lines = output.stdout.split("\n"); + expect(lines[0]).toBe("Usage from 2026-09-05T12:00:00.000Z to 2026-10-05T12:00:00.000Z"); + expect(lines[1]).toContain("Figures cover from 2026-10-05T11:59:00.000Z"); + expect(lines).toContain("Totals"); + }); + + it("prints the HISTORY_NOT_KEPT message and exits with its code when the window ends before the history", async () => { + const { output, run } = statsRun(["--since", "30d"], async () => { + throw new SimlockError( + "HISTORY_NOT_KEPT", + "domain", + "The event history does not reach back to the end of that window; its oldest event is from 2026-10-05T11:59:00.000Z.", + { oldestTs: NOW - 60_000 }, + ); + }); + + await expect(run).resolves.toBe(12); + + expect(output.stdout).toBe(""); + expect(output.stderr).toContain('"code":"HISTORY_NOT_KEPT"'); + expect(output.stderr).toContain("oldest event is from 2026-10-05T11:59:00.000Z"); + }); + + it("closes the connection after the answer, and after a failure", async () => { + let closed = 0; + const output = outputCapture(); + const connect = (usage: SimlockAdminClient["usage"]) => async () => + fakeClient({ + close: async () => { + closed += 1; + }, + usage, + }); + + await runCli( + ["stats"], + output.environmentWith({ connectAdmin: connect(async (window) => usageAnswer(window)) }), + ); + await runCli( + ["stats"], + output.environmentWith({ + connectAdmin: connect(async () => { + throw new Error("boom"); + }), + }), + ); + + expect(closed).toBe(2); + }); + + it("prints its usage on --help without connecting", async () => { + const output = outputCapture(); + let connected = false; + + const exitCode = await runCli( + ["stats", "--help"], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ); + + expect(exitCode).toBe(0); + expect(connected).toBe(false); + expect(output.stdout).toBe( + "Usage: simlock stats [--since | --from [--to ]] [--json]\n", + ); + }); +}); diff --git a/src/cli/stats.test.ts b/src/cli/stats.test.ts new file mode 100644 index 00000000..b0fc0c1a --- /dev/null +++ b/src/cli/stats.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from "vitest"; + +import type { UsageFigures, UsageOutput } from "../contract/index.js"; +import { formatUsage } from "./stats.js"; + +const NONE = { count: 0, max: null, p50: null, p95: null }; + +function figuresFixture(overrides: Partial = {}): UsageFigures { + return { + boot: NONE, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + errors: { byCode: {} }, + granted: 0, + held: NONE, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: NONE, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: NONE, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: NONE, + ...overrides, + }; +} + +function usageFixture(overrides: Partial = {}): UsageOutput { + return { + bucketMs: 900_000, + coversFrom: Date.parse("2026-10-04T10:00:00.000Z"), + partial: false, + platforms: { android: figuresFixture(), ios: figuresFixture() }, + requesters: [], + series: [], + totals: figuresFixture(), + window: { + from: Date.parse("2026-10-04T10:00:00.000Z"), + to: Date.parse("2026-10-05T10:00:00.000Z"), + }, + workers: [], + ...overrides, + }; +} + +const BUSY = figuresFixture({ + boot: { count: 2, max: 40_000, p50: 20_000, p95: 40_000 }, + bySource: { booted: 3, provisioned: 1, warm: 6 }, + errors: { byCode: { "device.purge-failed": 2 } }, + granted: 10, + held: { count: 9, max: 3_725_000, p50: 300_000, p95: 1_800_000 }, + incidents: { crashRecovered: 1, lost: 4, quarantineRecovered: 3, quarantined: 2 }, + provisioning: { count: 1, max: 90_000, p50: 90_000, p95: 90_000 }, + queue: { meanDepth: 0.4, peakDepth: 2 }, + rejected: { byReason: { "no-wait": 1, timeout: 1 }, total: 2 }, + requests: 12, + turnaround: { count: 9, max: 3_800_000, p50: 310_000, p95: 1_900_000 }, + utilisation: { + ram: { limitBytes: 16 * 1024 ** 3, meanBytes: 1024 ** 3, peakBytes: 2 * 1024 ** 3 }, + slots: { max: 4, mean: 1.5, peak: 3 }, + }, + wait: { count: 11, max: 9_100, p50: 1_200, p95: 4_000 }, +}); + +describe("formatUsage", () => { + it("opens with the window and, when the history falls short of it, the line that says where the figures start", () => { + const partial = formatUsage( + usageFixture({ coversFrom: Date.parse("2026-10-05T09:59:00.000Z"), partial: true }), + ).split("\n"); + const whole = formatUsage(usageFixture()); + + expect(partial[0]).toBe("Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z"); + expect(partial[1]).toBe( + "Figures cover from 2026-10-05T09:59:00.000Z: the history does not reach back to the start of the window.", + ); + expect(whole).not.toContain("Figures cover from"); + }); + + it("prints every total on a line of its own", () => { + const lines = formatUsage(usageFixture({ totals: BUSY })).split("\n"); + + expect(lines).toContain("Totals"); + expect(lines).toContain(" Requests: 12 (10 granted, 2 rejected)"); + expect(lines).toContain(" Granted: warm 6, booted 3, provisioned 1"); + expect(lines).toContain(" Rejected: no-wait 1, timeout 1"); + expect(lines).toContain(" Wait: p50 1.2s, p95 4s, max 9.1s (11 samples)"); + expect(lines).toContain(" Held: p50 5m 0s, p95 30m 0s, max 1h 2m (9 samples)"); + expect(lines).toContain(" Turnaround: p50 5m 10s, p95 31m 40s, max 1h 3m (9 samples)"); + expect(lines).toContain(" Provisioning: p50 1m 30s, p95 1m 30s, max 1m 30s (1 sample)"); + expect(lines).toContain(" Boot: p50 20s, p95 40s, max 40s (2 samples)"); + expect(lines).toContain(" Slots: peak 3 of 4, mean 1.5"); + expect(lines).toContain(" RAM: peak 2.0 GiB of 16.0 GiB, mean 1.0 GiB"); + expect(lines).toContain(" Queue: peak depth 2, mean 0.4"); + expect(lines).toContain( + " Incidents: 2 quarantined, 1 recovered after a crash, 3 recovered from quarantine, 4 lost", + ); + expect(lines).toContain(" Errors: device.purge-failed 2"); + }); + + it("says plainly what it has no figure for, and leaves out rows it has nothing to say in", () => { + const text = formatUsage(usageFixture()); + + expect(text).toContain(" Requests: 0 (0 granted, 0 rejected)"); + expect(text).toContain(" Wait: no samples"); + expect(text).toContain(" Slots: not known"); + expect(text).toContain(" Queue: not known"); + expect(text).not.toContain("RAM:"); + expect(text).not.toContain("Errors:"); + expect(text).not.toContain("Rejected:"); + expect(text).not.toContain("Granted:"); + expect(text).not.toContain("Workers"); + expect(text).not.toContain("Requesters"); + }); + + it("prints a row for each platform, each worker with its label, and each requester with its label", () => { + const lines = formatUsage( + usageFixture({ + platforms: { android: figuresFixture(), ios: BUSY }, + requesters: [ + { + granted: 2, + heldTotalMs: 725_000, + id: "tok_a", + label: "ci-bot", + rejected: 1, + requests: 3, + }, + { granted: 0, heldTotalMs: 0, id: "tok_b", rejected: 0, requests: 1 }, + ], + totals: BUSY, + workers: [ + { ...BUSY, id: "wrk_1", label: "mac-mini-1" }, + { ...figuresFixture(), id: "wrk_2" }, + ], + }), + ).split("\n"); + + expect(lines).toContain("Platforms"); + expect(lines).toContain( + " ios 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3", + ); + expect(lines).toContain( + " android 0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak -", + ); + expect(lines).toContain("Workers"); + expect(lines).toContain( + " mac-mini-1 (wrk_1) 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3", + ); + expect(lines).toContain( + " wrk_2 0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak -", + ); + expect(lines).toContain("Requesters"); + expect(lines).toContain(" ci-bot (tok_a) 3 requests, 2 granted, 1 rejected, held 12m 5s"); + expect(lines).toContain(" tok_b 1 request, 0 granted, 0 rejected, held 0s"); + }); +}); diff --git a/src/cli/stats.ts b/src/cli/stats.ts new file mode 100644 index 00000000..c9b87d3e --- /dev/null +++ b/src/cli/stats.ts @@ -0,0 +1,6 @@ +import type { UsageOutput } from "../contract/index.js"; + +/** The human rendering of `usage.get`'s answer for `simlock stats`. */ +export function formatUsage(_usage: UsageOutput): string { + return ""; +} diff --git a/src/contract/errors.test.ts b/src/contract/errors.test.ts index c59742c8..cc7237f3 100644 --- a/src/contract/errors.test.ts +++ b/src/contract/errors.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; -import { ERROR_TABLE, fromWireError, isSimlockError } from "./errors.js"; +import { + CODES_WITH_DECLARED_DETAILS, + ERROR_TABLE, + fromWireError, + isSimlockError, +} from "./errors.js"; describe("SimlockError", () => { it("narrows details by code", () => { @@ -18,6 +23,22 @@ describe("SimlockError", () => { } }); + it("HISTORY_NOT_KEPT is a domain error with exit code 12, HTTP status 422, and the oldest held time in its details", () => { + expect(ERROR_TABLE.HISTORY_NOT_KEPT).toEqual({ + cliExitCode: 12, + code: "HISTORY_NOT_KEPT", + httpStatus: 422, + kind: "domain", + }); + + const error = fromWireError("HISTORY_NOT_KEPT", "not kept", { oldestTs: 42 }); + + expect(error.code).toBe("HISTORY_NOT_KEPT"); + if (error.code === "HISTORY_NOT_KEPT") expect(error.details.oldestTs).toBe(42); + else throw new Error("expected HISTORY_NOT_KEPT"); + expect(CODES_WITH_DECLARED_DETAILS.has("HISTORY_NOT_KEPT")).toBe(true); + }); + it("wraps an unrecognized code as UNKNOWN_DAEMON_ERROR instead of throwing", () => { const error = fromWireError("SOME_FUTURE_CODE", "a newer daemon said so"); expect(isSimlockError(error)).toBe(true); diff --git a/src/contract/errors.ts b/src/contract/errors.ts index 7400c212..ae029795 100644 --- a/src/contract/errors.ts +++ b/src/contract/errors.ts @@ -88,6 +88,11 @@ export interface ErrorDetailsMap { COMPONENT_IN_USE: { readonly devices: number; readonly foreignDevices: number }; /** ADR 0010 §8: a `component.remove` while an install or removal runs or waits on the platform. */ COMPONENT_BUSY: Record; + /** + * ADR 0016 §5: a `usage.get` window that ends before the oldest event the history holds. + * `oldestTs` is the oldest time the history reaches, so a caller can ask for a window it has. + */ + HISTORY_NOT_KEPT: { readonly oldestTs: number }; UNKNOWN_LEASE: { readonly leaseId: string }; /** A `simlock ` verb the owning driver will not proxy (ADR 0001, decision 7). Carries * the tool so a caller can say which wrapper refused without re-parsing the message. */ @@ -197,6 +202,7 @@ const CODES_WITH_DECLARED_DETAILS_BY_CODE: Record DOWNLOAD_TIMEOUT: true, DOWNLOADS_DISABLED: true, COMPONENT_IN_USE: true, + HISTORY_NOT_KEPT: true, UNKNOWN_LEASE: true, PASSTHROUGH_REFUSED: true, UNKNOWN_PASSTHROUGH_TOOL: true, @@ -376,6 +382,8 @@ export const ERROR_TABLE: { readonly [Code in SimlockErrorCode]: ErrorTableEntry WORKER_CONNECTED: { code: "WORKER_CONNECTED", kind: "domain", cliExitCode: 2, httpStatus: 409 }, // Exit 12, beside `UNKNOWN_MODEL`: the "what you named does not exist" class, which a script // branches on differently from "the fleet is busy" or "that cannot apply here". + // Exit 12 and 422, with the other "what you asked for is not there" codes (ADR 0016 §5). + HISTORY_NOT_KEPT: { code: "HISTORY_NOT_KEPT", kind: "domain", cliExitCode: 12, httpStatus: 422 }, UNKNOWN_WORKER: { code: "UNKNOWN_WORKER", kind: "domain", cliExitCode: 12, httpStatus: 404 }, WORKER_UNREACHABLE: { code: "WORKER_UNREACHABLE", diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 1c520b62..74f8d452 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -41,6 +41,7 @@ const ROLE_MATRIX: ReadonlyArray<{ { name: "events.replay", input: {}, role: "admin" }, { name: "events.subscribe", input: {}, role: "admin" }, { name: "events.unsubscribe", input: {}, role: "admin" }, + { name: "usage.get", input: { from: 0, to: 60_000 }, role: "admin" }, { name: "token.create", input: { role: "agent" }, role: "admin" }, { name: "driver.passthrough", input: { args: ["devices"], tool: "adb" }, role: "agent" }, { @@ -112,6 +113,7 @@ const EFFECT_MATRIX: ReadonlyArray<{ { name: "events.replay", input: {}, effect: "read" }, { name: "events.subscribe", input: {}, effect: "read" }, { name: "events.unsubscribe", input: {}, effect: "read" }, + { name: "usage.get", input: { from: 0, to: 60_000 }, effect: "read" }, { name: "token.create", input: { role: "agent" }, effect: "write" }, { name: "driver.passthrough", input: { args: ["devices"], tool: "adb" }, effect: "read" }, { @@ -911,3 +913,25 @@ describe("lease.request names a model, a class, or nothing (ADR 0015 §1)", () = expect(requestedClass({ model: "iPhone 16" })).toBeUndefined(); }); }); + +describe("usage.get input", () => { + const parse = (input: unknown) => OPERATIONS["usage.get"].input.safeParse(input); + const DAY = 24 * 60 * 60 * 1000; + + it("takes epoch milliseconds with from before to, up to ninety days apart", () => { + expect(parse({ from: 0, to: 1 }).success).toBe(true); + expect(parse({ from: 1_000, to: 1_000 + 90 * DAY }).success).toBe(true); + expect(parse({ from: 1_000, to: 1_001 + 90 * DAY }).success).toBe(false); + expect(parse({ from: 5, to: 5 }).success).toBe(false); + expect(parse({ from: 6, to: 5 }).success).toBe(false); + }); + + it("refuses a bound that is not an integer, is negative, or no date can hold", () => { + expect(parse({ from: 0.5, to: 10 }).success).toBe(false); + expect(parse({ from: -1, to: 10 }).success).toBe(false); + expect(parse({ from: 0, to: 8.64e15 + 1 }).success).toBe(false); + expect(parse({ from: 8.64e15 - 1, to: 8.64e15 }).success).toBe(true); + expect(parse({ from: "0", to: 10 }).success).toBe(false); + expect(parse({ from: 0 }).success).toBe(false); + }); +}); diff --git a/src/contract/operations.ts b/src/contract/operations.ts index 93be1f8c..6a490b96 100644 --- a/src/contract/operations.ts +++ b/src/contract/operations.ts @@ -36,6 +36,8 @@ import { statusLeaseSchema, tokenRecordSchema, tokenRoleSchema, + usageOutputSchema, + usageWindowSchema, waitingRequestSchema, workerViewSchema, } from "./schemas.js"; @@ -641,6 +643,22 @@ export const eventsReplay = defineOperation({ output: z.array(eventEnvelopeSchema), }); +// ---- usage.get ---------------------------------------------------------------------------- + +/** + * ADR 0016 §1: the figures for a window, computed from the event history when asked. Admin-only, + * as `events.replay` is: it reads the same history. A window that ends before the oldest held + * event is refused with `HISTORY_NOT_KEPT`. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const usageGet = defineOperation({ + name: "usage.get", + role: "admin", + effect: "read", + input: usageWindowSchema, + output: usageOutputSchema, +}); + // ---- events.subscribe --------------------------------------------------------------------- // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. @@ -947,6 +965,7 @@ export const OPERATIONS = { "events.replay": eventsReplay, "events.subscribe": eventsSubscribe, "events.unsubscribe": eventsUnsubscribe, + "usage.get": usageGet, "token.create": tokenCreate, "token.list": tokenList, "token.revoke": tokenRevoke, diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 997fcbf2..ff9cc5bd 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -1016,3 +1016,112 @@ export const statusLeaseSchema = leaseRecordSchema.extend({ /** Which worker holds this lease; absent on a worker's own answer, set by a gateway. */ workerId: z.string().optional(), }); + +// ---- usage.get (ADR 0016) ----------------------------------------------------------------- + +/** The widest window `usage.get` answers: ninety days, so the history one call reads is bounded. */ +const USAGE_MAX_WINDOW_MS = 90 * 24 * 60 * 60 * 1000; + +/** + * `usage.get`'s input. The window is wire input: both ends are epoch milliseconds a date can + * hold, `from` is before `to`, and the span is bounded before anything is read (safety rule 10). + */ +export const usageWindowSchema = z + .object({ + from: z.number().int().min(0).max(MAX_DATE_MS), + to: z.number().int().min(0).max(MAX_DATE_MS), + }) + .refine((window) => window.from < window.to, { message: "from must be before to" }) + .refine((window) => window.to - window.from <= USAGE_MAX_WINDOW_MS, { + message: "the window must not be longer than 90 days", + }); + +/** Milliseconds, `null` for a percentile over no samples. */ +const usageSamplesSchema = z.object({ + count: z.number().int().nonnegative(), + max: z.number().nullable(), + p50: z.number().nullable(), + p95: z.number().nullable(), +}); + +const usageCountsSchema = z.record(z.string(), z.number().int().nonnegative()); + +/** One scope's figures: the fleet's, a platform's, or a worker's. */ +const usageFiguresSchema = z.object({ + bySource: z.object({ + booted: z.number().int().nonnegative(), + provisioned: z.number().int().nonnegative(), + warm: z.number().int().nonnegative(), + }), + boot: usageSamplesSchema, + errors: z.object({ byCode: usageCountsSchema }), + granted: z.number().int().nonnegative(), + held: usageSamplesSchema, + incidents: z.object({ + crashRecovered: z.number().int().nonnegative(), + lost: z.number().int().nonnegative(), + quarantineRecovered: z.number().int().nonnegative(), + quarantined: z.number().int().nonnegative(), + }), + provisioning: usageSamplesSchema, + queue: z.object({ meanDepth: z.number().nullable(), peakDepth: z.number().nullable() }), + rejected: z.object({ + byReason: usageCountsSchema, + total: z.number().int().nonnegative(), + }), + requests: z.number().int().nonnegative(), + turnaround: usageSamplesSchema, + utilisation: z.object({ + /** Present only when `capacity.changed` carried a RAM budget in the window. */ + ram: z + .object({ + limitBytes: z.number(), + meanBytes: z.number(), + peakBytes: z.number(), + }) + .optional(), + slots: z.object({ + max: z.number().nullable(), + mean: z.number().nullable(), + peak: z.number().nullable(), + }), + }), + wait: usageSamplesSchema, +}); + +export const usageOutputSchema = z.object({ + /** The width of one series point, chosen by the daemon (ADR 0016 §8). */ + bucketMs: z.number().int().positive(), + /** The later of the window's start and the oldest timestamp the history holds (ADR 0016 §5). */ + coversFrom: z.number(), + /** True when `coversFrom` is later than the window's start: the history does not reach it. */ + partial: z.boolean(), + platforms: z.object({ android: usageFiguresSchema, ios: usageFiguresSchema }), + requesters: z.array( + z.object({ + granted: z.number().int().nonnegative(), + heldTotalMs: z.number().nonnegative(), + id: z.string(), + label: z.string().optional(), + rejected: z.number().int().nonnegative(), + requests: z.number().int().nonnegative(), + }), + ), + /** One point per bucket, each as it stands at the bucket's end; `null` where nothing is known. */ + series: z.array( + z.object({ + at: z.number(), + queueDepth: z.number().nullable(), + ramUsedBytes: z.number().optional(), + slotsMax: z.number().nullable(), + slotsUsed: z.number().nullable(), + waiting: z.number().int().nonnegative(), + }), + ), + totals: usageFiguresSchema, + window: z.object({ from: z.number(), to: z.number() }), + workers: z.array(usageFiguresSchema.extend({ id: z.string(), label: z.string().optional() })), +}); + +export type UsageFigures = z.infer; +export type UsageOutput = z.infer; diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts new file mode 100644 index 00000000..0d077e1d --- /dev/null +++ b/src/core/usage/compute-usage.test.ts @@ -0,0 +1,735 @@ +import { describe, expect, it } from "vitest"; + +import type { EventEnvelope, EventMap, EventName } from "../../bus/index.js"; +import { computeUsage, seriesBucketMs, type UsageOptions } from "./compute-usage.js"; + +const SECOND = 1_000; +const MINUTE = 60 * SECOND; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; +const T0 = 1_000_000_000; +const WINDOW = { from: T0, to: T0 + HOUR }; +const WORKER: UsageOptions = { fleet: false, workerId: "self" }; +const FLEET: UsageOptions = { fleet: true, requesterPrefix: "gw:g1:" }; + +let sequence = 0; + +/** One envelope; `extra` is merged into the payload, which is how a relayed event gets its `workerId`. */ +function at(timestamp: number, event: EventName, payload: Record): EventEnvelope { + sequence += 1; + return { + event, + id: `evt_${sequence}`, + module: "test", + payload: payload as unknown as EventMap[EventName], + seq: sequence, + timestamp, + } as EventEnvelope; +} + +const requested = (ts: number, requestId: string, requester = "agent-a", platform = "ios") => + at(ts, "lease.requested", { + requestId, + requestSpec: { platform }, + requester, + waitPolicy: "wait", + }); + +const granted = ( + ts: number, + requestId: string, + leaseId: string, + source = "warm", + requester = "agent-a", + extra: Record = {}, +) => + at(ts, "lease.granted", { + deviceId: `dev-${leaseId}`, + leaseId, + requestId, + requester, + source, + ...extra, + }); + +const released = (ts: number, leaseId: string, extra: Record = {}) => + at(ts, "lease.released", { + deviceId: `dev-${leaseId}`, + leaseId, + ownerId: "owner", + reason: "explicit", + ...extra, + }); + +const rejected = ( + ts: number, + requestId: string, + reason: string, + requester = "agent-a", + extra: Record = {}, +) => + at(ts, "lease.rejected", { + reason, + requestId, + requestSpec: { platform: "ios" }, + requester, + ...extra, + }); + +const figures = (running: number, reserved: number, maxRunning: number, warm = 0) => ({ + maxRunning, + reserved, + running, + warm, +}); + +const capacity = ( + ts: number, + global: ReturnType, + ramBudget?: { usedBytes: number; limitBytes: number }, + extra: Record = {}, +) => + at(ts, "capacity.changed", { + android: figures(0, 0, global.maxRunning), + global, + ios: global, + ...(ramBudget === undefined ? {} : { ramBudget }), + ...extra, + }); + +const queueChanged = (ts: number, depth: number, extra: Record = {}) => + at(ts, "queue.changed", { depth, ...extra }); + +describe("computeUsage", () => { + it("counts a request granted, held and released as one request, one grant, and one sample in wait, held and turnaround with the expected milliseconds", () => { + const usage = computeUsage( + [requested(T0 + 1_000, "r1"), granted(T0 + 3_000, "r1", "l1"), released(T0 + 13_000, "l1")], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + expect(usage.totals.held).toEqual({ count: 1, max: 10_000, p50: 10_000, p95: 10_000 }); + expect(usage.totals.turnaround).toEqual({ count: 1, max: 12_000, p50: 12_000, p95: 12_000 }); + expect(usage.platforms.ios.requests).toBe(1); + expect(usage.platforms.android.requests).toBe(0); + }); + + it("ends a lease's held time at its expiry as at its release", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1"), + granted(T0 + 2_000, "r1", "l1"), + at(T0 + 9_000, "lease.expired", { deviceId: "dev-l1", leaseId: "l1", ownerId: "owner" }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.held).toEqual({ count: 1, max: 7_000, p50: 7_000, p95: 7_000 }); + expect(usage.totals.turnaround.p50).toBe(8_000); + }); + + it("reports p50, p95 and max of a set of twenty known waits", () => { + const events: EventEnvelope[] = []; + for (let index = 1; index <= 20; index += 1) { + // Request `index` waits index * 100 ms: 100, 200, ... 2000. + events.push(requested(T0 + index * 10_000, `r${index}`, `agent-${index}`)); + events.push(granted(T0 + index * 10_000 + index * 100, `r${index}`, `l${index}`)); + } + + const { wait } = computeUsage(events, WINDOW, WORKER).totals; + + expect(wait).toEqual({ count: 20, max: 2_000, p50: 1_000, p95: 1_900 }); + }); + + it("reports a percentile over no samples as null", () => { + const { wait, held } = computeUsage([], WINDOW, WORKER).totals; + + expect(wait).toEqual({ count: 0, max: null, p50: null, p95: null }); + expect(held).toEqual({ count: 0, max: null, p50: null, p95: null }); + }); + + it("counts a rejected request under its reason and not under grants", () => { + const usage = computeUsage( + [requested(T0 + 1_000, "r1"), rejected(T0 + 6_000, "r1", "timeout")], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { timeout: 1 }, total: 1 }); + expect(usage.totals.wait.p50).toBe(5_000); + expect(usage.totals.turnaround.count).toBe(0); + }); + + it("attributes grants to warm, booted and provisioned from lease.granted.source", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "a"), + granted(T0 + 2_000, "r1", "l1", "warm", "a"), + requested(T0 + 1_000, "r2", "b"), + granted(T0 + 2_000, "r2", "l2", "booted", "b"), + requested(T0 + 1_000, "r3", "c"), + granted(T0 + 2_000, "r3", "l3", "provisioned", "c"), + requested(T0 + 1_000, "r4", "d"), + granted(T0 + 2_000, "r4", "l4", "provisioned", "d"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.granted).toBe(4); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 2, warm: 1 }); + }); + + it("reads provisioning and boot durations from device.provisioned and device.ready", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.provisioned", { + deviceId: "d1", + driver: "fake", + duration: 90_000, + spec: { platform: "ios" }, + }), + at(T0 + 2_000, "device.ready", { bootDuration: 20_000, deviceId: "d1" }), + at(T0 + 3_000, "device.provisioned", { + deviceId: "d2", + driver: "fake", + duration: 30_000, + spec: { platform: "android" }, + }), + at(T0 + 4_000, "device.ready", { bootDuration: 40_000, deviceId: "d2" }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.provisioning).toEqual({ count: 2, max: 90_000, p50: 30_000, p95: 90_000 }); + expect(usage.totals.boot).toEqual({ count: 2, max: 40_000, p50: 20_000, p95: 40_000 }); + expect(usage.platforms.ios.provisioning.p50).toBe(90_000); + expect(usage.platforms.android.boot.p50).toBe(40_000); + }); + + it("derives peak and time-weighted mean slot utilisation from capacity.changed steps, including the step in force before the window starts", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(1, 0, 4)), + capacity(T0 + 10 * MINUTE, figures(3, 1, 4)), + capacity(T0 + 30 * MINUTE, figures(0, 0, 4)), + ], + WINDOW, + WORKER, + ); + + // 1 slot for 10 minutes, 4 for 20, 0 for 30: (10 + 80 + 0) / 60. + expect(usage.totals.utilisation.slots).toEqual({ max: 4, mean: 1.5, peak: 4 }); + }); + + it("reports RAM utilisation when capacity.changed carries ramBudget and omits it otherwise", () => { + const withRam = computeUsage( + [ + capacity(T0 - 1_000, figures(0, 0, 4), { limitBytes: 1_000, usedBytes: 100 }), + capacity(T0 + 30 * MINUTE, figures(1, 0, 4), { limitBytes: 1_000, usedBytes: 500 }), + ], + WINDOW, + WORKER, + ); + const withoutRam = computeUsage([capacity(T0 - 1_000, figures(0, 0, 4))], WINDOW, WORKER); + + expect(withRam.totals.utilisation.ram).toEqual({ + limitBytes: 1_000, + meanBytes: 300, + peakBytes: 500, + }); + expect(withoutRam.totals.utilisation).not.toHaveProperty("ram"); + }); + + it("derives peak and mean queue depth from queue.changed", () => { + const usage = computeUsage( + [ + queueChanged(T0 - 1_000, 0), + queueChanged(T0 + 10 * MINUTE, 2), + queueChanged(T0 + 40 * MINUTE, 0), + ], + WINDOW, + WORKER, + ); + + // 0 for 10 minutes, 2 for 30, 0 for 20: 60 / 60. + expect(usage.totals.queue).toEqual({ meanDepth: 1, peakDepth: 2 }); + }); + + it("counts quarantined, crashRecovered, quarantineRecovered and lost from the named device events and nothing else", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.quarantined", { deviceId: "d1", maxRetries: 3, nextRetryAt: 0 }), + at(T0 + 2_000, "device.recovered", { + attempts: 1, + deviceId: "d2", + duration: 5, + leaseId: "l", + }), + at(T0 + 3_000, "device.quarantine-recovered", { + attempts: 1, + deviceId: "d1", + strategy: "erase", + }), + at(T0 + 4_000, "device.recovery-failed", { + attempts: 3, + deviceId: "d3", + error: "x", + leaseId: "l", + reason: "attempts-exhausted", + }), + at(T0 + 5_000, "device.quarantine-abandoned", { attempts: 3, deviceId: "d4" }), + at(T0 + 6_000, "device.quarantine-stranded", { attempts: 3, deviceId: "d5", error: "x" }), + // Facts that sit next to those and are not any of the four. + at(T0 + 7_000, "device.crash-detected", { + deviceId: "d2", + leaseId: "l", + observed: "stopped", + platform: "ios", + }), + at(T0 + 8_000, "device.deleted", { deviceId: "d4", initiator: "x" }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.incidents).toEqual({ + crashRecovered: 1, + lost: 3, + quarantineRecovered: 1, + quarantined: 1, + }); + }); + + it("counts a request whose lease.requested is before from and whose grant is inside the window under neither requests nor grants", () => { + const usage = computeUsage( + [ + requested(T0 - 5_000, "r1"), + granted(T0 + 1_000, "r1", "l1"), + released(T0 + 5_000, "l1"), + // A request inside the window, so the window is seen counting at all. + requested(T0 + 6_000, "r2", "agent-b"), + granted(T0 + 6_500, "r2", "l2", "booted", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, warm: 0 }); + expect(usage.totals.wait).toMatchObject({ count: 1, p50: 500 }); + expect(usage.totals.held.count).toBe(0); + expect(usage.requesters.map((requester) => requester.id)).toEqual(["agent-b"]); + }); + + it("gives no held or turnaround sample for a lease still open at to, and counts its request and grant", () => { + const usage = computeUsage( + [ + requested(WINDOW.to - 10_000, "r1"), + granted(WINDOW.to - 5_000, "r1", "l1"), + // The end falls after the window: it is not seen. + released(WINDOW.to + 5_000, "l1"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait.count).toBe(1); + expect(usage.totals.held.count).toBe(0); + expect(usage.totals.turnaround.count).toBe(0); + }); + + it("counts a rejection with no preceding lease.requested under rejected and not under requests", () => { + const usage = computeUsage( + [rejected(T0 + 1_000, "r-refused", "already-leased")], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { "already-leased": 1 }, total: 1 }); + expect(usage.totals.wait.count).toBe(0); + }); + + it("counts a request from before from that is rejected inside the window under neither requests nor rejections", () => { + const usage = computeUsage( + [ + requested(T0 - 5_000, "r1"), + rejected(T0 + 1_000, "r1", "timeout"), + // A request inside the window, so the window is seen counting at all. + requested(T0 + 2_000, "r2", "agent-b"), + rejected(T0 + 3_000, "r2", "cancelled", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.rejected).toEqual({ byReason: { cancelled: 1 }, total: 1 }); + }); + + it("reports null series points and excludes them from peak and mean before the first step when no step precedes the window", () => { + const usage = computeUsage( + [capacity(T0 + 30 * MINUTE, figures(2, 0, 4)), queueChanged(T0 + 30 * MINUTE, 3)], + WINDOW, + WORKER, + ); + + expect(usage.series).toHaveLength(60); + // Point 28 ends at the 29th minute, point 29 at the 30th, where the first step lands. + expect(usage.series[28]).toMatchObject({ queueDepth: null, slotsMax: null, slotsUsed: null }); + expect(usage.series[29]).toMatchObject({ queueDepth: 3, slotsMax: 4, slotsUsed: 2 }); + expect(usage.totals.utilisation.slots).toEqual({ max: 4, mean: 2, peak: 2 }); + expect(usage.totals.queue).toEqual({ meanDepth: 3, peakDepth: 3 }); + }); + + it("reports slots, RAM, queue depth and waiting requests in each series point as they stand at the bucket's end", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(1, 0, 4), { limitBytes: 1_000, usedBytes: 250 }), + queueChanged(T0 - 1_000, 0), + requested(T0 + 90 * SECOND, "r1"), + queueChanged(T0 + 90 * SECOND, 1), + granted(T0 + 150 * SECOND, "r1", "l1"), + queueChanged(T0 + 150 * SECOND, 0), + ], + { from: T0, to: T0 + 5 * MINUTE }, + WORKER, + ); + + expect(usage.bucketMs).toBe(MINUTE); + expect(usage.series.map((point) => point.at)).toEqual( + [1, 2, 3, 4, 5].map((n) => T0 + n * MINUTE), + ); + expect(usage.series.map((point) => point.waiting)).toEqual([0, 1, 0, 0, 0]); + expect(usage.series.map((point) => point.queueDepth)).toEqual([0, 1, 0, 0, 0]); + expect(usage.series[0]).toMatchObject({ ramUsedBytes: 250, slotsMax: 4, slotsUsed: 1 }); + }); + + it("groups by platform and by workerId", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "a", "ios"), + at(T0 + 1_500, "request.dispatched", { + model: "m", + platform: "ios", + queuedMs: 0, + reason: "warm-hit", + requestId: "r1", + requesterId: "a", + stage: "s", + workerId: "w1", + }), + granted(T0 + 2_000, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), + requested(T0 + 3_000, "r2", "b", "android"), + at(T0 + 3_500, "request.dispatched", { + model: "m", + platform: "android", + queuedMs: 0, + reason: "free-capacity", + requestId: "r2", + requesterId: "b", + stage: "s", + workerId: "w2", + }), + granted(T0 + 4_000, "w2-r", "l2", "provisioned", "gw:g1:b", { workerId: "w2" }), + at(T0 + 4_500, "device.provisioned", { + deviceId: "dev-l2", + driver: "fake", + duration: 7_000, + spec: { platform: "android" }, + workerId: "w2", + }), + ], + WINDOW, + { ...FLEET, workerLabels: { w1: "mac-1" } }, + ); + + expect(usage.platforms.ios).toMatchObject({ granted: 1, requests: 1 }); + expect(usage.platforms.android).toMatchObject({ granted: 1, requests: 1 }); + expect(usage.platforms.android.provisioning.p50).toBe(7_000); + expect(usage.workers.map((worker) => [worker.id, worker.label, worker.granted])).toEqual([ + ["w1", "mac-1", 1], + ["w2", undefined, 1], + ]); + expect(usage.workers[1]?.provisioning.p50).toBe(7_000); + expect(usage.workers[0]?.provisioning.count).toBe(0); + expect(usage.totals.granted).toBe(2); + }); + + it("with fleet: true counts a fleet request once, from the gateway's own events, and attributes its grant to the worker named by request.dispatched", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + at(T0 + 1_100, "request.dispatched", { + model: "m", + platform: "ios", + queuedMs: 100, + reason: "warm-hit", + requestId: "gw-r1", + requesterId: "agent-a", + stage: "s", + workerId: "w1", + }), + // The worker's own copy of the same request. + requested(T0 + 1_200, "w-r1", "gw:g1:agent-a", "ios"), + at(T0 + 1_300, "lease.requested", { + requestId: "w-r1", + requestSpec: { platform: "ios" }, + requester: "gw:g1:agent-a", + waitPolicy: "noWait", + workerId: "w1", + }), + granted(T0 + 4_000, "w-r1", "l1", "booted", "gw:g1:agent-a", { workerId: "w1" }), + released(T0 + 9_000, "l1", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.bySource.booted).toBe(1); + expect(usage.totals.wait.p50).toBe(3_000); + expect(usage.totals.held.p50).toBe(5_000); + expect(usage.workers).toHaveLength(1); + expect(usage.workers[0]).toMatchObject({ granted: 1, id: "w1", requests: 1 }); + }); + + it("with fleet: true ignores relayed lease.requested, lease.queued and queue.changed, and a relayed lease.rejected that matches no dispatched request", () => { + const usage = computeUsage( + [ + queueChanged(T0 + 1_000, 1), + at(T0 + 1_500, "queue.changed", { depth: 9, workerId: "w1" }), + at(T0 + 2_000, "lease.requested", { + requestId: "w-r1", + requestSpec: { platform: "ios" }, + requester: "gw:g1:ghost", + waitPolicy: "noWait", + workerId: "w1", + }), + at(T0 + 2_100, "lease.queued", { queuePosition: 1, requestId: "w-r1", workerId: "w1" }), + rejected(T0 + 2_200, "w-r1", "no-wait", "gw:g1:ghost", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: {}, total: 0 }); + expect(usage.totals.queue.peakDepth).toBe(1); + }); + + it("with fleet: true settles a dispatched request as rejected, with the worker's reason, from the relayed lease.rejected of the worker it was dispatched to", () => { + const dispatched = (workerId: string) => + at(T0 + 2_000, "request.dispatched", { + model: "m", + platform: "ios", + queuedMs: 0, + reason: "free-capacity", + requestId: "gw-r1", + requesterId: "agent-a", + stage: "s", + workerId, + }); + const base = [requested(T0 + 1_000, "gw-r1")]; + + const usage = computeUsage( + [ + ...base, + dispatched("w1"), + // Another worker's rejection for the same requester is not this request's outcome. + rejected(T0 + 2_500, "w2-r", "no-wait", "gw:g1:agent-a", { workerId: "w2" }), + rejected(T0 + 3_000, "w1-r", "boot-timeout", "gw:g1:agent-a", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(1); + expect(usage.totals.granted).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { "boot-timeout": 1 }, total: 1 }); + expect(usage.workers[0]).toMatchObject({ id: "w1", rejected: { total: 1 } }); + expect(usage.workers).toHaveLength(1); + }); + + it("with fleet: true does not join a requester's second dispatch's grant to its first request, which stays open", () => { + const dispatched = (ts: number, requestId: string) => + at(ts, "request.dispatched", { + model: "m", + platform: "ios", + queuedMs: 0, + reason: "free-capacity", + requestId, + requesterId: "agent-a", + stage: "s", + workerId: "w1", + }); + + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + dispatched(T0 + 1_100, "gw-r1"), + requested(T0 + 10_000, "gw-r2"), + dispatched(T0 + 10_100, "gw-r2"), + granted(T0 + 12_000, "w-r2", "l2", "warm", "gw:g1:agent-a", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(2); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + }); + + it("with fleet: true joins a grant the worker emitted before the gateway announced the dispatch", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + // A warm grant reaches the gateway ahead of the response that announces the dispatch. + granted(T0 + 1_400, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), + at(T0 + 1_500, "request.dispatched", { + model: "m", + platform: "ios", + queuedMs: 0, + reason: "warm-hit", + requestId: "gw-r1", + requesterId: "agent-a", + stage: "s", + workerId: "w1", + }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait.p50).toBe(400); + }); + + it("sets partial and coversFrom when the oldest event is inside the window", () => { + const partial = computeUsage( + [requested(T0 + 5 * MINUTE, "r1"), requested(T0 + 9 * MINUTE, "r2", "b")], + WINDOW, + WORKER, + ); + const whole = computeUsage( + [capacity(T0 - 1_000, figures(0, 0, 2)), requested(T0 + 5 * MINUTE, "r1")], + WINDOW, + WORKER, + ); + const stated = computeUsage([requested(T0 + 5 * MINUTE, "r1")], WINDOW, { + ...WORKER, + oldestTs: T0 - DAY, + }); + const empty = computeUsage([], WINDOW, WORKER); + + expect(partial).toMatchObject({ coversFrom: T0 + 5 * MINUTE, partial: true }); + expect(whole).toMatchObject({ coversFrom: T0, partial: false }); + expect(stated).toMatchObject({ coversFrom: T0, partial: false }); + expect(empty).toMatchObject({ coversFrom: T0, partial: false }); + }); + + it("picks 1-minute buckets for one hour, 15-minute buckets for one day and 1-hour buckets for seven days, and never more than 200 points for a 90-day window", () => { + expect(seriesBucketMs(HOUR)).toBe(MINUTE); + expect(seriesBucketMs(DAY)).toBe(15 * MINUTE); + expect(seriesBucketMs(7 * DAY)).toBe(HOUR); + + const ninety = computeUsage([], { from: T0, to: T0 + 90 * DAY }, WORKER); + expect(ninety.bucketMs).toBe(DAY); + expect(ninety.series).toHaveLength(90); + // The edges: 200 minutes is the last window one minute serves, 201 moves to five. + expect(seriesBucketMs(200 * MINUTE)).toBe(MINUTE); + expect(seriesBucketMs(200 * MINUTE + 1)).toBe(5 * MINUTE); + expect(seriesBucketMs(1_000 * MINUTE)).toBe(5 * MINUTE); + expect(seriesBucketMs(1_000 * MINUTE + 1)).toBe(15 * MINUTE); + expect(seriesBucketMs(3_000 * MINUTE)).toBe(15 * MINUTE); + expect(seriesBucketMs(3_000 * MINUTE + 1)).toBe(HOUR); + expect(seriesBucketMs(200 * HOUR)).toBe(HOUR); + expect(seriesBucketMs(200 * HOUR + 1)).toBe(6 * HOUR); + expect(seriesBucketMs(1_200 * HOUR)).toBe(6 * HOUR); + expect(seriesBucketMs(1_200 * HOUR + 1)).toBe(DAY); + }); + + it("lists requesters by request count with the label the caller supplied", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "tok_b"), + granted(T0 + 2_000, "r1", "l1", "warm", "tok_b"), + released(T0 + 7_000, "l1"), + requested(T0 + 3_000, "r2", "tok_a"), + requested(T0 + 4_000, "r3", "tok_a"), + rejected(T0 + 5_000, "r3", "timeout", "tok_a"), + requested(T0 + 6_000, "r4", "tok_a"), + ], + WINDOW, + { ...WORKER, labels: { tok_b: "ci-bot" } }, + ); + + expect(usage.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: "tok_a", rejected: 1, requests: 3 }, + { granted: 1, heldTotalMs: 5_000, id: "tok_b", label: "ci-bot", rejected: 0, requests: 1 }, + ]); + }); + + it("gives a worker one entry for itself, named by the workerId it is given", () => { + const usage = computeUsage( + [requested(T0 + 1_000, "r1"), granted(T0 + 2_000, "r1", "l1")], + WINDOW, + { fleet: false, workerId: "wrk_me", workerLabels: { wrk_me: "my-mac" } }, + ); + + expect(usage.workers).toHaveLength(1); + expect(usage.workers[0]).toMatchObject({ + granted: 1, + id: "wrk_me", + label: "my-mac", + requests: 1, + }); + }); + + it("counts failures that carry no reason of their own by event name under errors.byCode", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.purge-failed", { + attemptedStrategy: "erase", + deviceId: "d1", + duration: 1, + error: "x", + leaseId: "l", + }), + at(T0 + 2_000, "device.purge-failed", { + attemptedStrategy: "erase", + deviceId: "d2", + duration: 1, + error: "x", + leaseId: "l", + }), + at(T0 + 3_000, "component.install-failed", { + componentId: "26.4", + durationMs: 1, + error: "x", + platform: "ios", + }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.errors.byCode).toEqual({ + "component.install-failed": 1, + "device.purge-failed": 2, + }); + }); +}); diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts new file mode 100644 index 00000000..18d29d85 --- /dev/null +++ b/src/core/usage/compute-usage.ts @@ -0,0 +1,59 @@ +import type { EventEnvelope } from "../../bus/index.js"; +import type { UsageFigures, UsageOutput } from "../../contract/index.js"; + +export interface UsageWindow { + readonly from: number; + readonly to: number; +} + +export interface UsageOptions { + readonly fleet: boolean; + readonly labels?: Readonly>; + readonly requesterPrefix?: string; + readonly workerId?: string; + readonly workerLabels?: Readonly>; + readonly oldestTs?: number; +} + +const NO_SAMPLES = { count: 0, max: null, p50: null, p95: null }; + +function emptyFigures(): UsageFigures { + return { + boot: NO_SAMPLES, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + errors: { byCode: {} }, + granted: 0, + held: NO_SAMPLES, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: NO_SAMPLES, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: NO_SAMPLES, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: NO_SAMPLES, + }; +} + +/** The series bucket width for a window of `spanMs` (ADR 0016 §8). */ +export function seriesBucketMs(_spanMs: number): number { + return 60_000; +} + +export function computeUsage( + _events: readonly EventEnvelope[], + window: UsageWindow, + _options: UsageOptions, +): UsageOutput { + return { + bucketMs: 60_000, + coversFrom: window.from, + partial: false, + platforms: { android: emptyFigures(), ios: emptyFigures() }, + requesters: [], + series: [], + totals: emptyFigures(), + window, + workers: [], + }; +} diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 28d42874..243501df 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -91,8 +91,8 @@ function stallOptions( function resolveEventHistoryOverride( eventBus: EventBus, filesystem: MemoryFilesystem, - override: Pick | undefined, -): Pick { + override: Pick | undefined, +): Pick { return ( override ?? new EventHistory({ bus: eventBus, filesystem, logger: new NoopLogger(), path: "/events.jsonl" }) @@ -189,7 +189,7 @@ async function buildDispatcher( * takes no `--set`), rather than `ScriptedProcessRunner`'s scripted chunks. */ readonly passthroughOverride?: PassthroughResolver; /** Stands in for the event history, so `events.replay` can be checked against it. */ - readonly eventHistory?: Pick; + readonly eventHistory?: Pick; /** `status.get`'s host block; a fixed machine with no tools by default. */ readonly hostFacts?: () => HostFacts; /** Replaces the capacity block, for a test about one strategy's options. */ @@ -1093,6 +1093,7 @@ describe("Dispatcher: events.replay", () => { const asked: unknown[] = []; const { dispatcher } = await buildDispatcher({ eventHistory: { + ...noUsageHistory, replay: async (input) => { asked.push(input); return fromHistory; @@ -1107,6 +1108,116 @@ describe("Dispatcher: events.replay", () => { }); }); +/** The parts of the history `usage.get` reads, for a test that is not about it. */ +const noUsageHistory = { + latestId: () => undefined, + read: async () => ({ events: [], oldestTs: undefined }), +}; + +describe("Dispatcher: usage.get", () => { + const HOUR = 3_600_000; + const WINDOW = { from: 10 * HOUR, to: 11 * HOUR }; + + /** A history that counts its reads and whose newest event the test moves. */ + function countingHistory(oldestTs: number | undefined = undefined) { + const state = { newest: "evt_1" as string | undefined, reads: 0 }; + const history = { + latestId: () => state.newest, + read: async () => { + state.reads += 1; + return { events: [], oldestTs }; + }, + replay: async () => [], + }; + return { history, state }; + } + + it("the worker handler answers a second call inside the same bucket from the memo without reading the history, and reads again after an event arrives or the bucket moves", async () => { + const { history, state } = countingHistory(); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + const call = (window: { from: number; to: number }) => + dispatcher.dispatch("usage.get", window, session({ role: "admin" })); + + const first = await call(WINDOW); + const second = await call({ from: WINDOW.from + 30_000, to: WINDOW.to + 30_000 }); + + expect(state.reads).toBe(1); + expect(second).toEqual(first); + // The answer's window is the rounded one, not the one asked for. + expect(second.window).toEqual(WINDOW); + + state.newest = "evt_2"; + await call(WINDOW); + expect(state.reads).toBe(2); + + await call({ from: WINDOW.from + 60_000, to: WINDOW.to + 60_000 }); + expect(state.reads).toBe(3); + }); + + it("the worker handler answers HISTORY_NOT_KEPT with the oldest held timestamp when the window ends before it", async () => { + const { history } = countingHistory(WINDOW.to + 5 * HOUR); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch("usage.get", WINDOW, session({ role: "admin" })), + ).rejects.toMatchObject({ + code: "HISTORY_NOT_KEPT", + details: { oldestTs: WINDOW.to + 5 * HOUR }, + }); + }); + + it("the worker handler answers for itself as one worker, with the token label beside a requester it knows", async () => { + const { dispatcher, eventBus, tokens } = await buildDispatcher(); + const { record } = await tokens.create("agent", "ci-bot"); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: record.id, + waitPolicy: "wait", + }, + "test", + ); + + const usage = await dispatcher.dispatch( + "usage.get", + { from: 0, to: 600_000 }, + session({ role: "admin" }), + ); + + expect(usage.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: record.id, label: "ci-bot", rejected: 0, requests: 1 }, + ]); + expect(usage.workers.map((worker) => worker.id)).toEqual(["instance-1"]); + expect(usage.totals.requests).toBe(1); + }); + + it("usage.get is an admin operation an agent token cannot call", async () => { + const { history, state } = countingHistory(); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch("usage.get", WINDOW, session({ role: "agent" })), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(state.reads).toBe(0); + }); + + it("refuses a window that is not ordered or is longer than ninety days as a bad request", async () => { + const { dispatcher } = await buildDispatcher(); + const call = (window: unknown) => + dispatcher.dispatch("usage.get", window, session({ role: "admin" })); + + await expect(call({ from: 5, to: 5 })).rejects.toMatchObject({ code: "BAD_REQUEST" }); + await expect(call({ from: 0, to: 91 * 24 * HOUR })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + await expect(call({ from: 0, to: 90 * 24 * HOUR })).resolves.toMatchObject({ + window: { from: 0 }, + }); + }); +}); + describe("Dispatcher: nuke.run", () => { it("rejects an agent session with FORBIDDEN, without deleting anything", async () => { const { dispatcher, registry } = await buildDispatcher({ includeNuke: true }); diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index 91600cd8..46f25880 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -103,7 +103,7 @@ export interface DispatcherOptions { readonly config: Config; readonly doctor?: Doctor; /** Answers `events.replay`: the ring, or the event file for a `sinceTs`. */ - readonly eventHistory: Pick; + readonly eventHistory: Pick; readonly leases: LeaseCommands; readonly logger?: Logger; /** Classifies a thrown error for the `operation` log line (`classifyError` in production). @@ -260,6 +260,7 @@ export class Dispatcher { "nuke.run": this.#nukeRun, "config.get": this.#configGet, "events.replay": this.#eventsReplay, + "usage.get": this.#usageGet, "events.subscribe": this.#eventsSubscribe, "events.unsubscribe": this.#eventsUnsubscribe, "token.create": this.#tokenCreate, @@ -629,6 +630,10 @@ export class Dispatcher { #eventsReplay: Handler<"events.replay"> = (input) => this.options.eventHistory.replay(input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }); + #usageGet: Handler<"usage.get"> = () => { + throw new DispatchError("INTERNAL", "usage.get is not implemented"); + }; + #eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { const subscriptionId = session.manageEventSubscription(true); if (subscriptionId === undefined) { diff --git a/src/daemon/server.ts b/src/daemon/server.ts index cfd0c813..409c5f2c 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -146,7 +146,7 @@ export interface DaemonServerEngineOptions { readonly components: Pick; readonly doctor?: Doctor; /** What `events.replay` answers from; see `EventHistory`. */ - readonly eventHistory: Pick; + readonly eventHistory: Pick; readonly leases: LeaseCommands; /** `status.get`'s host block (ADR 0008 §5); see `DispatcherOptions.hostFacts`. */ readonly hostFacts: () => HostFacts; @@ -1090,6 +1090,12 @@ export class DaemonServer { frame.payload ?? {}, this.#session(connection), ); + case "usage.get": + return this.#dispatcher.dispatch( + "usage.get", + frame.payload ?? {}, + this.#session(connection), + ); case "events.subscribe": return this.#dispatcher.dispatch( "events.subscribe", diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index e52ba6c6..b012381b 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -75,6 +75,8 @@ const gatewayConfig = { }; class FakeTokens implements GatewayTokenStore { + constructor(private readonly records: readonly { id: string; label?: string }[] = []) {} + readonly created: string[] = []; readonly revoked: string[] = []; @@ -87,7 +89,10 @@ class FakeTokens implements GatewayTokenStore { } async list() { - return [{ createdAt: 1, id: "tok_1", role: "worker" as const }]; + return [ + { createdAt: 1, id: "tok_1", role: "worker" as const }, + ...this.records.map((record) => ({ createdAt: 1, role: "agent" as const, ...record })), + ]; } async revoke(id: string) { @@ -123,7 +128,9 @@ class FakeDirectory implements WorkerDirectory { function harness( options: { - readonly eventHistory?: Pick; + readonly eventHistory?: Pick; + /** The tokens the gateway's store lists besides its worker token; none by default. */ + readonly tokenRecords?: readonly { id: string; label?: string }[]; /** The gateway's `http` block; enabled on 127.0.0.1:4700 by default. */ readonly http?: (typeof gatewayConfig)["http"]; } = {}, @@ -138,7 +145,7 @@ function harness( leaseMaxTtlMs: gatewayConfig.lease.maxTtlMs, retentionMs: 24 * 60 * 60_000, }); - const tokens = new FakeTokens(); + const tokens = new FakeTokens(options.tokenRecords); /** C-2: every id `closeUplinksForToken` was actually called with, in call order. */ const closedUplinkTokens: string[] = []; const directory = new FakeDirectory(); @@ -554,6 +561,8 @@ describe("GatewayDispatcher", () => { const asked: unknown[] = []; const { dispatcher } = harness({ eventHistory: { + latestId: () => undefined, + read: async () => ({ events: [], oldestTs: undefined }), replay: async (input) => { asked.push(input); return fromHistory; @@ -567,6 +576,108 @@ describe("GatewayDispatcher", () => { expect(asked).toEqual([{ sinceTs: 10 }]); }); + it("the gateway handler strips its own gw: prefix before the label lookup and leaves another prefix alone", async () => { + const { dispatcher, eventBus } = harness({ + tokenRecords: [{ id: "tok_a", label: "ci-bot" }], + }); + for (const [index, requester] of [ + `${GATEWAY_REQUESTER_PREFIX}tok_a`, + "gw:other-instance:tok_a", + "tok_a", + ].entries()) { + eventBus.emit( + "lease.requested", + { + requestId: `req_${index}`, + requestSpec: { platform: "ios" }, + requester, + waitPolicy: "wait", + }, + "gateway", + ); + } + + const usage = await dispatcher.dispatch("usage.get", { from: 0, to: 600_000 }, session()); + + expect( + Object.fromEntries(usage.requesters.map((requester) => [requester.id, requester.label])), + ).toEqual({ + "gw:instance-1:tok_a": "ci-bot", + "gw:other-instance:tok_a": undefined, + tok_a: "ci-bot", + }); + }); + + it("answers usage.get with fleet totals and one entry per worker, labelled from the registry", async () => { + const { dispatcher, eventBus, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: "agent-1", + waitPolicy: "wait", + }, + "gateway", + ); + eventBus.emit( + "request.dispatched", + { + model: "m", + platform: "ios", + queuedMs: 0, + reason: "warm-hit", + requestId: "req_1", + requesterId: "agent-1", + stage: "warm-hit", + workerId: "wrk_1", + }, + "gateway", + ); + eventBus.republish({ + event: "lease.granted", + id: "evt_worker_1", + module: "lease-lifecycle", + payload: { + deviceId: "dev_1", + leaseId: "l1", + requestId: "wreq_1", + requester: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + source: "warm", + workerId: "wrk_1", + } as never, + timestamp: 1_000, + }); + + const usage = await dispatcher.dispatch("usage.get", { from: 0, to: 600_000 }, session()); + + expect(usage.totals).toMatchObject({ granted: 1, requests: 1 }); + expect(usage.workers).toMatchObject([{ granted: 1, id: "wrk_1", label: "mac-mini-1" }]); + }); + + it("the gateway handler answers HISTORY_NOT_KEPT with the oldest held timestamp, and refuses an agent token before it reads any history", async () => { + let reads = 0; + const { dispatcher } = harness({ + eventHistory: { + latestId: () => "evt_1", + read: async () => { + reads += 1; + return { events: [], oldestTs: 99 * 3_600_000 }; + }, + replay: async () => [], + }, + }); + + await expect( + dispatcher.dispatch("usage.get", { from: 0, to: 3_600_000 }, session()), + ).rejects.toMatchObject({ code: "HISTORY_NOT_KEPT", details: { oldestTs: 99 * 3_600_000 } }); + await expect( + dispatcher.dispatch("usage.get", { from: 0, to: 3_600_000 }, session({ role: "agent" })), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(reads).toBe(1); + }); + it("mints and revokes its own tokens, worker join tokens included", async () => { const { dispatcher, tokens } = harness(); diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index b5e8de31..33408ff5 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -96,7 +96,7 @@ export interface GatewayDispatcherOptions { /** The gateway's own config -- what `config.get` returns (ADR 0005 §34). */ readonly config: GatewayConfig; /** Answers `events.replay`: the ring, or the event file for a `sinceTs`. */ - readonly eventHistory: Pick; + readonly eventHistory: Pick; readonly workers: WorkerRegistry; /** Each worker's live link, for `worker.install-component` (ADR 0010 §7). `GatewayService` * satisfies it. */ @@ -164,6 +164,7 @@ export class GatewayDispatcher { "status.get": this.#statusGet, "config.get": this.#configGet, "events.replay": this.#eventsReplay, + "usage.get": this.#usageGet, "events.subscribe": this.#eventsSubscribe, "events.unsubscribe": this.#eventsUnsubscribe, "token.create": this.#tokenCreate, @@ -263,6 +264,10 @@ export class GatewayDispatcher { #eventsReplay: Handler<"events.replay"> = (input) => this.options.eventHistory.replay(input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }); + #usageGet: Handler<"usage.get"> = () => { + throw new DispatchError("INTERNAL", "usage.get is not implemented"); + }; + #eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { const subscriptionId = session.manageEventSubscription(true); if (subscriptionId === undefined) { diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index 8e294f45..411d366d 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -218,6 +218,53 @@ describe("connectSimlock: handshake", () => { await expect(callPromise).resolves.toEqual(answer); }); + it("asks usage.get for a window from an admin client and returns the daemon's answer, refusing a window the contract bounds before sending it", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const client = await connectPromise; + const none = { count: 0, max: null, p50: null, p95: null }; + const figures = { + boot: none, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + errors: { byCode: {} }, + granted: 0, + held: none, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: none, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: none, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: none, + }; + const answer = { + bucketMs: 60_000, + coversFrom: 1_000, + partial: false, + platforms: { android: figures, ios: figures }, + requesters: [], + series: [], + totals: figures, + window: { from: 1_000, to: 3_601_000 }, + workers: [], + }; + + const callPromise = client.usage({ from: 1_000, to: 3_601_000 }); + await flushMicrotasks(); + const call = connection.lastSentOf("usage.get")!; + expect(call.payload).toEqual({ from: 1_000, to: 3_601_000 }); + connection.reply(call.id, answer); + + await expect(callPromise).resolves.toEqual(answer); + + const sentBefore = connection.sent.length; + await expect(client.usage({ from: 5, to: 5 })).rejects.toMatchObject({ code: "BAD_REQUEST" }); + expect(connection.sent).toHaveLength(sentBefore); + }); + it("wraps a malformed daemon response instead of throwing a raw parse failure", async () => { const connection = new ScriptedConnection(); const connectPromise = connectSimlock({ connection }); diff --git a/src/simlock-client/client.ts b/src/simlock-client/client.ts index 3001d961..13c639ce 100644 --- a/src/simlock-client/client.ts +++ b/src/simlock-client/client.ts @@ -49,6 +49,8 @@ import type { ExecOptions, EventsReplayInput, EventsReplayOutput, + UsageGetInput, + UsageGetOutput, InstallComponentOnWorkersOptions, InstallComponentOptions, LeaseCancelInput, @@ -109,6 +111,8 @@ export type { ExecOptions, EventsReplayInput, EventsReplayOutput, + UsageGetInput, + UsageGetOutput, EventsSubscribeOutput, EventsUnsubscribeOutput, InstallComponentOnWorkersOptions, @@ -237,6 +241,8 @@ export interface SimlockAdminClient extends SimlockClient { * does not gate this operation by role at all -- see the caveat in the PR report). */ stopDaemon(): Promise; replayEvents(input?: EventsReplayInput): Promise; + /** ADR 0016: the usage figures for a window, computed by the daemon from its event history. */ + usage(window: UsageGetInput): Promise; subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise>; createToken(input: TokenCreateInput): Promise; listTokens(): Promise; @@ -396,6 +402,7 @@ function buildDegradedClient( }, ), replayEvents: () => rejected(), + usage: () => rejected(), subscribeEvents: () => rejected(), createToken: () => rejected(), listTokens: () => rejected(), @@ -575,6 +582,10 @@ class SimlockClientImpl { return this.#call("events.replay", input); } + usage(_window: UsageGetInput): Promise { + return Promise.reject(new Error("usage is not implemented")); + } + async subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise> { const unsubscribePush = this.#wire.onEvent((payload) => listener(payload as EventPush)); const result = await this.#call("events.subscribe", {}); diff --git a/src/simlock-client/types.ts b/src/simlock-client/types.ts index d290c3e6..42f332d9 100644 --- a/src/simlock-client/types.ts +++ b/src/simlock-client/types.ts @@ -55,6 +55,8 @@ export type SimlockConfig = OpOutput<"config.get">; export type DaemonStopOutput = OpOutput<"daemon.stop">; export type EventsReplayInput = OpInput<"events.replay">; export type EventsReplayOutput = OpOutput<"events.replay">; +export type UsageGetInput = OpInput<"usage.get">; +export type UsageGetOutput = OpOutput<"usage.get">; export type EventsSubscribeOutput = OpOutput<"events.subscribe">; export type EventsUnsubscribeOutput = OpOutput<"events.unsubscribe">; export type TokenCreateInput = OpInput<"token.create">; From 9debfbd478c8a1b145ed58a0268fab442a859ed2 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 16:29:09 +0200 Subject: [PATCH 02/23] feat(bus): the history carries the step in force at the start of a window 50 failing -> 45 failing --- src/bus/event-file.test.ts | 2 +- src/bus/event-file.ts | 81 ++++++++++++++++++++++++++++++-------- src/bus/index.ts | 7 ++++ 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index 20d0d9e9..a38b3ee4 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -532,10 +532,10 @@ describe("readEventFile", () => { // Nothing is at or before 50, so nothing is carried and nothing is repeated. expect(none.map((entry) => entry.id)).toEqual([ "evt_1", - "evt_2", "evt_3", "evt_4", "evt_5", + "evt_2", "evt_6", "evt_7", "evt_8", diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index 66d429ac..f20b15ea 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -8,6 +8,9 @@ export const EVENT_FILE_NAME = "events.jsonl"; /** * Every envelope in the event file newer than `sinceTs`, by `timestamp` then `seq` (ADR 0014 §5). + * With `carry`, also the latest envelope of each named event at or before `sinceTs` -- one per + * `workerId` where the payload names one -- so a reader of a step function (ADR 0016 §3) is told + * the step in force when its window opens. * The current file is read first, then its generations `.1`, `.2` and so on until * one is missing (ADR 0016 §4). Newest first means a rotation landing mid-read can only make a * generation show up twice -- never make one go missing -- and the repeat is dropped by `id`. @@ -33,22 +36,43 @@ export async function readEventFile( export async function readEventHistory( filesystem: Filesystem, path: string, - { sinceTs }: { readonly sinceTs: number; readonly carry?: readonly EventName[] }, + { sinceTs, carry = [] }: { readonly sinceTs: number; readonly carry?: readonly EventName[] }, ): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { const generations = await readGenerations(filesystem, path); const seen = new Set(); const events: EventEnvelope[] = []; + const carried = new Map(); let oldestTs: number | undefined; // Oldest generation first, so events that tie on time and seq keep the order they were written. for (const line of generations.reverse().flat()) { const envelope = parseEnvelope(line); if (envelope === undefined) continue; oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); - if (envelope.timestamp <= sinceTs || seen.has(eventKey(envelope))) continue; + if (envelope.timestamp <= sinceTs) { + keepIfCarried(carried, envelope, carry); + continue; + } + if (seen.has(eventKey(envelope))) continue; seen.add(eventKey(envelope)); events.push(envelope); } - return { events: events.sort(byTimeThenSeq), oldestTs }; + return { events: [...carried.values(), ...events].sort(byTimeThenSeq), oldestTs }; +} + +/** + * Keeps `envelope` as the step in force for its event and worker when it is a named event and no + * later one is kept already. Ties on time and seq go to the one met last, the one written last. + */ +function keepIfCarried( + carried: Map, + envelope: EventEnvelope, + carry: readonly EventName[], +): void { + if (!carry.includes(envelope.event)) return; + const workerId = (envelope.payload as { readonly workerId?: unknown }).workerId; + const key = `${envelope.event}\u0000${typeof workerId === "string" ? workerId : ""}`; + const kept = carried.get(key); + if (kept === undefined || byTimeThenSeq(kept, envelope) <= 0) carried.set(key, envelope); } /** The lines of the current file, then of each generation, newest first, until two in a row are missing. */ @@ -142,33 +166,58 @@ export class EventHistory { /** * Without `sinceTs`, the ring, as `simlock events` has always answered. With it, the event - * file while the writer is writing; the ring when there is no file to trust. + * file while the writer is writing; the ring when there is no file to trust. `carry` adds + * the step in force for each named event at `sinceTs` (see `readEventFile`). */ async replay( input: { readonly sinceTs?: number; readonly carry?: readonly EventName[] } = {}, ): Promise { - const { bus, filesystem, logger, path } = this.#options; - if (input.sinceTs === undefined) return bus.replay(); - if (!this.#writing) return bus.replay({ sinceTs: input.sinceTs }); + if (input.sinceTs === undefined) return this.#options.bus.replay(); + return (await this.read({ carry: input.carry ?? [], sinceTs: input.sinceTs })).events; + } + + /** + * What `replay` answers for `sinceTs`, with the oldest timestamp the history holds -- not only + * what is newer than `sinceTs` -- so a reader can say how far back the history reaches (ADR + * 0016 §5). From the ring when there is no file to trust, and then it is the ring's oldest. + */ + async read(input: { + readonly sinceTs: number; + readonly carry: readonly EventName[]; + }): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { + const { filesystem, logger, path } = this.#options; + if (!this.#writing) return this.#readRing(input); try { - return await readEventFile(filesystem, path, { sinceTs: input.sinceTs }); + return await readEventHistory(filesystem, path, input); } catch (error: unknown) { logger.warn("Event file read failed; replaying from memory", { error: error instanceof Error ? error.message : String(error), path, }); - return bus.replay({ sinceTs: input.sinceTs }); + return this.#readRing(input); } } - async read(_input: { - readonly sinceTs: number; - readonly carry: readonly EventName[]; - }): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { - return { events: [], oldestTs: undefined }; + /** The id of the event published last, `undefined` before the first. */ + latestId(): string | undefined { + return this.#options.bus.latestId(); } - latestId(): string | undefined { - return undefined; + #readRing({ + sinceTs, + carry, + }: { + readonly sinceTs: number; + readonly carry: readonly EventName[]; + }): { readonly events: EventEnvelope[]; readonly oldestTs: number | undefined } { + const ring = this.#options.bus.replay(); + const carried = new Map(); + let oldestTs: number | undefined; + for (const envelope of ring) { + oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); + if (envelope.timestamp <= sinceTs) keepIfCarried(carried, envelope, carry); + } + const newer = ring.filter((envelope) => envelope.timestamp > sinceTs); + return { events: [...carried.values(), ...newer].sort(byTimeThenSeq), oldestTs }; } } diff --git a/src/bus/index.ts b/src/bus/index.ts index 2e42ceea..14125dd3 100644 --- a/src/bus/index.ts +++ b/src/bus/index.ts @@ -376,6 +376,7 @@ export class EventBus { readonly #events: Array; #nextEventIndex = 0; #eventCount = 0; + #latestId: string | undefined; constructor( private readonly clock: Clock, @@ -461,7 +462,13 @@ export class EventBus { .sort(byTimeThenSeq); } + /** The id of the envelope published last, by arrival; `undefined` before the first. */ + latestId(): string | undefined { + return this.#latestId; + } + #append(envelope: EventEnvelope): void { + this.#latestId = envelope.id; this.#events[this.#nextEventIndex] = envelope; this.#nextEventIndex = (this.#nextEventIndex + 1) % this.capacity; if (this.#eventCount < this.capacity) { From 79aa1448f56fd89602e791c12e5f6b3aed15b83e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 16:34:44 +0200 Subject: [PATCH 03/23] feat(usage): computeUsage reads the figures out of a window's events 45 failing -> 17 failing --- src/core/usage/compute-usage.test.ts | 20 +- src/core/usage/compute-usage.ts | 330 ++++++++++++++++-- src/core/usage/read-events.ts | 500 +++++++++++++++++++++++++++ src/core/usage/timeline.test.ts | 66 ++++ src/core/usage/timeline.ts | 81 +++++ 5 files changed, 955 insertions(+), 42 deletions(-) create mode 100644 src/core/usage/read-events.ts create mode 100644 src/core/usage/timeline.test.ts create mode 100644 src/core/usage/timeline.ts diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index 0d077e1d..c141caf3 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -9,7 +9,7 @@ const HOUR = 60 * MINUTE; const DAY = 24 * HOUR; const T0 = 1_000_000_000; const WINDOW = { from: T0, to: T0 + HOUR }; -const WORKER: UsageOptions = { fleet: false, workerId: "self" }; +const WORKER: UsageOptions = { fleet: false, workers: [{ id: "self" }] }; const FLEET: UsageOptions = { fleet: true, requesterPrefix: "gw:g1:" }; let sequence = 0; @@ -376,6 +376,19 @@ describe("computeUsage", () => { expect(usage.totals.requests).toBe(1); expect(usage.totals.rejected).toEqual({ byReason: { cancelled: 1 }, total: 1 }); + + // The history a window is read from holds nothing from before it, so a rejection whose + // request is not there is told from one refused at admission by its reason. + const unseen = computeUsage( + [ + rejected(T0 + 1_000, "r-old", "timeout"), + rejected(T0 + 2_000, "r-old-2", "boot-timeout"), + rejected(T0 + 3_000, "r-refused", "already-leased", "agent-b"), + ], + WINDOW, + WORKER, + ); + expect(unseen.totals.rejected).toEqual({ byReason: { "already-leased": 1 }, total: 1 }); }); it("reports null series points and excludes them from peak and mean before the first step when no step precedes the window", () => { @@ -452,7 +465,7 @@ describe("computeUsage", () => { }), ], WINDOW, - { ...FLEET, workerLabels: { w1: "mac-1" } }, + { ...FLEET, workers: [{ id: "w1", label: "mac-1" }] }, ); expect(usage.platforms.ios).toMatchObject({ granted: 1, requests: 1 }); @@ -482,7 +495,6 @@ describe("computeUsage", () => { workerId: "w1", }), // The worker's own copy of the same request. - requested(T0 + 1_200, "w-r1", "gw:g1:agent-a", "ios"), at(T0 + 1_300, "lease.requested", { requestId: "w-r1", requestSpec: { platform: "ios" }, @@ -687,7 +699,7 @@ describe("computeUsage", () => { const usage = computeUsage( [requested(T0 + 1_000, "r1"), granted(T0 + 2_000, "r1", "l1")], WINDOW, - { fleet: false, workerId: "wrk_me", workerLabels: { wrk_me: "my-mac" } }, + { fleet: false, workers: [{ id: "wrk_me", label: "my-mac" }] }, ); expect(usage.workers).toHaveLength(1); diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts index 18d29d85..cc64296f 100644 --- a/src/core/usage/compute-usage.ts +++ b/src/core/usage/compute-usage.ts @@ -1,59 +1,313 @@ +import { byTimeThenSeq } from "../../bus/order.js"; import type { EventEnvelope } from "../../bus/index.js"; import type { UsageFigures, UsageOutput } from "../../contract/index.js"; +import { + type CapacityStep, + type CapacityValue, + type DeviceFact, + type Platform, + type ReadEvents, + readEvents, + type RequestFact, + type UsageWindow, +} from "./read-events.js"; +import { peakAndMean, segmentsOf, type Step, valuesAt } from "./timeline.js"; -export interface UsageWindow { - readonly from: number; - readonly to: number; -} +export type { UsageWindow } from "./read-events.js"; export interface UsageOptions { + /** The gateway rule (ADR 0016 §6): request facts from the gateway's own events, device facts + * from the events its workers relayed. */ readonly fleet: boolean; + /** Requester id to label, for the requesters the caller found a label for (ADR 0016 §7). */ readonly labels?: Readonly>; + /** What a gateway puts in front of a requester it forwards (`gw::`). A fleet request + * is matched with its worker's events by the requester carrying it. */ readonly requesterPrefix?: string; - readonly workerId?: string; - readonly workerLabels?: Readonly>; - readonly oldestTs?: number; + /** The workers there are to report: a worker's own entry, a gateway's registry. A worker the + * events name and this list does not is reported too. */ + readonly workers?: readonly { readonly id: string; readonly label?: string | undefined }[]; + /** The oldest timestamp the history holds, when the events passed start at the window's start + * and so cannot say. */ + readonly oldestTs?: number | undefined; } -const NO_SAMPLES = { count: 0, max: null, p50: null, p95: null }; +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +/** The widths a series may have, narrowest first (ADR 0016 §8). */ +const BUCKET_WIDTHS_MS = [MINUTE, 5 * MINUTE, 15 * MINUTE, HOUR, 6 * HOUR, 24 * HOUR]; +const MAX_SERIES_POINTS = 200; + +/** The series bucket width for a window of `spanMs`: the narrowest that keeps 200 points. */ +export function seriesBucketMs(spanMs: number): number { + return ( + BUCKET_WIDTHS_MS.find((width) => Math.ceil(spanMs / width) <= MAX_SERIES_POINTS) ?? + (BUCKET_WIDTHS_MS.at(-1) as number) + ); +} -function emptyFigures(): UsageFigures { +/** + * The usage figures for `window`, from the events of a history (ADR 0016). Pure: events in, numbers + * out. `events` are the envelopes the history holds for the window, with the step in force at its + * start for each capacity and queue timeline; any order, and any envelope outside the window is + * left out. + */ +export function computeUsage( + events: readonly EventEnvelope[], + window: UsageWindow, + options: UsageOptions, +): UsageOutput { + const sorted = [...events].sort(byTimeThenSeq); + const workers = options.workers ?? []; + const read = readEvents(sorted, window, { + fleet: options.fleet, + requesterPrefix: options.requesterPrefix ?? "", + self: workers[0]?.id ?? "local", + }); + const bucketMs = seriesBucketMs(window.to - window.from); + const oldest = options.oldestTs ?? sorted[0]?.timestamp; + const coversFrom = oldest === undefined ? window.from : Math.max(window.from, oldest); + const figures = (scope: Scope) => figuresFor(read, window, scope); + const labelOf = new Map(workers.map((worker) => [worker.id, worker.label])); + const ids = new Set([...labelOf.keys(), ...read.workers]); return { - boot: NO_SAMPLES, - bySource: { booted: 0, provisioned: 0, warm: 0 }, - errors: { byCode: {} }, - granted: 0, - held: NO_SAMPLES, - incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, - provisioning: NO_SAMPLES, - queue: { meanDepth: null, peakDepth: null }, - rejected: { byReason: {}, total: 0 }, - requests: 0, - turnaround: NO_SAMPLES, - utilisation: { slots: { max: null, mean: null, peak: null } }, - wait: NO_SAMPLES, + bucketMs, + coversFrom, + partial: coversFrom > window.from, + platforms: { android: figures({ platform: "android" }), ios: figures({ platform: "ios" }) }, + requesters: requestersOf(read.requests, options.labels ?? {}), + series: seriesOf(read, window, bucketMs), + totals: figures({ queue: true }), + window: { from: window.from, to: window.to }, + workers: [...ids].sort().map((id) => { + const label = labelOf.get(id); + return { + ...figures({ queue: !options.fleet, worker: id }), + id, + ...(label === undefined ? {} : { label }), + }; + }), }; } -/** The series bucket width for a window of `spanMs` (ADR 0016 §8). */ -export function seriesBucketMs(_spanMs: number): number { - return 60_000; +interface Scope { + readonly platform?: Platform; + readonly worker?: string; + /** Whether the figures carry the queue's depth, which is the host's or the fleet's, not a + * platform's or a worker's. */ + readonly queue?: boolean; } -export function computeUsage( - _events: readonly EventEnvelope[], +function inScope( + item: { readonly platform?: Platform | undefined; readonly worker?: string | undefined }, + scope: Scope, +): boolean { + return ( + (scope.platform === undefined || item.platform === scope.platform) && + (scope.worker === undefined || item.worker === scope.worker) + ); +} + +function figuresFor(read: ReadEvents, window: UsageWindow, scope: Scope): UsageFigures { + const requests = read.requests.filter((fact) => inScope(fact, scope)); + const devices = read.devices.filter((fact) => inScope(fact, scope)); + const asked = requests.filter((fact) => fact.requestedAt !== undefined); + const granted = requests.filter((fact) => fact.outcome?.kind === "granted"); + const rejected = requests.flatMap((fact) => + fact.outcome?.kind === "rejected" ? [fact.outcome.reason] : [], + ); + return { + ...deviceFigures(devices), + bySource: sourcesOf(granted), + granted: granted.length, + held: summarise(granted.flatMap((fact) => spanOf(fact.outcome?.at, fact.endedAt))), + queue: + scope.queue === true ? queueOf(read.queue, window) : { meanDepth: null, peakDepth: null }, + rejected: { byReason: countBy(rejected), total: rejected.length }, + requests: asked.length, + turnaround: summarise( + asked.flatMap((fact) => + fact.outcome?.kind === "granted" ? spanOf(fact.requestedAt, fact.endedAt) : [], + ), + ), + utilisation: utilisationOf(read.capacity, window, scope), + wait: summarise(asked.flatMap((fact) => spanOf(fact.requestedAt, fact.outcome?.at))), + }; +} + +/** The milliseconds from `start` to `end`, when both are known. */ +function spanOf(start: number | undefined, end: number | undefined): number[] { + return start === undefined || end === undefined ? [] : [end - start]; +} + +function sourcesOf(granted: readonly RequestFact[]): UsageFigures["bySource"] { + const sources = { booted: 0, provisioned: 0, warm: 0 }; + for (const fact of granted) { + const source = fact.outcome?.kind === "granted" ? fact.outcome.source : undefined; + if (source === "warm" || source === "booted" || source === "provisioned") sources[source] += 1; + } + return sources; +} + +/** The figures a device event brings: durations, incidents, and failures by name. */ +function deviceFigures( + devices: readonly DeviceFact[], +): Pick { + const durations = (kind: DeviceFact["kind"]) => + summarise(devices.filter((fact) => fact.kind === kind).map((fact) => fact.value as number)); + const named = (kind: DeviceFact["kind"]) => + countBy(devices.filter((fact) => fact.kind === kind).map((fact) => fact.value as string)); + const incidents = named("incident"); + return { + boot: durations("boot"), + errors: { byCode: named("error") }, + incidents: { + crashRecovered: incidents.crashRecovered ?? 0, + lost: incidents.lost ?? 0, + quarantineRecovered: incidents.quarantineRecovered ?? 0, + quarantined: incidents.quarantined ?? 0, + }, + provisioning: durations("provisioning"), + }; +} + +function countBy(values: readonly string[]): Record { + const counts: Record = {}; + for (const value of values) counts[value] = (counts[value] ?? 0) + 1; + return counts; +} + +/** p50, p95 and max by nearest rank: the smallest sample at or above that share of the samples. */ +function summarise(values: readonly number[]): UsageFigures["wait"] { + if (values.length === 0) return { count: 0, max: null, p50: null, p95: null }; + const sorted = [...values].sort((left, right) => left - right); + const rank = (share: number) => + sorted[Math.max(0, Math.ceil(share * sorted.length) - 1)] as number; + return { count: sorted.length, max: sorted.at(-1) as number, p50: rank(0.5), p95: rank(0.95) }; +} + +function queueOf(steps: readonly Step[], window: UsageWindow): UsageFigures["queue"] { + const figures = peakAndMean(segmentsOf(steps, window.from, window.to), (values) => values[0]); + return figures === undefined + ? { meanDepth: null, peakDepth: null } + : { meanDepth: figures.mean, peakDepth: figures.peak }; +} + +const sumOf = (values: readonly number[]): number | undefined => + values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0); + +/** The RAM figure summed over the workers that report a budget; `undefined` when none does. */ +const ramOf = (values: readonly CapacityValue[], key: "used" | "limit"): number | undefined => + sumOf(values.flatMap((value) => (value.ram === undefined ? [] : [value.ram[key]]))); + +/** What one scope's capacity steps say: a platform's own entry, or the global one. */ +function scopedSteps(steps: readonly CapacityStep[], scope: Scope): Step[] { + const platform = scope.platform; + return steps + .filter((step) => scope.worker === undefined || step.key === scope.worker) + .map((step) => ({ + at: step.at, + key: step.key, + value: platform === undefined ? step.value : step.value.platforms[platform], + })); +} + +function utilisationOf( + steps: readonly CapacityStep[], window: UsageWindow, - _options: UsageOptions, -): UsageOutput { + scope: Scope, +): UsageFigures["utilisation"] { + const segments = segmentsOf(scopedSteps(steps, scope), window.from, window.to); + const slots = peakAndMean(segments, (values) => sumOf(values.map((value) => value.used))); + const ram = peakAndMean(segments, (values) => ramOf(values, "used")); + const maxima = segments.flatMap((segment) => + segment.end > segment.start ? (sumOf(segment.values.map((value) => value.max)) ?? []) : [], + ); + const limits = segments.flatMap((segment) => ramOf(segment.values, "limit") ?? []); return { - bucketMs: 60_000, - coversFrom: window.from, - partial: false, - platforms: { android: emptyFigures(), ios: emptyFigures() }, - requesters: [], - series: [], - totals: emptyFigures(), - window, - workers: [], + ...(ram === undefined + ? {} + : { ram: { limitBytes: limits.at(-1) as number, meanBytes: ram.mean, peakBytes: ram.peak } }), + slots: { + max: maxima.length === 0 ? null : Math.max(...maxima), + mean: slots?.mean ?? null, + peak: slots?.peak ?? null, + }, }; } + +function requestersOf( + requests: readonly RequestFact[], + labels: Readonly>, +): UsageOutput["requesters"] { + const byId = new Map(); + for (const fact of requests) { + const entry = byId.get(fact.requester) ?? { + granted: 0, + heldTotalMs: 0, + id: fact.requester, + rejected: 0, + requests: 0, + }; + byId.set(fact.requester, countRequest(entry, fact)); + } + return [...byId.values()] + .sort((left, right) => right.requests - left.requests || (left.id < right.id ? -1 : 1)) + .map((entry) => { + const label = Object.hasOwn(labels, entry.id) ? labels[entry.id] : undefined; + return label === undefined ? entry : { ...entry, label }; + }); +} + +type Requester = UsageOutput["requesters"][number]; + +/** `entry` with `fact` counted in it: a request made, a grant and the time it was held, a rejection. */ +function countRequest(entry: Requester, fact: RequestFact): Requester { + const { outcome } = fact; + const held = outcome?.kind === "granted" ? (spanOf(outcome.at, fact.endedAt)[0] ?? 0) : 0; + return { + ...entry, + granted: entry.granted + (outcome?.kind === "granted" ? 1 : 0), + heldTotalMs: entry.heldTotalMs + held, + rejected: entry.rejected + (outcome?.kind === "rejected" ? 1 : 0), + requests: entry.requests + (fact.requestedAt === undefined ? 0 : 1), + }; +} + +function seriesOf(read: ReadEvents, window: UsageWindow, bucketMs: number): UsageOutput["series"] { + const points = Math.ceil((window.to - window.from) / bucketMs); + const ends = Array.from({ length: points }, (_, index) => + Math.min(window.from + (index + 1) * bucketMs, window.to), + ); + const capacity = valuesAt(scopedSteps(read.capacity, {}), ends); + const depths = valuesAt(read.queue, ends); + const waiting = waitingAt(read.requests, ends); + return ends.map((at, index) => { + const values = capacity[index] as readonly CapacityValue[]; + const ram = ramOf(values, "used"); + return { + at, + queueDepth: (depths[index] as readonly number[])[0] ?? null, + ...(ram === undefined ? {} : { ramUsedBytes: ram }), + slotsMax: sumOf(values.map((value) => value.max)) ?? null, + slotsUsed: sumOf(values.map((value) => value.used)) ?? null, + waiting: waiting[index] as number, + }; + }); +} + +/** How many requests have been made and not yet settled at each of `times`, which ascend. */ +function waitingAt(requests: readonly RequestFact[], times: readonly number[]): number[] { + const ascending = (values: number[]) => values.sort((left, right) => left - right); + const made = ascending(requests.flatMap((fact) => fact.requestedAt ?? [])); + const settled = ascending( + requests.flatMap((fact) => (fact.requestedAt === undefined ? [] : (fact.outcome?.at ?? []))), + ); + let madeBy = 0; + let settledBy = 0; + return times.map((time) => { + while (madeBy < made.length && (made[madeBy] as number) <= time) madeBy += 1; + while (settledBy < settled.length && (settled[settledBy] as number) <= time) settledBy += 1; + return madeBy - settledBy; + }); +} diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts new file mode 100644 index 00000000..2eab2751 --- /dev/null +++ b/src/core/usage/read-events.ts @@ -0,0 +1,500 @@ +import type { EventEnvelope } from "../../bus/index.js"; +import type { UsageFigures } from "../../contract/index.js"; +import type { Step } from "./timeline.js"; + +/** + * Reads a window's events into the facts the figures count (ADR 0016 §2, §6): one fact for each + * request, one for each device event that counts, and the capacity and queue steps. Nothing here + * adds a number up. + */ + +export type Platform = "ios" | "android"; + +export interface RequestFact { + readonly requester: string; + readonly platform: Platform | undefined; + /** The worker that served it: a worker's own, or the one a gateway dispatched it to. */ + readonly worker: string | undefined; + /** `undefined` for a rejection refused before the request was stored, which is no request. */ + readonly requestedAt: number | undefined; + readonly outcome?: Granted | Rejected; + /** When the lease ended, if the window saw it. */ + readonly endedAt?: number; +} + +type RequestFactBase = Pick; + +interface Granted { + readonly kind: "granted"; + readonly at: number; + readonly source: string; +} + +interface Rejected { + readonly kind: "rejected"; + readonly at: number; + readonly reason: string; +} + +export interface DeviceFact { + readonly kind: "provisioning" | "boot" | "incident" | "error"; + readonly worker: string | undefined; + readonly platform: Platform | undefined; + /** The duration for the first two, the incident or error name for the others. */ + readonly value: number | string; +} + +export interface CapacityValue { + readonly used: number; + readonly max: number; + readonly ram: { readonly used: number; readonly limit: number } | undefined; +} + +export type CapacityStep = Step< + CapacityValue & { readonly platforms: Readonly> } +>; + +export interface ReadEvents { + readonly requests: readonly RequestFact[]; + readonly devices: readonly DeviceFact[]; + readonly capacity: readonly CapacityStep[]; + readonly queue: readonly Step[]; + readonly workers: ReadonlySet; +} + +export interface ReadOptions { + readonly fleet: boolean; + /** The id a worker's own events are attributed to. */ + readonly self: string; + /** What a gateway puts in front of a requester it forwards. */ + readonly requesterPrefix: string; +} + +export interface UsageWindow { + readonly from: number; + readonly to: number; +} + +type Payload = Readonly>; +type Incident = keyof UsageFigures["incidents"]; + +/** A rejection whose request the events do not show was refused before it was stored, or belongs + * to a request from before the window; only these two reasons are ever given before admission. */ +const REFUSED_AT_ADMISSION = new Set(["already-leased", "killed"]); +/** The device events that count as an incident, by what the figure calls them. */ +const INCIDENT_OF: Readonly> = { + "device.quarantine-abandoned": "lost", + "device.quarantine-recovered": "quarantineRecovered", + "device.quarantine-stranded": "lost", + "device.quarantined": "quarantined", + "device.recovered": "crashRecovered", + "device.recovery-failed": "lost", +}; +/** Failures that carry no reason of their own among the rejections, counted by their event name. */ +const ERROR_EVENTS = new Set(["device.purge-failed", "component.install-failed"]); + +const text = (payload: Payload, key: string): string | undefined => { + const value = payload[key]; + return typeof value === "string" ? value : undefined; +}; + +const number = (payload: Payload, key: string): number | undefined => { + const value = payload[key]; + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +}; + +const platformOf = (value: unknown): Platform | undefined => + value === "ios" || value === "android" ? value : undefined; + +const objectOf = (value: unknown): Payload | undefined => + typeof value === "object" && value !== null ? (value as Payload) : undefined; + +/** A relayed grant or rejection, kept for the request that will claim it. */ +interface Relayed { + readonly index: number; + readonly kind: "granted" | "rejected"; + readonly at: number; + readonly payload: Payload; +} + +interface Dispatch { + readonly index: number; + readonly requestId: string; + readonly requester: string; + readonly worker: string; +} + +interface Request { + readonly index: number; + readonly at: number; + readonly requester: string; + readonly platform: Platform | undefined; +} + +/** What one event brings to a handler. */ +interface Seen { + readonly event: EventEnvelope; + readonly index: number; + readonly payload: Payload; + /** The worker the fact is about: a worker's own id, or on a gateway the one that relayed it. */ + readonly worker: string | undefined; + /** A gateway's own fact, as against one a worker relayed. Always true on a worker. */ + readonly own: boolean; + readonly within: boolean; +} + +export function readEvents( + sorted: readonly EventEnvelope[], + window: UsageWindow, + options: ReadOptions, +): ReadEvents { + const reader = new Reader(window, options); + sorted.forEach((event, index) => reader.see(event, index)); + return reader.finish(); +} + +class Reader { + readonly #devices: DeviceFact[] = []; + readonly #capacity: CapacityStep[] = []; + readonly #queue: Step[] = []; + readonly #workers = new Set(); + readonly #devicePlatform = new Map(); + readonly #requested = new Map(); + readonly #ownRejections = new Map< + string, + Rejected & { readonly requester: string; readonly platform: Platform | undefined } + >(); + readonly #grants = new Map(); + readonly #ends = new Map(); + readonly #dispatches = new Map(); + readonly #boundaries = new Map(); + readonly #relayed = new Map(); + readonly #handlers: Record void> = { + "capacity.changed": (seen) => this.#capacityChanged(seen), + "device.provisioned": (seen) => this.#provisioned(seen), + "device.ready": (seen) => this.#ready(seen), + "lease.expired": (seen) => this.#ended(seen), + "lease.granted": (seen) => this.#granted(seen), + "lease.rejected": (seen) => this.#rejected(seen), + "lease.released": (seen) => this.#ended(seen), + "lease.requested": (seen) => this.#leaseRequested(seen), + "queue.changed": (seen) => this.#queueChanged(seen), + "request.dispatched": (seen) => this.#dispatched(seen), + }; + + constructor( + private readonly window: UsageWindow, + private readonly options: ReadOptions, + ) {} + + see(event: EventEnvelope, index: number): void { + const payload = (event.payload ?? {}) as Payload; + const relayedFrom = text(payload, "workerId"); + const seen: Seen = { + event, + index, + own: !this.options.fleet || event.event === "request.dispatched" || relayedFrom === undefined, + payload, + within: event.timestamp > this.window.from && event.timestamp <= this.window.to, + worker: this.options.fleet ? relayedFrom : this.options.self, + }; + (this.#handlers[event.event] ?? ((other) => this.#deviceEvent(other)))(seen); + } + + finish(): ReadEvents { + return { + capacity: this.#capacity, + devices: this.#devices, + queue: this.#queue, + requests: [...this.#requests(), ...this.#refusedAtAdmission()], + workers: this.#workers, + }; + } + + #key(worker: string | undefined, id: string): string { + return `${worker ?? ""}\u0000${id}`; + } + + /** On a gateway a device fact needs a worker behind it; on a worker it is always its own. */ + #counts(seen: Seen): boolean { + return seen.within && (!this.options.fleet || seen.worker !== undefined); + } + + #capacityChanged(seen: Seen): void { + const { event, payload, worker } = seen; + if (event.timestamp > this.window.to || (this.options.fleet && worker === undefined)) return; + const step = capacityStep(payload, event.timestamp, worker ?? ""); + if (step !== undefined) this.#capacity.push(step); + if (worker !== undefined) this.#workers.add(worker); + } + + #queueChanged(seen: Seen): void { + const depth = number(seen.payload, "depth"); + if (seen.event.timestamp > this.window.to || !seen.own || depth === undefined) return; + this.#queue.push({ at: seen.event.timestamp, key: "queue", value: depth }); + } + + #boundary(requester: string | undefined, index: number): void { + if (requester === undefined) return; + this.#boundaries.set(requester, [...(this.#boundaries.get(requester) ?? []), index]); + } + + #leaseRequested(seen: Seen): void { + if (!seen.own) return; + const requestId = text(seen.payload, "requestId"); + const requester = text(seen.payload, "requester"); + this.#boundary(requester, seen.index); + if (!seen.within || requestId === undefined || requester === undefined) return; + this.#requested.set(requestId, { + at: seen.event.timestamp, + index: seen.index, + platform: requestPlatform(seen.payload), + requester, + }); + } + + #dispatched(seen: Seen): void { + const requestId = text(seen.payload, "requestId"); + const requester = text(seen.payload, "requesterId"); + const worker = text(seen.payload, "workerId"); + this.#boundary(requester, seen.index); + if (!this.options.fleet || !seen.within) return; + if (requestId === undefined || requester === undefined || worker === undefined) return; + this.#dispatches.set(requestId, { index: seen.index, requestId, requester, worker }); + } + + #relay(seen: Seen, kind: Relayed["kind"]): void { + const requester = text(seen.payload, "requester"); + if (requester === undefined || seen.worker === undefined) return; + const key = this.#key(seen.worker, requester); + const relayed = { at: seen.event.timestamp, index: seen.index, kind, payload: seen.payload }; + this.#relayed.set(key, [...(this.#relayed.get(key) ?? []), relayed]); + } + + #rejected(seen: Seen): void { + const requestId = text(seen.payload, "requestId"); + const reason = text(seen.payload, "reason"); + if (!seen.within || requestId === undefined || reason === undefined) return; + const requester = text(seen.payload, "requester"); + if (seen.own) { + if (requester === undefined) return; + this.#ownRejections.set(requestId, { + at: seen.event.timestamp, + kind: "rejected", + platform: requestPlatform(seen.payload), + reason, + requester, + }); + } else { + this.#relay(seen, "rejected"); + } + } + + #granted(seen: Seen): void { + if (!seen.within) return; + if (this.options.fleet) { + this.#relay(seen, "granted"); + return; + } + const requestId = text(seen.payload, "requestId"); + const leaseId = text(seen.payload, "leaseId"); + if (requestId === undefined || leaseId === undefined) return; + this.#grants.set(requestId, { + at: seen.event.timestamp, + kind: "granted", + leaseId, + source: text(seen.payload, "source") ?? "", + }); + } + + #ended(seen: Seen): void { + const leaseId = text(seen.payload, "leaseId"); + if (!this.#counts(seen) || leaseId === undefined) return; + this.#ends.set(this.#key(seen.worker, leaseId), seen.event.timestamp); + } + + #provisioned(seen: Seen): void { + const deviceId = text(seen.payload, "deviceId"); + const platform = platformOf(objectOf(seen.payload.spec)?.platform); + const duration = number(seen.payload, "duration"); + if (seen.worker !== undefined && deviceId !== undefined && platform !== undefined) { + this.#devicePlatform.set(this.#key(seen.worker, deviceId), platform); + } + if (this.#counts(seen) && duration !== undefined) { + this.#addDevice(seen, { kind: "provisioning", platform, value: duration }); + } + } + + #ready(seen: Seen): void { + const deviceId = text(seen.payload, "deviceId"); + const duration = number(seen.payload, "bootDuration"); + if (!this.#counts(seen) || duration === undefined) return; + this.#addDevice(seen, { + kind: "boot", + platform: this.#platformOfDevice(seen, deviceId), + value: duration, + }); + } + + #deviceEvent(seen: Seen): void { + const incident = INCIDENT_OF[seen.event.event]; + if (!this.#counts(seen) || (incident === undefined && !ERROR_EVENTS.has(seen.event.event))) + return; + const platform = + platformOf(seen.payload.platform) ?? + this.#platformOfDevice(seen, text(seen.payload, "deviceId")); + this.#addDevice(seen, { + kind: incident === undefined ? "error" : "incident", + platform, + value: incident ?? seen.event.event, + }); + } + + #platformOfDevice(seen: Seen, deviceId: string | undefined): Platform | undefined { + return deviceId === undefined + ? undefined + : this.#devicePlatform.get(this.#key(seen.worker, deviceId)); + } + + #addDevice(seen: Seen, fact: Omit): void { + this.#devices.push({ ...fact, worker: seen.worker }); + if (seen.worker !== undefined) this.#workers.add(seen.worker); + } + + // ---- joining a request with what became of it ------------------------------------------- + + /** One fact for each request the window saw made, joined with its outcome and its end. */ + #requests(): RequestFact[] { + const facts: RequestFact[] = []; + for (const [requestId, request] of this.#requested) { + facts.push(this.#factFor(requestId, request)); + } + return facts; + } + + #factFor(requestId: string, request: Request): RequestFact { + const rejection = this.#ownRejections.get(requestId); + const own = this.options.fleet ? undefined : this.options.self; + const base = { + platform: request.platform, + requestedAt: request.at, + requester: request.requester, + }; + if (rejection !== undefined) { + const outcome: Rejected = { at: rejection.at, kind: "rejected", reason: rejection.reason }; + return { ...base, outcome, worker: own }; + } + return this.options.fleet + ? this.#fleetFact(base, requestId, request) + : this.#workerFact(base, requestId, own); + } + + #workerFact(base: RequestFactBase, requestId: string, own: string | undefined): RequestFact { + const grant = this.#grants.get(requestId); + return grant === undefined + ? { ...base, worker: own } + : this.#withEnd({ ...base, outcome: grant, worker: own }, grant.leaseId, own); + } + + /** A fleet request's outcome is the worker's, found through its dispatch (ADR 0016 §6). */ + #fleetFact(base: RequestFactBase, requestId: string, request: Request): RequestFact { + const dispatch = this.#dispatches.get(requestId); + if (dispatch === undefined) return { ...base, worker: undefined }; + this.#workers.add(dispatch.worker); + const worker = dispatch.worker; + const relayed = this.#fleetOutcome(dispatch, request.index); + if (relayed === undefined) return { ...base, worker }; + if (relayed.kind === "rejected") { + const reason = text(relayed.payload, "reason") ?? ""; + return { ...base, outcome: { at: relayed.at, kind: "rejected", reason }, worker }; + } + const source = text(relayed.payload, "source") ?? ""; + return this.#withEnd( + { ...base, outcome: { at: relayed.at, kind: "granted", source }, worker }, + text(relayed.payload, "leaseId") ?? "", + worker, + ); + } + + #withEnd(fact: RequestFact, leaseId: string, worker: string | undefined): RequestFact { + const endedAt = this.#ends.get(this.#key(worker, leaseId)); + return endedAt === undefined ? fact : { ...fact, endedAt }; + } + + /** + * The first relayed grant or rejection from the worker a request was dispatched to, for its + * namespaced requester, that falls before the requester's next request (ADR 0016 §6). A rejection + * must come at or after the dispatch. A grant may come before it: a warm device is granted + * before the answer that announces the dispatch reaches the gateway, so a grant counts from the + * request. + */ + #fleetOutcome(dispatch: Dispatch, requestIndex: number): Relayed | undefined { + const next = (this.#boundaries.get(dispatch.requester) ?? []).find( + (index) => index > dispatch.index, + ); + const key = this.#key(dispatch.worker, `${this.options.requesterPrefix}${dispatch.requester}`); + return (this.#relayed.get(key) ?? []).find( + (candidate) => + candidate.index >= (candidate.kind === "granted" ? requestIndex : dispatch.index) && + (next === undefined || candidate.index < next), + ); + } + + /** A gateway's or worker's own rejection of a request the window did not see made, when its + * reason is one given before admission. */ + #refusedAtAdmission(): RequestFact[] { + const facts: RequestFact[] = []; + for (const [requestId, rejection] of this.#ownRejections) { + if (this.#requested.has(requestId) || !REFUSED_AT_ADMISSION.has(rejection.reason)) continue; + facts.push({ + outcome: { at: rejection.at, kind: "rejected", reason: rejection.reason }, + platform: rejection.platform, + requestedAt: undefined, + requester: rejection.requester, + worker: this.options.fleet ? undefined : this.options.self, + }); + } + return facts; + } +} + +function requestPlatform(payload: Payload): Platform | undefined { + return platformOf(objectOf(payload.requestSpec)?.platform); +} + +function capacityEntry( + entry: unknown, +): + | { readonly running: number; readonly reserved: number; readonly maxRunning: number } + | undefined { + const figures = objectOf(entry); + if (figures === undefined) return undefined; + const running = number(figures, "running"); + const reserved = number(figures, "reserved"); + const maxRunning = number(figures, "maxRunning"); + return running === undefined || reserved === undefined || maxRunning === undefined + ? undefined + : { maxRunning, reserved, running }; +} + +/** A `capacity.changed` as a step, or `undefined` for a payload that is not one. */ +function capacityStep(payload: Payload, at: number, key: string): CapacityStep | undefined { + const global = capacityEntry(payload.global); + const ios = capacityEntry(payload.ios); + const android = capacityEntry(payload.android); + if (global === undefined || ios === undefined || android === undefined) return undefined; + const budget = objectOf(payload.ramBudget); + const used = budget === undefined ? undefined : number(budget, "usedBytes"); + const limit = budget === undefined ? undefined : number(budget, "limitBytes"); + const ram = used === undefined || limit === undefined ? undefined : { limit, used }; + const valueOf = (entry: NonNullable>): CapacityValue => ({ + max: entry.maxRunning, + ram, + used: entry.running + entry.reserved, + }); + return { + at, + key, + value: { ...valueOf(global), platforms: { android: valueOf(android), ios: valueOf(ios) } }, + }; +} diff --git a/src/core/usage/timeline.test.ts b/src/core/usage/timeline.test.ts new file mode 100644 index 00000000..7c7123dd --- /dev/null +++ b/src/core/usage/timeline.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; + +import { peakAndMean, segmentsOf, type Step, valuesAt } from "./timeline.js"; + +const step = (at: number, key: string, value: number): Step => ({ at, key, value }); +const sum = (values: readonly number[]): number | undefined => + values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0); + +describe("segmentsOf", () => { + it("starts with the step in force at the start of the window and cuts a segment at each later step", () => { + const segments = segmentsOf([step(5, "a", 1), step(20, "a", 3), step(40, "a", 0)], 10, 50); + + expect(segments).toEqual([ + { end: 20, start: 10, values: [1] }, + { end: 40, start: 20, values: [3] }, + { end: 50, start: 40, values: [0] }, + ]); + }); + + it("holds no value before the first step, and reads a step at the very start of the window as in force", () => { + const segments = segmentsOf([step(30, "a", 2)], 10, 50); + const atStart = segmentsOf([step(10, "a", 2)], 10, 50); + + expect(segments).toEqual([ + { end: 30, start: 10, values: [] }, + { end: 50, start: 30, values: [2] }, + ]); + expect(atStart).toEqual([{ end: 50, start: 10, values: [2] }]); + }); + + it("leaves out steps after the end of the window, and keeps one value for each key", () => { + const segments = segmentsOf( + [step(5, "a", 1), step(6, "b", 10), step(30, "a", 2), step(80, "b", 99)], + 10, + 50, + ); + + expect(segments).toEqual([ + { end: 30, start: 10, values: [1, 10] }, + { end: 50, start: 30, values: [2, 10] }, + ]); + }); +}); + +describe("valuesAt", () => { + it("answers the values in force at each time, counting a step at that time", () => { + const steps = [step(5, "a", 1), step(20, "a", 3), step(20, "b", 7)]; + + expect(valuesAt(steps, [4, 5, 19, 20, 99])).toEqual([[], [1], [1], [3, 7], [3, 7]]); + }); +}); + +describe("peakAndMean", () => { + it("weighs each segment by its time and leaves out the time before the first step", () => { + const segments = segmentsOf([step(30, "a", 4), step(40, "a", 0)], 0, 60); + + expect(peakAndMean(segments, sum)).toEqual({ mean: 40 / 30, peak: 4 }); + expect(peakAndMean(segmentsOf([], 0, 60), sum)).toBeUndefined(); + }); + + it("gives no weight to a segment with no time in it", () => { + const segments = segmentsOf([step(10, "a", 100), step(10, "a", 2)], 0, 20); + + expect(peakAndMean(segments, sum)).toEqual({ mean: 2, peak: 2 }); + }); +}); diff --git a/src/core/usage/timeline.ts b/src/core/usage/timeline.ts new file mode 100644 index 00000000..02cf5ca8 --- /dev/null +++ b/src/core/usage/timeline.ts @@ -0,0 +1,81 @@ +/** + * Step functions over time (ADR 0016 §3): each key (a worker, or the one queue) holds its latest + * value until it steps. The window's start is told by the step in force there, which is the last + * one at or before it; where there is none, the time before the first step is unknown, not zero. + */ +export interface Step { + readonly at: number; + readonly key: string; + readonly value: Value; +} + +export interface Segment { + readonly start: number; + readonly end: number; + /** The value each key holds throughout the segment; empty before any key has stepped. */ + readonly values: readonly Value[]; +} + +/** The segments of `[from, to]`, one for each stretch between steps. `steps` is in time order. */ +export function segmentsOf( + steps: readonly Step[], + from: number, + to: number, +): Segment[] { + const current = new Map(); + let index = 0; + for (; index < steps.length && (steps[index] as Step).at <= from; index += 1) { + const step = steps[index] as Step; + current.set(step.key, step.value); + } + const segments: Segment[] = []; + let start = from; + for (; index < steps.length; index += 1) { + const step = steps[index] as Step; + if (step.at > to) break; + segments.push({ end: step.at, start, values: [...current.values()] }); + current.set(step.key, step.value); + start = step.at; + } + segments.push({ end: to, start, values: [...current.values()] }); + return segments; +} + +/** The values in force at each of `times`, which ascend. A step at a time is already in force. */ +export function valuesAt( + steps: readonly Step[], + times: readonly number[], +): (readonly Value[])[] { + const current = new Map(); + let index = 0; + return times.map((time) => { + for (; index < steps.length && (steps[index] as Step).at <= time; index += 1) { + const step = steps[index] as Step; + current.set(step.key, step.value); + } + return [...current.values()]; + }); +} + +/** + * The peak and the time-weighted mean of what `read` takes from each segment, over the stretches + * where it is known. `undefined` when no stretch with time in it is known. + */ +export function peakAndMean( + segments: readonly Segment[], + read: (values: readonly Value[]) => number | undefined, +): { readonly peak: number; readonly mean: number } | undefined { + let peak: number | undefined; + let weighted = 0; + let known = 0; + for (const segment of segments) { + const duration = segment.end - segment.start; + if (duration <= 0) continue; + const value = read(segment.values); + if (value === undefined) continue; + peak = Math.max(peak ?? value, value); + weighted += value * duration; + known += duration; + } + return peak === undefined ? undefined : { mean: weighted / known, peak }; +} From eec7f49580cb09ad3839c4cd02cd0b3ac5d61d49 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 16:38:18 +0200 Subject: [PATCH 04/23] feat(usage): usage.get answers on a worker and a gateway, and simlock stats prints it 17 failing -> 0 failing (unit) --- src/cli/index.test.ts | 2 +- src/cli/index.ts | 68 +++++++++++++- src/cli/stats.ts | 163 ++++++++++++++++++++++++++++++++- src/core/usage/usage-reader.ts | 107 ++++++++++++++++++++++ src/daemon/dispatch.ts | 19 ++++ src/daemon/dispatcher.test.ts | 14 +-- src/daemon/dispatcher.ts | 13 ++- src/gateway/boundary.test.ts | 6 +- src/gateway/dispatcher.ts | 19 +++- src/simlock-client/client.ts | 4 +- 10 files changed, 394 insertions(+), 21 deletions(-) create mode 100644 src/core/usage/usage-reader.ts diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 18dfadbb..a2846aeb 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -5403,7 +5403,7 @@ describe("CLI: stats", () => { for (const argv of [["--since", "1h"], ["--from", "2026-10-05T10:00:00Z"], ["--json"]]) { const { output, run } = statsRun(argv, async (window) => usageAnswer(window)); await expect(run, argv.join(" ")).resolves.toBe(0); - expect(output.stderr, argv.join(" ")).toBe(""); + expect(output.stderr, argv.join(" ")).not.toContain("USAGE"); } for (const argv of [ ["--since", "1h", "--from", "2026-10-05T10:00:00Z"], diff --git a/src/cli/index.ts b/src/cli/index.ts index 6ffbf516..1f8327f2 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -55,6 +55,7 @@ import { type LeaseRenewal, } from "../lease-policy/index.js"; import { followLog, type Signals } from "./follow-log.js"; +import { formatUsage } from "./stats.js"; import { spawnPassthrough } from "./passthrough.js"; import { ERROR_TABLE } from "../contract/index.js"; import { parseDurationMs } from "../contract/duration.js"; @@ -65,7 +66,7 @@ const USAGE = `Usage: simlock [options] Commands: lease, release, status, list, catalog, cleanup, doctor, nuke, events, - daemon, config, token + stats, daemon, config, token worker Inspect and manage the workers of a gateway; on a single host, list shows the host itself @@ -618,6 +619,8 @@ export async function runCli( return await runNuke(rest.slice(1), environment, token); case "events": return await runEvents(rest.slice(1), environment, token); + case "stats": + return await runStats(rest.slice(1), environment, token); case "daemon": return await runDaemon(rest.slice(1), environment, token); case "config": @@ -1539,6 +1542,69 @@ async function runEvents( } } +const DAY_MS = 24 * 60 * 60 * 1000; + +/** `simlock stats`: the usage figures for a window, from the daemon's event history (ADR 0016). */ +async function runStats( + argv: readonly string[], + environment: CliEnvironment, + token: string | undefined, +): Promise { + const values = commandArgs(argv, { + from: { type: "string" }, + help: { type: "boolean", short: "h" }, + json: { type: "boolean" }, + since: { type: "string" }, + to: { type: "string" }, + }); + if (values.help) { + environment.stdout.write( + "Usage: simlock stats [--since | --from [--to ]] [--json]\n", + ); + return 0; + } + const window = statsWindow(values, environment.clock.now()); + const client = await connectDaemonClient(environment, token); + try { + const usage = await client.usage(window); + if (values.json) writeResult(environment, usage); + else environment.stdout.write(`${formatUsage(usage)}\n`); + return 0; + } finally { + await client.close(); + } +} + +/** + * The window `simlock stats` asks for: `--since` back from now, or `--from` to `--to` (now when + * `--to` is left out), the last 24 hours when none is given. Flags that disagree are usage errors. + */ +function statsWindow( + values: Readonly>, + now: number, +): { readonly from: number; readonly to: number } { + const { from, since, to } = values; + if (from !== undefined && since !== undefined) { + throw new UsageError("stats takes --since or --from, not both"); + } + if (to !== undefined && from === undefined) throw new UsageError("--to needs --from"); + if (typeof from !== "string") { + return { from: now - (typeof since === "string" ? parseDuration(since) : DAY_MS), to: now }; + } + const window = { from: parseTime(from), to: typeof to === "string" ? parseTime(to) : now }; + if (window.from >= window.to) { + throw new UsageError("--from must be earlier than --to, or than now when --to is left out"); + } + return window; +} + +/** Parses an ISO time only at the CLI boundary. */ +function parseTime(value: string): number { + const milliseconds = Date.parse(value); + if (Number.isNaN(milliseconds)) throw new UsageError(`Invalid time: ${value}`); + return milliseconds; +} + /** * Connects for `simlock events`. History alone (`--since` without `--follow`, passed as * `historySinceTs`) needs no daemon: with nothing listening it prints the event file's history diff --git a/src/cli/stats.ts b/src/cli/stats.ts index c9b87d3e..3f9d56df 100644 --- a/src/cli/stats.ts +++ b/src/cli/stats.ts @@ -1,6 +1,163 @@ -import type { UsageOutput } from "../contract/index.js"; +import type { UsageFigures, UsageOutput } from "../contract/index.js"; /** The human rendering of `usage.get`'s answer for `simlock stats`. */ -export function formatUsage(_usage: UsageOutput): string { - return ""; +export function formatUsage(usage: UsageOutput): string { + const lines = [`Usage from ${iso(usage.window.from)} to ${iso(usage.window.to)}`]; + if (usage.partial) { + lines.push( + `Figures cover from ${iso(usage.coversFrom)}: the history does not reach back to the start of the window.`, + ); + } + lines.push("", "Totals", ...totalRows(usage.totals)); + lines.push( + "", + "Platforms", + ...rows(Object.entries(usage.platforms), (figures) => summary(figures)), + ); + if (usage.workers.length > 0) { + lines.push( + "", + "Workers", + ...rows( + usage.workers.map((worker) => [ + worker.label === undefined ? worker.id : `${worker.label} (${worker.id})`, + worker, + ]), + (figures) => summary(figures), + ), + ); + } + if (usage.requesters.length > 0) { + lines.push( + "", + "Requesters", + ...rows( + usage.requesters.map((requester) => [ + requester.label === undefined ? requester.id : `${requester.label} (${requester.id})`, + requester, + ]), + (requester) => + `${plural(requester.requests, "request")}, ${requester.granted} granted, ` + + `${requester.rejected} rejected, held ${duration(requester.heldTotalMs)}`, + ), + ); + } + return lines.join("\n"); +} + +const iso = (ms: number): string => new Date(ms).toISOString(); + +const plural = (count: number, noun: string): string => `${count} ${noun}${count === 1 ? "" : "s"}`; + +/** One row for each `[name, item]`, the names set to a common width. */ +function rows( + entries: readonly (readonly [string, Item])[], + render: (item: Item) => string, +): string[] { + const width = Math.max(...entries.map(([name]) => name.length)); + return entries.map(([name, item]) => ` ${name.padEnd(width)} ${render(item)}`); +} + +/** A platform's or a worker's figures on one line. */ +function summary(figures: UsageFigures): string { + return ( + `${plural(figures.requests, "request")}, ${figures.granted} granted, ` + + `${figures.rejected.total} rejected, wait p50 ${duration(figures.wait.p50)}, ` + + `held p50 ${duration(figures.held.p50)}, slots peak ${figures.utilisation.slots.peak ?? "-"}` + ); +} + +const row = (label: string, value: string): string => ` ${label.padEnd(14)}${value}`; + +/** `name count` pairs, by name. */ +function list(counts: Readonly>): string { + return Object.entries(counts) + .sort(([left], [right]) => (left < right ? -1 : 1)) + .map(([name, count]) => `${name} ${count}`) + .join(", "); +} + +function totalRows(totals: UsageFigures): string[] { + return [ + ...requestRows(totals), + row("Wait:", samples(totals.wait)), + row("Held:", samples(totals.held)), + row("Turnaround:", samples(totals.turnaround)), + row("Provisioning:", samples(totals.provisioning)), + row("Boot:", samples(totals.boot)), + ...capacityRows(totals), + row( + "Incidents:", + `${totals.incidents.quarantined} quarantined, ${totals.incidents.crashRecovered} recovered after a crash, ` + + `${totals.incidents.quarantineRecovered} recovered from quarantine, ${totals.incidents.lost} lost`, + ), + ...(Object.keys(totals.errors.byCode).length === 0 + ? [] + : [row("Errors:", list(totals.errors.byCode))]), + ]; +} + +/** The requests, and what became of them, leaving out the rows with nothing in them. */ +function requestRows({ bySource, granted, rejected, requests }: UsageFigures): string[] { + return [ + row("Requests:", `${requests} (${granted} granted, ${rejected.total} rejected)`), + ...(granted === 0 + ? [] + : [ + row( + "Granted:", + `warm ${bySource.warm}, booted ${bySource.booted}, provisioned ${bySource.provisioned}`, + ), + ]), + ...(rejected.total === 0 ? [] : [row("Rejected:", list(rejected.byReason))]), + ]; +} + +function capacityRows({ queue, utilisation }: UsageFigures): string[] { + const { ram, slots } = utilisation; + return [ + row( + "Slots:", + slots.peak === null || slots.max === null || slots.mean === null + ? "not known" + : `peak ${figure(slots.peak)} of ${figure(slots.max)}, mean ${figure(slots.mean)}`, + ), + ...(ram === undefined + ? [] + : [ + row( + "RAM:", + `peak ${gibibytes(ram.peakBytes)} of ${gibibytes(ram.limitBytes)}, mean ${gibibytes(ram.meanBytes)}`, + ), + ]), + row( + "Queue:", + queue.peakDepth === null || queue.meanDepth === null + ? "not known" + : `peak depth ${figure(queue.peakDepth)}, mean ${figure(queue.meanDepth)}`, + ), + ]; +} + +function samples(figures: UsageFigures["wait"]): string { + if (figures.p50 === null || figures.p95 === null || figures.max === null) return "no samples"; + return ( + `p50 ${duration(figures.p50)}, p95 ${duration(figures.p95)}, max ${duration(figures.max)} ` + + `(${plural(figures.count, "sample")})` + ); +} + +/** A number to one decimal, without a trailing `.0`. */ +const figure = (value: number): string => String(Math.round(value * 10) / 10); + +const gibibytes = (bytes: number): string => `${(bytes / 1024 ** 3).toFixed(1)} GiB`; + +/** A time in the unit a person reads it in; `-` for a figure there is none of. */ +function duration(ms: number | null): string { + if (ms === null) return "-"; + if (ms === 0) return "0s"; + if (ms < 1_000) return `${ms}ms`; + if (ms < 60_000) return `${figure(ms / 1_000)}s`; + if (ms < 3_600_000) return `${Math.floor(ms / 60_000)}m ${Math.floor((ms % 60_000) / 1_000)}s`; + return `${Math.floor(ms / 3_600_000)}h ${Math.floor((ms % 3_600_000) / 60_000)}m`; } diff --git a/src/core/usage/usage-reader.ts b/src/core/usage/usage-reader.ts new file mode 100644 index 00000000..1ba1a8fa --- /dev/null +++ b/src/core/usage/usage-reader.ts @@ -0,0 +1,107 @@ +import type { EventEnvelope, EventName } from "../../bus/index.js"; +import type { UsageOutput } from "../../contract/index.js"; +import { computeUsage, seriesBucketMs, type UsageWindow } from "./compute-usage.js"; + +/** The two timelines whose step in force at a window's start is read with it (ADR 0016 §3). */ +const STEP_EVENTS: readonly EventName[] = ["capacity.changed", "queue.changed"]; + +/** The part of the event history `usage.get` reads (`EventHistory` satisfies it). */ +export interface UsageHistory { + read(input: { + readonly sinceTs: number; + readonly carry: readonly EventName[]; + }): Promise<{ readonly events: readonly EventEnvelope[]; readonly oldestTs: number | undefined }>; + /** The id of the event published last; the memo is good while it has not moved. */ + latestId(): string | undefined; +} + +export interface UsageReaderOptions { + readonly history: UsageHistory; + /** Token id to label, from the store the answering daemon owns (ADR 0016 §7). */ + readonly tokenLabels: () => Promise>; + /** The workers to report: a worker's own entry, a gateway's registry. Read on each answer. */ + readonly workers: () => readonly { readonly id: string; readonly label?: string | undefined }[]; + /** Set on a gateway: the figures are the fleet's (ADR 0016 §6), and this is the prefix it + * stamps on the requesters it forwards (`gw::`). */ + readonly fleet?: { readonly requesterPrefix: string }; +} + +/** What `get` answers: the figures, or how far back the history reaches when it is not far enough. */ +export type UsageResult = { readonly usage: UsageOutput } | { readonly oldestTs: number }; + +/** + * Answers `usage.get` for a worker or a gateway from the same code (ADR 0016): reads the history + * for the window, hands it to `computeUsage`, joins token labels. The window is rounded out to the + * series bucket, and the last answer is kept by that window and the newest event, so a caller that + * asks again within the bucket while nothing happened reads nothing. + */ +export class UsageReader { + #memo: { readonly key: string; readonly usage: UsageOutput } | undefined; + + constructor(private readonly options: UsageReaderOptions) {} + + async get(asked: UsageWindow): Promise { + const bucketMs = seriesBucketMs(asked.to - asked.from); + // Out to the bucket on both sides, so the window asked for is inside the one answered. + const window = { + from: Math.floor(asked.from / bucketMs) * bucketMs, + to: Math.ceil(asked.to / bucketMs) * bucketMs, + }; + const key = `${window.from}:${window.to}:${this.options.history.latestId() ?? ""}`; + if (this.#memo?.key === key) return { usage: this.#memo.usage }; + + const { events, oldestTs } = await this.options.history.read({ + carry: STEP_EVENTS, + sinceTs: window.from, + }); + if (oldestTs !== undefined && oldestTs > asked.to) return { oldestTs }; + const usage = computeUsage(events, window, { + fleet: this.options.fleet !== undefined, + labels: await this.#labelsFor(events), + oldestTs, + requesterPrefix: this.options.fleet?.requesterPrefix ?? "", + workers: this.options.workers(), + }); + this.#memo = { key, usage }; + return { usage }; + } + + /** The label of each requester the events name that the token store knows (ADR 0016 §7). */ + async #labelsFor(events: readonly EventEnvelope[]): Promise> { + const prefix = this.options.fleet?.requesterPrefix ?? ""; + const requesters = requestersIn(events); + if (requesters.size === 0) return {}; + const tokens = await this.options.tokenLabels(); + const labels: Record = {}; + for (const requester of requesters) { + const own = prefix !== "" && requester.startsWith(prefix); + const label = tokens.get(own ? requester.slice(prefix.length) : requester); + if (label !== undefined) labels[requester] = label; + } + return labels; + } +} + +/** Every requester id the events name, as a request, a rejection or a dispatch has it. */ +function requestersIn(events: readonly EventEnvelope[]): Set { + const requesters = new Set(); + for (const event of events) { + const payload = event.payload as { + readonly requester?: unknown; + readonly requesterId?: unknown; + }; + for (const requester of [payload.requester, payload.requesterId]) { + if (typeof requester === "string") requesters.add(requester); + } + } + return requesters; +} + +/** The labelled tokens of a store as the reader takes them: token id to label. */ +export function tokenLabelMap( + tokens: readonly { readonly id: string; readonly label?: string | undefined }[], +): Map { + return new Map( + tokens.flatMap((token) => (token.label === undefined ? [] : [[token.id, token.label]])), + ); +} diff --git a/src/daemon/dispatch.ts b/src/daemon/dispatch.ts index 07244276..4650353f 100644 --- a/src/daemon/dispatch.ts +++ b/src/daemon/dispatch.ts @@ -26,6 +26,7 @@ import { type OperationDefinition, type OperationName, type Role, + type UsageOutput, type leaseProgressSchema, } from "../contract/index.js"; @@ -390,3 +391,21 @@ function parseDispatchOutput( `Internal: ${operationName} produced a response that does not match its contract output schema`, ); } + +/** + * What a `usage.get` handler answers with: the figures, or `HISTORY_NOT_KEPT` carrying the oldest + * time the history reaches when the window ends before it (ADR 0016 §5). Both dispatchers read + * their answer through here, so the one message and the one `details` shape exist once. + */ +export function usageAnswer( + result: { readonly usage: UsageOutput } | { readonly oldestTs: number }, +): UsageOutput { + if ("usage" in result) return result.usage; + const oldest = new Date(result.oldestTs); + const when = Number.isNaN(oldest.getTime()) ? String(result.oldestTs) : oldest.toISOString(); + throw new DispatchError( + "HISTORY_NOT_KEPT", + `The event history does not reach back to the end of that window; its oldest event is from ${when}.`, + { oldestTs: result.oldestTs }, + ); +} diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 243501df..c8210e0f 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -1137,20 +1137,22 @@ describe("Dispatcher: usage.get", () => { const { dispatcher } = await buildDispatcher({ eventHistory: history }); const call = (window: { from: number; to: number }) => dispatcher.dispatch("usage.get", window, session({ role: "admin" })); + const minute = 60_000; + const asked = { from: WINDOW.from + 10_000, to: WINDOW.to + 10_000 }; - const first = await call(WINDOW); - const second = await call({ from: WINDOW.from + 30_000, to: WINDOW.to + 30_000 }); + const first = await call(asked); + const second = await call({ from: asked.from + 20_000, to: asked.to + 20_000 }); expect(state.reads).toBe(1); expect(second).toEqual(first); - // The answer's window is the rounded one, not the one asked for. - expect(second.window).toEqual(WINDOW); + // The answer's window is the window asked for rounded out to the bucket, not the one asked for. + expect(second.window).toEqual({ from: WINDOW.from, to: WINDOW.to + minute }); state.newest = "evt_2"; - await call(WINDOW); + await call(asked); expect(state.reads).toBe(2); - await call({ from: WINDOW.from + 60_000, to: WINDOW.to + 60_000 }); + await call({ from: asked.from + minute, to: asked.to + minute }); expect(state.reads).toBe(3); }); diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index 46f25880..0704a93a 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -54,7 +54,9 @@ import { runDispatch, type DispatchSession, type ErasedHandler, + usageAnswer, } from "./dispatch.js"; +import { tokenLabelMap, UsageReader } from "../core/usage/usage-reader.js"; export { DispatchError, type ContractDispatcher, type DispatchSession } from "./dispatch.js"; @@ -232,9 +234,16 @@ export class Dispatcher { #viewCatalog: { readonly readAt: number; readonly catalog: Promise } | undefined; /** Ends the bus subscriptions that drop `#viewCatalog`; see `dispose`. */ readonly #unsubscribe: (() => void)[] = []; + /** Answers `usage.get` from the event history, as this host's own entry (ADR 0016, ADR 0012). */ + readonly #usage: UsageReader; constructor(private readonly options: DispatcherOptions) { this.#logger = options.logger ?? new NoopLogger(); + this.#usage = new UsageReader({ + history: options.eventHistory, + tokenLabels: async () => tokenLabelMap((await options.tokens?.list()) ?? []), + workers: () => [{ id: options.instanceId, label: options.config.gateway.label }], + }); // Observers only (architecture rule 5): dropping a kept read decides nothing. for (const event of WORKER_VIEW_CATALOG_EVENTS) { const unsubscribe = options.eventBus?.subscribe(event, () => { @@ -630,9 +639,7 @@ export class Dispatcher { #eventsReplay: Handler<"events.replay"> = (input) => this.options.eventHistory.replay(input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }); - #usageGet: Handler<"usage.get"> = () => { - throw new DispatchError("INTERNAL", "usage.get is not implemented"); - }; + #usageGet: Handler<"usage.get"> = async (input) => usageAnswer(await this.#usage.get(input)); #eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { const subscriptionId = session.manageEventSubscription(true); diff --git a/src/gateway/boundary.test.ts b/src/gateway/boundary.test.ts index 8f15ecb7..f2bf9d1f 100644 --- a/src/gateway/boundary.test.ts +++ b/src/gateway/boundary.test.ts @@ -54,7 +54,10 @@ const ALLOWED_DAEMON_IMPORTS = ["daemon/dispatch.js"]; * (now-removed) blanket `core` entry above asked the PR that widened this list to do: they are * `wait-queue.js`'s own transitive *type* imports (`DeviceRecord`/`LeaseRecord`/`DeviceSpec` and * `DeviceRequest`), which the fleet queue and coordinator need to name the same shapes wait-queue - * itself is typed against rather than redeclaring them. Neither module is the registry, capacity, + * itself is typed against rather than redeclaring them. `usage/usage-reader.js` is `usage.get`'s one + * answerer (ADR 0016 §1, §6): the same reader serves a worker and a gateway, so the window rounding + * and the memo exist once. It imports the bus, the contract and the pure figures module, nothing of + * the registry, capacity or lifecycle engine. Neither module is the registry, capacity, * or lifecycle engine §33 keeps off limits -- `domain.js` is pure data shapes and a spec-equality * predicate, `driver.js` is interface declarations only. */ @@ -64,6 +67,7 @@ const ALLOWED_CORE_IMPORTS = [ "core/serialized-decision.js", "core/domain.js", "core/driver.js", + "core/usage/usage-reader.js", ]; describe("gateway module boundary", () => { diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 33408ff5..0793480c 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -46,9 +46,11 @@ import { runDispatch, type DispatchSession, type ErasedHandler, + usageAnswer, } from "../daemon/dispatch.js"; import type { Clock, Logger } from "../ports/index.js"; import { NoopLogger } from "../ports/index.js"; +import { tokenLabelMap, UsageReader } from "../core/usage/usage-reader.js"; import { aggregateCatalog, aggregateStatus, type AggregateStatusOptions } from "./aggregate.js"; import { relayComponentInstall } from "./component-relay.js"; import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; @@ -148,16 +150,27 @@ export interface GatewayDispatcherOptions { * (`FleetLeaseIndex#project`/`#all`) -- kept separate from `coordinator` because this * dispatcher only ever *reads* it, never mutates it. `isGatewayRequester` marks the requests * this gateway sent a worker, which `list.get`'s `requests` already lists as its own. */ - readonly leaseIndex: Pick; + readonly leaseIndex: Pick< + FleetLeaseIndex, + "project" | "all" | "isGatewayRequester" | "requesterPrefix" + >; } export class GatewayDispatcher { readonly #logger: Logger; readonly #dispatchLogger: Logger; readonly #handlers: Record, ErasedHandler>; + /** Answers `usage.get` as the fleet, from the gateway's own merged history (ADR 0016 §6). */ + readonly #usage: UsageReader; constructor(private readonly options: GatewayDispatcherOptions) { this.#logger = options.logger ?? new NoopLogger(); + this.#usage = new UsageReader({ + fleet: { requesterPrefix: options.leaseIndex.requesterPrefix }, + history: options.eventHistory, + tokenLabels: async () => tokenLabelMap((await options.tokens?.list()) ?? []), + workers: () => options.workers.views().map((view) => ({ id: view.id, label: view.label })), + }); this.#dispatchLogger = this.#logger.child("dispatch"); this.#handlers = { "catalog.get": this.#catalogGet, @@ -264,9 +277,7 @@ export class GatewayDispatcher { #eventsReplay: Handler<"events.replay"> = (input) => this.options.eventHistory.replay(input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }); - #usageGet: Handler<"usage.get"> = () => { - throw new DispatchError("INTERNAL", "usage.get is not implemented"); - }; + #usageGet: Handler<"usage.get"> = async (input) => usageAnswer(await this.#usage.get(input)); #eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { const subscriptionId = session.manageEventSubscription(true); diff --git a/src/simlock-client/client.ts b/src/simlock-client/client.ts index 13c639ce..9dc94a81 100644 --- a/src/simlock-client/client.ts +++ b/src/simlock-client/client.ts @@ -582,8 +582,8 @@ class SimlockClientImpl { return this.#call("events.replay", input); } - usage(_window: UsageGetInput): Promise { - return Promise.reject(new Error("usage is not implemented")); + usage(window: UsageGetInput): Promise { + return this.#call("usage.get", window); } async subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise> { From 4462bf7685917a6a67c78b4334d7656a9f8b4856 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 16:40:54 +0200 Subject: [PATCH 05/23] docs: simlock stats, the client's usage call, and the usage reader's tests --- docs/CLI.md | 117 ++++++++++++- docs/CLIENT.md | 28 +++ docs/internal/ARCHITECTURE.md | 15 ++ e2e/usage-stats.test.ts | 8 +- src/core/usage/usage-reader.test.ts | 259 ++++++++++++++++++++++++++++ 5 files changed, 424 insertions(+), 3 deletions(-) create mode 100644 src/core/usage/usage-reader.test.ts diff --git a/docs/CLI.md b/docs/CLI.md index 85c56711..e6980407 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -3,7 +3,7 @@ Part of the user manual: every command the simlock CLI is expected to implement. Results are JSON on **stdout**; progress/diagnostics are JSON lines on **stderr** — this is the default output, not an opt-in, because -agents are the primary audience. `status`, `catalog`, and +agents are the primary audience. `status`, `catalog`, `stats`, and `daemon ` are the exception: they default to a human-oriented view for interactive/operator use and accept `--json` to switch to the structured form. Every other command's output is already @@ -80,6 +80,7 @@ command starts it again) to bring the platform up. | 12 | `INSUFFICIENT_DISK_SPACE` | not enough free disk space to install a component | | 12 | `LICENSE_NOT_ACCEPTED` | a required license (e.g. an Android SDK license) is not accepted | | 12 | `UNKNOWN_WORKER` | `worker drain`/`undrain` or `component install --worker` naming a worker the gateway does not know | +| 12 | `HISTORY_NOT_KEPT` | `stats` for a window that ends before the oldest event the history holds | | 12 | `DOWNLOADS_DISABLED` | `component install` on a machine whose `downloads.policy` is `"never"` | | 12 | `COMPONENT_NOT_OWNED` | `component remove` of a component Simlock did not install, or one that changed on disk since | | 12 | `COMPONENT_IN_USE` | `component remove` of a component a device uses, Simlock's or your own | @@ -1802,6 +1803,120 @@ gateway's order. A replay (`simlock events`, `--since`) prints events by millisecond. `--follow` prints each live push as it arrives, so a relayed event from a worker whose clock is behind prints after a later gateway event. +## `simlock stats [--since | --from [--to ]] [--json]` + +The usage figures for a window: how many requests there were, how long they +waited and held their devices, how full the host was, and what went wrong. They +are worked out when you ask, from the same event history `simlock events` +prints, so they agree with it for the same window and they survive a daemon +restart. Nothing is counted separately and nothing is kept beyond the history, +so a window reaches back only as far as `eventLog.retention` and +`eventLog.maxBytes` keep events (see [CONFIGURATION.md](CONFIGURATION.md)). + +- `--since 6h` is the last six hours, up to now. Durations take `ms`, `s`, `m`, + `h` and `d` units, as for `simlock events`. +- `--from 2026-10-04T00:00:00Z --to 2026-10-05T00:00:00Z` names the window. With + `--from` alone the window runs to now. +- With none of them, the window is the last 24 hours. +- `--since` with `--from`, `--to` without `--from`, and a `--from` that is not + earlier than `--to` are usage errors (exit 2). A window can be at most 90 + days long. +- `--json` prints the daemon's answer unchanged, as one JSON object; without it + you get a table. + +The table opens with the window, then the totals, then a row for each platform, +each worker and each requester: + +``` +Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z + +Totals + Requests: 12 (10 granted, 2 rejected) + Granted: warm 6, booted 3, provisioned 1 + Rejected: no-wait 1, timeout 1 + Wait: p50 1.2s, p95 4s, max 9.1s (11 samples) + Held: p50 5m 0s, p95 30m 0s, max 1h 2m (9 samples) + Turnaround: p50 5m 10s, p95 31m 40s, max 1h 3m (9 samples) + Provisioning: p50 1m 30s, p95 1m 30s, max 1m 30s (1 sample) + Boot: p50 20s, p95 40s, max 40s (2 samples) + Slots: peak 3 of 4, mean 1.5 + Queue: peak depth 2, mean 0.4 + Incidents: 0 quarantined, 1 recovered after a crash, 0 recovered from quarantine, 0 lost + +Platforms + ios 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3 + android 0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak - + +Workers + mac-mini-1 (wrk_1) 12 requests, 10 granted, 2 rejected, wait p50 1.2s, held p50 5m 0s, slots peak 3 + +Requesters + ci-bot (tok_a) 3 requests, 2 granted, 1 rejected, held 12m 5s +``` + +What the figures count: + +- **Requests** are the lease requests made in the window. A request belongs to + the window it was made in: its grant, rejection and end are joined from + events up to the end of the window, so a lease still held at the end of the + window counts as a request and a grant and gives no held or turnaround time. + A request made before the window that is granted or rejected inside it is in + no count. +- **Granted** is the grants of those requests, by how the device came to be + ready: `warm` was already ready, `booted` was started from shutdown, + `provisioned` was created for the request. +- **Rejected** are the requests that ended without a device, by reason + (`timeout`, `no-wait`, `cancelled`, ...), plus the requests refused before + they were stored (a requester that already holds a lease). Those are in the + window their rejection falls in and are not requests, so granted plus + rejected can be more than requests. +- **Wait** is the time from the request to its grant or rejection. **Held** is + the time from the grant to the release or expiry of the lease. **Turnaround** + is the time from the request to that end. **Provisioning** and **Boot** are the + durations of creating and starting devices in the window. Each shows its median + (`p50`), `p95` and longest, and how many samples it has; `-` or "no samples" + where there are none. +- **Slots** is how many devices were running or reserved against the most the + host allows: the highest value and the average over time. **RAM** appears when + the host keeps a RAM budget. **Queue** is how many requests were waiting. Where + the history has nothing before the first record in the window, that stretch of + time is left out of the figures rather than counted as zero. +- **Incidents** count devices quarantined, devices recovered after a crash, + devices recovered from quarantine, and devices lost (a recovery that failed, or + a quarantined device given up on). **Errors** counts failures that carry no + reason of their own among the rejections: purges that failed and component + installs that failed. +- A requester that is a token shows the token's label beside its id. + +The window is widened to whole steps of the time series `--json` carries, which +is 1 minute for a window of a few hours and grows to 1 day for 90 days; the +window in the answer is the widened one. + +Against a **gateway** the totals are the fleet's and there is a row for each +worker, with the grants, hold times and device figures of that worker. Requests, +waits and rejections come from the gateway's own queue; the queue figure is the +fleet queue's. A worker's own `simlock stats` covers that worker only: its row +is itself, and a request that arrived through a gateway shows under the +requester id the gateway gave it. + +If the history does not reach back to the start of the window (a new daemon, or +older events deleted by retention), the figures cover what it does hold and say +so: + +``` +Usage from 2026-09-05T12:00:00.000Z to 2026-10-05T12:00:00.000Z +Figures cover from 2026-10-05T11:59:00.000Z: the history does not reach back to the start of the window. +``` + +In `--json` that is `partial: true` and `coversFrom`. If the window ends before +the oldest event the history holds, there is nothing to show: the command fails +with `HISTORY_NOT_KEPT` (exit 12), and the message names the time the history +reaches back to. A daemon with no history at all has nothing to refuse and +answers with zeros. + +`stats` needs the daemon, and the admin credential (see +[Admin credential resolution](#admin-credential-resolution)). + ## `simlock daemon ` Manage the daemon explicitly. Other commands auto-start it on demand; `daemon` diff --git a/docs/CLIENT.md b/docs/CLIENT.md index e4780710..2b4de1ed 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -16,6 +16,7 @@ import { connectSimlockAdmin } from "simlock/admin"; // agent + admin role `connectSimlockAdmin` returns a superset of `connectSimlock`'s client — every agent-role method plus the admin-role ones (`list`, `runCleanup`, `runNuke`, `getConfig`, `stopDaemon`, `replayEvents`/`subscribeEvents` (each event carries an `id`), +`usage`, `createToken`/`listTokens`/`revokeToken`, `installComponent`, `removeComponent`). The split exists so `simlock/client` doesn't even show admin methods in a caller's editor; the daemon's own role check is what actually stops an agent-role session from @@ -519,6 +520,33 @@ const { waiting = [] } = await client.getStatus(); `list({ kind: "requests" })` lists both, each worker's entries with their `workerId`, the same list as `GET /v1/lease-requests`. +## How much was used: `usage` + +`simlock/admin` only. `usage({ from, to })` returns the usage figures for a +window, the same answer `simlock stats --json` prints (see +[CLI.md](CLI.md), under `simlock stats`, +for what each figure counts). `from` and `to` are epoch milliseconds, `from` +before `to`, at most 90 days apart; the client refuses anything else before it +sends a frame. + +```ts +const usage = await admin.usage({ from: Date.now() - 6 * 3_600_000, to: Date.now() }); + +usage.totals.requests; // lease requests made in the window +usage.totals.wait.p95; // milliseconds, or null when nothing waited +usage.workers[0]?.label; // one entry for each worker; a worker lists itself +usage.series; // one point per bucket, for a chart +``` + +The daemon computes the figures from its event history, so they cover only what +the history holds: `partial` is `true` and `coversFrom` says where they start when +it does not reach the start of the window. `window` in the answer is the window +asked for, widened to a whole number of `bucketMs`, and the series never has more +than 200 points. A window that ends before the oldest event the history holds +rejects with `HISTORY_NOT_KEPT`; its `details.oldestTs` is the oldest time the +history reaches. Against a gateway the totals are the fleet's and `workers` has +one entry for each worker. + ## One connection, no reconnect, no retry This is the one thing to internalize before building anything on top of this diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index 9c0f8a2a..6e610226 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -1919,6 +1919,21 @@ The CLI reads the file itself only for `simlock events --since` when no daemon answers; `--follow` subscribes first, replays, and drops replayed pushes, so the join neither loses nor repeats an event. +`usage.get` (ADR 0016) is the third reader of that history, and the only one that +turns it into numbers. `computeUsage` (`src/core/usage/`) is pure: it takes +the envelopes of a window, with the latest `capacity.changed` and `queue.changed` +at or before the window's start that `EventHistory.read`'s `carry` adds, and +returns the figures. `readEvents` reads them into one fact per request, joined +by `requestId` and `leaseId`, and one per device event; `compute-usage.ts` adds +them up by platform, by worker and by requester. On a gateway (`fleet`) request +facts come from its own events and device facts from the events its workers +relayed, and a request is joined to its worker's grant or rejection through +`request.dispatched`; that rule is in `readEvents` and nowhere else. One +`UsageReader` serves both dispatchers: it rounds the window out to the series +bucket, keeps its last answer by that window and the newest event id, and joins +token labels, so the daemon's two handlers differ only in `fleet`. `simlock stats` +prints what the operation returns and computes nothing. + ## Device requests A request names a device in one of three forms (ADR 0015 §1): an exact diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index 1f7e5e03..15d49148 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -93,10 +93,14 @@ describe("simlock stats", () => { ).toBe(0); const usage = await stats(env); - const history = inWindow(await env.events(), usage.window); + const everything = await env.events(); + const history = inWindow(everything, usage.window); const named = (name: string) => history.filter((event) => event.event === name); - expect(usage.partial).toBe(false); + // The daemon is a minute old, so the history does not reach back an hour: the figures say + // where they start. + expect(usage.partial).toBe(true); + expect(usage.coversFrom).toBe(Math.min(...everything.map((event) => event.timestamp))); expect(usage.totals.requests).toBe(named("lease.requested").length); expect(usage.totals.granted).toBe(2); expect(usage.totals.bySource).toMatchObject( diff --git a/src/core/usage/usage-reader.test.ts b/src/core/usage/usage-reader.test.ts new file mode 100644 index 00000000..871aadf8 --- /dev/null +++ b/src/core/usage/usage-reader.test.ts @@ -0,0 +1,259 @@ +import { describe, expect, it } from "vitest"; + +import type { EventEnvelope, EventName } from "../../bus/index.js"; +import { tokenLabelMap, UsageReader, type UsageHistory } from "./usage-reader.js"; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const T0 = 10 * HOUR; + +function requested( + timestamp: number, + requester: string, + requestId = `req_${timestamp}`, +): EventEnvelope { + return { + event: "lease.requested", + id: `evt_${requestId}`, + module: "test", + payload: { requestId, requestSpec: { platform: "ios" }, requester, waitPolicy: "wait" }, + seq: timestamp, + timestamp, + } as unknown as EventEnvelope; +} + +/** A history that records what it was asked for and answers what the test sets. */ +function history(events: readonly EventEnvelope[] = [], oldestTs: number | undefined = undefined) { + const state = { + asked: [] as { sinceTs: number; carry: readonly EventName[] }[], + events, + newest: "evt_1" as string | undefined, + oldestTs, + }; + const source: UsageHistory = { + latestId: () => state.newest, + read: async (input) => { + state.asked.push(input); + return { events: state.events, oldestTs: state.oldestTs }; + }, + }; + return { source, state }; +} + +function reader( + source: UsageHistory, + extra: Partial[0]> = {}, +) { + const calls = { labels: 0 }; + const usage = new UsageReader({ + history: source, + tokenLabels: async () => { + calls.labels += 1; + return tokenLabelMap([{ id: "tok_a", label: "ci-bot" }, { id: "tok_plain" }]); + }, + workers: () => [{ id: "self" }], + ...extra, + }); + return { calls, usage }; +} + +const answer = (result: Awaited>) => { + if (!("usage" in result)) throw new Error("expected an answer"); + return result.usage; +}; + +describe("UsageReader", () => { + it("widens the window out to the bucket on both sides and reads the history from its start, with the two step events carried", async () => { + const { source, state } = history(); + const { usage } = reader(source); + + const result = answer(await usage.get({ from: T0 + 10_000, to: T0 + HOUR + 10_000 })); + + expect(result.window).toEqual({ from: T0, to: T0 + HOUR + MINUTE }); + expect(result.bucketMs).toBe(MINUTE); + expect(state.asked).toEqual([{ carry: ["capacity.changed", "queue.changed"], sinceTs: T0 }]); + }); + + it("leaves a window that is already on bucket edges as it is", async () => { + const { source } = history(); + const { usage } = reader(source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result.window).toEqual({ from: T0, to: T0 + HOUR }); + }); + + it("refuses a window that ends before the oldest held event, with that time, and answers one that ends exactly at it", async () => { + const { source } = history([], T0 + HOUR + 1); + const { usage } = reader(source); + const edge = reader(history([], T0 + HOUR).source).usage; + + await expect(usage.get({ from: T0, to: T0 + HOUR })).resolves.toEqual({ + oldestTs: T0 + HOUR + 1, + }); + answer(await edge.get({ from: T0, to: T0 + HOUR })); + }); + + it("answers an empty history for any window, as covering all of it", async () => { + const { usage } = reader(history().source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result).toMatchObject({ coversFrom: T0, partial: false }); + }); + + it("says the figures are partial when the history it read starts inside the window", async () => { + const { usage } = reader(history([requested(T0 + 5 * MINUTE, "a")], T0 + 5 * MINUTE).source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result).toMatchObject({ coversFrom: T0 + 5 * MINUTE, partial: true }); + expect(result.totals.requests).toBe(1); + }); + + it("keeps its answer while the window and the newest event are the same, and reads again when either changes", async () => { + const { source, state } = history(); + const { usage } = reader(source); + const ask = (shift: number) => usage.get({ from: T0 + shift, to: T0 + HOUR + shift }); + + const first = await ask(1_000); + const again = await ask(2_000); + expect(state.asked).toHaveLength(1); + expect(answer(again)).toBe(answer(first)); + + state.newest = undefined; + await ask(1_000); + expect(state.asked).toHaveLength(2); + state.newest = "evt_9"; + await ask(1_000); + expect(state.asked).toHaveLength(3); + await ask(1_000 + MINUTE); + expect(state.asked).toHaveLength(4); + // Each end of the window is part of the key. + await usage.get({ from: T0 + 1_000 + MINUTE, to: T0 + HOUR + 1_000 + 2 * MINUTE }); + expect(state.asked).toHaveLength(5); + }); + + it("does not keep a refusal: a history that has grown to cover the window answers it", async () => { + const { source, state } = history([], T0 + 2 * HOUR); + const { usage } = reader(source); + + await expect(usage.get({ from: T0, to: T0 + HOUR })).resolves.toEqual({ + oldestTs: T0 + 2 * HOUR, + }); + state.oldestTs = T0 - HOUR; + answer(await usage.get({ from: T0, to: T0 + HOUR })); + expect(state.asked).toHaveLength(2); + }); + + it("puts the token label beside each requester the token store knows, and no label beside one it does not", async () => { + const { source } = history([ + requested(T0 + 1_000, "tok_a"), + requested(T0 + 2_000, "tok_plain"), + requested(T0 + 3_000, "stranger"), + ]); + const { usage } = reader(source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(Object.fromEntries(result.requesters.map((entry) => [entry.id, entry.label]))).toEqual({ + stranger: undefined, + tok_a: "ci-bot", + tok_plain: undefined, + }); + }); + + it("reads no labels when the events name no requester", async () => { + const { source } = history([ + { + event: "daemon.stopping", + id: "evt_x", + module: "d", + payload: { reason: "x" }, + seq: 1, + timestamp: T0 + 1, + } as EventEnvelope, + ]); + const { calls, usage } = reader(source); + + answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(calls.labels).toBe(0); + }); + + it("strips its own prefix before the label lookup on a gateway, and leaves another prefix and a worker's own ids alone", async () => { + const events = [ + requested(T0 + 1_000, "gw:g1:tok_a"), + requested(T0 + 2_000, "gw:g2:tok_a"), + requested(T0 + 3_000, "tok_a"), + ]; + const gateway = reader(history(events).source, { fleet: { requesterPrefix: "gw:g1:" } }); + const worker = reader(history(events).source); + + const onGateway = answer(await gateway.usage.get({ from: T0, to: T0 + HOUR })); + const onWorker = answer(await worker.usage.get({ from: T0, to: T0 + HOUR })); + + const labels = (usage: typeof onGateway) => + Object.fromEntries(usage.requesters.map((entry) => [entry.id, entry.label])); + expect(labels(onGateway)).toEqual({ + "gw:g1:tok_a": "ci-bot", + "gw:g2:tok_a": undefined, + tok_a: "ci-bot", + }); + expect(labels(onWorker)).toEqual({ + "gw:g1:tok_a": undefined, + "gw:g2:tok_a": undefined, + tok_a: "ci-bot", + }); + }); + + it("counts as the fleet when it is given a prefix: a worker's relayed request is not a request", async () => { + const relayed = { + ...requested(T0 + 1_000, "gw:g1:tok_a"), + payload: { + requestId: "w_1", + requestSpec: { platform: "ios" }, + requester: "gw:g1:tok_a", + waitPolicy: "no-wait", + workerId: "wrk_1", + }, + } as unknown as EventEnvelope; + const events = [requested(T0 + 500, "tok_a", "own_1"), relayed]; + const gateway = reader(history(events).source, { fleet: { requesterPrefix: "gw:g1:" } }); + const worker = reader(history(events).source); + + const onGateway = answer(await gateway.usage.get({ from: T0, to: T0 + HOUR })); + const onWorker = answer(await worker.usage.get({ from: T0, to: T0 + HOUR })); + + expect(onGateway.totals.requests).toBe(1); + expect(onWorker.totals.requests).toBe(2); + }); + + it("reports the workers it is given, with their labels, read afresh for each answer", async () => { + const { source, state } = history(); + let workers = [{ id: "wrk_1", label: "mac" }]; + const { usage } = reader(source, { workers: () => workers }); + + const first = answer(await usage.get({ from: T0, to: T0 + HOUR })); + workers = [ + { id: "wrk_1", label: "mac" }, + { id: "wrk_2", label: "mini" }, + ]; + state.newest = "evt_2"; + const second = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(first.workers.map((worker) => [worker.id, worker.label])).toEqual([["wrk_1", "mac"]]); + expect(second.workers.map((worker) => worker.id)).toEqual(["wrk_1", "wrk_2"]); + }); +}); + +describe("tokenLabelMap", () => { + it("maps each labelled token's id to its label and leaves out one with none", () => { + expect(tokenLabelMap([{ id: "a", label: "x" }, { id: "b" }, { id: "c", label: "z" }])).toEqual( + new Map([ + ["a", "x"], + ["c", "z"], + ]), + ); + }); +}); From 6b9274145a1c59df8ce1bf572d4a4cb0e24e3a78 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 16:50:08 +0200 Subject: [PATCH 06/23] test: type the usage figures the e2e reads --- e2e/usage-stats.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index 15d49148..7e19789f 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -33,6 +33,7 @@ interface Figures { interface Usage { readonly window: { readonly from: number; readonly to: number }; readonly partial: boolean; + readonly coversFrom: number; readonly totals: Figures; readonly workers: readonly (Figures & { readonly id: string; readonly label?: string })[]; readonly requesters: readonly { readonly id: string; readonly requests: number }[]; From d37aef5f0a301c2e50df982017a719fa396ee42a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 18:20:26 +0200 Subject: [PATCH 07/23] test: kill the usage mutants that survived Adds tests for the usage error messages, the output schema's RAM and series fields, the no-token-store paths, the server route, the degraded client and the carry order; a step at the end of a window cuts no segment. --- src/bus/event-file.test.ts | 26 +++++++++++ src/cli/index.test.ts | 20 ++++++++ src/contract/operations.test.ts | 68 ++++++++++++++++++++++++++++ src/core/usage/compute-usage.test.ts | 19 ++++++++ src/core/usage/timeline.test.ts | 8 ++++ src/core/usage/timeline.ts | 2 +- src/core/usage/usage-reader.ts | 6 +-- src/daemon/dispatcher.test.ts | 50 +++++++++++++++++++- src/daemon/server.test.ts | 13 ++++++ src/gateway/dispatcher.test.ts | 23 +++++++++- src/simlock-client/client.test.ts | 3 ++ 11 files changed, 232 insertions(+), 6 deletions(-) diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index a38b3ee4..f0b8a218 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -490,6 +490,32 @@ describe("readEventFile", () => { expect(read.map((entry) => entry.seq)).toEqual([1, 3]); }); + it("readEventFile with carry keeps the later step when a line written afterwards is older, and the line written last on a tie", async () => { + const step = (id: string, seq: number, timestamp: number): EventEnvelope => + ({ + event: "capacity.changed", + id, + module: "test", + payload: {}, + seq, + timestamp, + }) as unknown as EventEnvelope; + const read = async (...written: EventEnvelope[]) => + ( + await readEventFile( + await filesystemWith({ "/data/events.jsonl": lines(...written) }), + "/data/events.jsonl", + { + carry: ["capacity.changed"], + sinceTs: 300, + }, + ) + ).map((entry) => entry.id); + + expect(await read(step("evt_2", 2, 200), step("evt_3", 3, 100))).toEqual(["evt_2"]); + expect(await read(step("evt_5", 5, 200), step("evt_6", 5, 200))).toEqual(["evt_6"]); + }); + it("readEventFile with carry returns the latest capacity.changed and queue.changed at or before sinceTs, one per worker id, and none when there is none", async () => { const step = ( seq: number, diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index a2846aeb..6170937c 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -5423,6 +5423,26 @@ describe("CLI: stats", () => { } }); + it("simlock stats names the flag combination or the time it refused in the usage error", async () => { + for (const [argv, message] of [ + [ + ["--since", "1h", "--from", "2026-10-05T10:00:00Z"], + "stats takes --since or --from, not both", + ], + [["--to", "2026-10-04T10:00:00Z"], "--to needs --from"], + [ + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-03T10:00:00Z"], + "--from must be earlier than --to, or than now when --to is left out", + ], + [["--from", "last tuesday"], "Invalid time: last tuesday"], + ] as const) { + const { output, run } = statsRun(argv, async (window) => usageAnswer(window)); + + await expect(run, argv.join(" ")).resolves.toBe(2); + expect(output.stderr, argv.join(" ")).toContain(`"message":"${message}"`); + } + }); + it("simlock stats --json prints the operation's output unchanged", async () => { const answer = usageAnswer( { from: NOW - HOUR, to: NOW }, diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 74f8d452..1b8e767d 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -935,3 +935,71 @@ describe("usage.get input", () => { expect(parse({ from: 0 }).success).toBe(false); }); }); + +describe("usage.get contract details", () => { + const issues = (input: unknown) => { + const result = OPERATIONS["usage.get"].input.safeParse(input); + return result.success ? [] : result.error.issues.map((issue) => issue.message); + }; + + it("usage.get names the bound it refused: from before to, and no more than ninety days", () => { + expect(issues({ from: 6, to: 5 })).toEqual(["from must be before to"]); + expect(issues({ from: 0, to: 91 * 24 * 60 * 60 * 1000 })).toEqual([ + "the window must not be longer than 90 days", + ]); + }); + + it("every operation carries the name it is registered under", () => { + for (const [key, operation] of Object.entries(OPERATIONS)) expect(operation.name).toBe(key); + }); + + it("usage.get output keeps the RAM figures of a scope and the RAM use of a series point", () => { + const samples = { count: 0, max: null, p50: null, p95: null }; + const figures = { + boot: samples, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + errors: { byCode: {} }, + granted: 0, + held: samples, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: samples, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: samples, + utilisation: { + ram: { limitBytes: 100, meanBytes: 40, peakBytes: 60 }, + slots: { max: 2, mean: 1, peak: 2 }, + }, + wait: samples, + }; + const point = { + at: 60_000, + queueDepth: 0, + ramUsedBytes: 60, + slotsMax: 2, + slotsUsed: 1, + waiting: 0, + }; + const output = { + bucketMs: 60_000, + coversFrom: 0, + partial: false, + platforms: { android: figures, ios: figures }, + requesters: [], + series: [point], + totals: figures, + window: { from: 0, to: 60_000 }, + workers: [], + }; + + const parsed = OPERATIONS["usage.get"].output.parse(output); + + expect(parsed.totals.utilisation.ram).toEqual({ + limitBytes: 100, + meanBytes: 40, + peakBytes: 60, + }); + expect(parsed.series).toEqual([point]); + }); +}); diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index c141caf3..10002fc3 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -361,6 +361,25 @@ describe("computeUsage", () => { expect(usage.totals.wait.count).toBe(0); }); + it("counts a killed rejection that names no request spec, with no preceding lease.requested, under rejected", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "lease.rejected", { + reason: "killed", + requestId: "r-killed", + requester: "agent-a", + }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.requests).toBe(0); + expect(usage.totals.rejected).toEqual({ byReason: { killed: 1 }, total: 1 }); + expect(usage.platforms.ios.rejected.total).toBe(0); + expect(usage.platforms.android.rejected.total).toBe(0); + }); + it("counts a request from before from that is rejected inside the window under neither requests nor rejections", () => { const usage = computeUsage( [ diff --git a/src/core/usage/timeline.test.ts b/src/core/usage/timeline.test.ts index 7c7123dd..3dd621d1 100644 --- a/src/core/usage/timeline.test.ts +++ b/src/core/usage/timeline.test.ts @@ -42,6 +42,14 @@ describe("segmentsOf", () => { }); }); +describe("segmentsOf at the end of the window", () => { + it("cuts no segment for a step at the very end of the window, which has no time in the window", () => { + const segments = segmentsOf([step(5, "a", 1), step(50, "a", 9)], 10, 50); + + expect(segments).toEqual([{ end: 50, start: 10, values: [1] }]); + }); +}); + describe("valuesAt", () => { it("answers the values in force at each time, counting a step at that time", () => { const steps = [step(5, "a", 1), step(20, "a", 3), step(20, "b", 7)]; diff --git a/src/core/usage/timeline.ts b/src/core/usage/timeline.ts index 02cf5ca8..b635bb89 100644 --- a/src/core/usage/timeline.ts +++ b/src/core/usage/timeline.ts @@ -32,7 +32,7 @@ export function segmentsOf( let start = from; for (; index < steps.length; index += 1) { const step = steps[index] as Step; - if (step.at > to) break; + if (step.at >= to) break; segments.push({ end: step.at, start, values: [...current.values()] }); current.set(step.key, step.value); start = step.at; diff --git a/src/core/usage/usage-reader.ts b/src/core/usage/usage-reader.ts index 1ba1a8fa..aef1822d 100644 --- a/src/core/usage/usage-reader.ts +++ b/src/core/usage/usage-reader.ts @@ -47,7 +47,7 @@ export class UsageReader { from: Math.floor(asked.from / bucketMs) * bucketMs, to: Math.ceil(asked.to / bucketMs) * bucketMs, }; - const key = `${window.from}:${window.to}:${this.options.history.latestId() ?? ""}`; + const key = `${window.from}:${window.to}:${String(this.options.history.latestId())}`; if (this.#memo?.key === key) return { usage: this.#memo.usage }; const { events, oldestTs } = await this.options.history.read({ @@ -68,13 +68,13 @@ export class UsageReader { /** The label of each requester the events name that the token store knows (ADR 0016 §7). */ async #labelsFor(events: readonly EventEnvelope[]): Promise> { - const prefix = this.options.fleet?.requesterPrefix ?? ""; + const prefix = this.options.fleet?.requesterPrefix; const requesters = requestersIn(events); if (requesters.size === 0) return {}; const tokens = await this.options.tokenLabels(); const labels: Record = {}; for (const requester of requesters) { - const own = prefix !== "" && requester.startsWith(prefix); + const own = prefix !== undefined && requester.startsWith(prefix); const label = tokens.get(own ? requester.slice(prefix.length) : requester); if (label !== undefined) labels[requester] = label; } diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index c8210e0f..9a7f64f4 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -200,6 +200,8 @@ async function buildDispatcher( ComponentInstaller, "claimProvision" | "inProgress" | "install" | "list" | "remove" >; + /** Leaves the token store out, as a daemon started without one is. */ + readonly withoutTokens?: boolean; /** `gateway.label` in this daemon's config; unset by default. */ readonly gatewayLabel?: string; /** The `http` block; disabled by default. */ @@ -306,7 +308,7 @@ async function buildDispatcher( reaper, registry, stalls: stallOptions(engine, driver, overrides), - tokens, + ...tokenOption(overrides.withoutTokens, tokens), version: "1.2.3", }); return { @@ -322,6 +324,11 @@ async function buildDispatcher( }; } +/** The `tokens` option, left out for a daemon started without a store. */ +function tokenOption(without: boolean | undefined, tokens: TokenStore): { tokens?: TokenStore } { + return without === true ? {} : { tokens }; +} + function session(overrides: Partial = {}): DispatchSession { return { manageEventSubscription: () => undefined, @@ -1195,6 +1202,47 @@ describe("Dispatcher: usage.get", () => { expect(usage.totals.requests).toBe(1); }); + it("the worker handler answers with no label for a requester when the daemon has no token store", async () => { + const { dispatcher, eventBus } = await buildDispatcher({ withoutTokens: true }); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: "tok_x", + waitPolicy: "wait", + }, + "test", + ); + + const usage = await dispatcher.dispatch( + "usage.get", + { from: 0, to: 600_000 }, + session({ role: "admin" }), + ); + + expect(usage.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: "tok_x", rejected: 0, requests: 1 }, + ]); + }); + + it("the worker handler's HISTORY_NOT_KEPT message names when the history begins", async () => { + const oldestTs = Date.parse("2026-10-05T10:00:00.000Z"); + const { history } = countingHistory(oldestTs); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch( + "usage.get", + { from: oldestTs - 2 * HOUR, to: oldestTs - HOUR }, + session({ role: "admin" }), + ), + ).rejects.toMatchObject({ + message: + "The event history does not reach back to the end of that window; its oldest event is from 2026-10-05T10:00:00.000Z.", + }); + }); + it("usage.get is an admin operation an agent token cannot call", async () => { const { history, state } = countingHistory(); const { dispatcher } = await buildDispatcher({ eventHistory: history }); diff --git a/src/daemon/server.test.ts b/src/daemon/server.test.ts index bfc04aaf..86d74080 100644 --- a/src/daemon/server.test.ts +++ b/src/daemon/server.test.ts @@ -416,6 +416,19 @@ describe("DaemonServer", () => { expect(harness.registry.snapshot.leases).toEqual([]); }); + it("routes usage.get to the dispatcher with the window the frame carries", async () => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + const answer = await client.request("usage.get", { from: 0, to: 3_600_000 }); + + expect(answer).toMatchObject({ + ok: true, + payload: { totals: { requests: 0 }, window: { from: 0, to: 3_600_000 } }, + }); + }); + it("serves the device catalog, omitting platforms with no registered driver", async () => { const harness = await createHarness(); const client = await createClient(harness.socketPath); diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index b012381b..a6aaf385 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -131,6 +131,8 @@ function harness( readonly eventHistory?: Pick; /** The tokens the gateway's store lists besides its worker token; none by default. */ readonly tokenRecords?: readonly { id: string; label?: string }[]; + /** Leaves the token store out, as a gateway started without one is. */ + readonly withoutTokens?: boolean; /** The gateway's `http` block; enabled on 127.0.0.1:4700 by default. */ readonly http?: (typeof gatewayConfig)["http"]; } = {}, @@ -194,7 +196,7 @@ function harness( // `classifyError`'s answer for the errors this dispatcher throws itself. errorCode: (error) => (error instanceof DispatchError ? error.code : undefined), logger: new JsonLinesLogger({ clock, module: "gateway", sink: logSink }), - tokens, + ...(options.withoutTokens === true ? {} : { tokens }), workers, }); return { @@ -608,6 +610,25 @@ describe("GatewayDispatcher", () => { }); }); + it("the gateway handler answers with no label for a requester when it has no token store", async () => { + const { dispatcher, eventBus } = harness({ withoutTokens: true }); + eventBus.emit( + "lease.requested", + { + requestId: "req_1", + requestSpec: { platform: "ios" }, + requester: "tok_x", + waitPolicy: "wait", + }, + "gateway", + ); + + const usage = await dispatcher.dispatch("usage.get", { from: 0, to: 600_000 }, session()); + + expect(usage.requesters.map((requester) => requester.label)).toEqual([undefined]); + expect(usage.requesters).toHaveLength(1); + }); + it("answers usage.get with fleet totals and one entry per worker, labelled from the registry", async () => { const { dispatcher, eventBus, workers } = harness(); workers.connected("wrk_1", "mac-mini-1", "0.3.0"); diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index 411d366d..62dbc4f5 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -69,6 +69,9 @@ describe("connectSimlock: handshake", () => { await expect(client.runCleanup()).rejects.toMatchObject({ code: "PROTOCOL_VERSION_UNSUPPORTED", }); + await expect(client.usage({ from: 0, to: 60_000 })).rejects.toMatchObject({ + code: "PROTOCOL_VERSION_UNSUPPORTED", + }); expect(connection.sent).toHaveLength(before); const stopPromise = client.stopDaemon(); From 500285e93b840d9867fadc9ec3d49753f08cc3e5 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 18:55:59 +0200 Subject: [PATCH 08/23] refactor: type the relayed answers so the usage reader has fewer dead branches --- src/bus/event-file.test.ts | 15 + src/cli/stats.test.ts | 184 ++++++ src/core/usage/compute-usage.test.ts | 235 +++++++ src/core/usage/compute-usage.ts | 21 +- src/core/usage/read-events.test.ts | 943 +++++++++++++++++++++++++++ src/core/usage/read-events.ts | 80 +-- 6 files changed, 1427 insertions(+), 51 deletions(-) create mode 100644 src/core/usage/read-events.test.ts diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index f0b8a218..fcd615ca 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -235,6 +235,21 @@ describe("EventHistory", () => { ]); }); + it("reads the ring with an event at exactly sinceTs as the step in force and not as a newer event", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + clock.advance(1_000); + const step = bus.emit("queue.changed", { depth: 4 }, "wait-queue"); + bus.emit("daemon.stopping", { reason: "at the edge" }, "daemon"); + clock.advance(1); + const after = bus.emit("daemon.stopping", { reason: "after" }, "daemon"); + + const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_000 }); + + expect(read.events).toEqual([step, after]); + }); + it("names the newest event's id, and none before the first event", () => { const bus = new EventBus(new FakeClock(1_000)); const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); diff --git a/src/cli/stats.test.ts b/src/cli/stats.test.ts index b0fc0c1a..7e0fae4a 100644 --- a/src/cli/stats.test.ts +++ b/src/cli/stats.test.ts @@ -153,3 +153,187 @@ describe("formatUsage", () => { expect(lines).toContain(" tok_b 1 request, 0 granted, 0 rejected, held 0s"); }); }); + +const PLATFORM_ROW = "0 requests, 0 granted, 0 rejected, wait p50 -, held p50 -, slots peak -"; +const TOTAL_ROWS = [ + "Totals", + " Requests: 0 (0 granted, 0 rejected)", + " Wait: no samples", + " Held: no samples", + " Turnaround: no samples", + " Provisioning: no samples", + " Boot: no samples", + " Slots: not known", + " Queue: not known", + " Incidents: 0 quarantined, 0 recovered after a crash, 0 recovered from quarantine, 0 lost", +]; + +describe("formatUsage layout", () => { + it("separates the sections with one blank line each, in the order of the header, totals, platforms, workers and requesters", () => { + const text = formatUsage( + usageFixture({ + coversFrom: Date.parse("2026-10-05T09:59:00.000Z"), + partial: true, + requesters: [{ granted: 0, heldTotalMs: 0, id: "tok_b", rejected: 0, requests: 1 }], + workers: [{ ...figuresFixture(), id: "wrk_2" }], + }), + ); + + expect(text).toBe( + [ + "Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z", + "Figures cover from 2026-10-05T09:59:00.000Z: the history does not reach back to the start of the window.", + "", + ...TOTAL_ROWS, + "", + "Platforms", + ` android ${PLATFORM_ROW}`, + ` ios ${PLATFORM_ROW}`, + "", + "Workers", + ` wrk_2 ${PLATFORM_ROW}`, + "", + "Requesters", + " tok_b 1 request, 0 granted, 0 rejected, held 0s", + ].join("\n"), + ); + }); + + it("prints a whole report without the optional sections when there are no workers or requesters", () => { + expect(formatUsage(usageFixture())).toBe( + [ + "Usage from 2026-10-04T10:00:00.000Z to 2026-10-05T10:00:00.000Z", + "", + ...TOTAL_ROWS, + "", + "Platforms", + ` android ${PLATFORM_ROW}`, + ` ios ${PLATFORM_ROW}`, + ].join("\n"), + ); + }); +}); + +describe("formatUsage rows that depend on the figures", () => { + it("lists the error codes and the rejection reasons by name, whatever order they arrived in", () => { + const lines = formatUsage( + usageFixture({ + totals: figuresFixture({ + errors: { byCode: { "c.zed": 1, "a.alpha": 2, "b.mid": 3 } }, + rejected: { byReason: { timeout: 1, "no-wait": 2, capacity: 3 }, total: 6 }, + }), + }), + ).split("\n"); + + expect(lines).toContain(" Errors: a.alpha 2, b.mid 3, c.zed 1"); + expect(lines).toContain(" Rejected: capacity 3, no-wait 2, timeout 1"); + }); + + it("prints the Errors row only when there are error codes", () => { + const without = formatUsage(usageFixture()); + const withOne = formatUsage( + usageFixture({ totals: figuresFixture({ errors: { byCode: { "x.y": 1 } } }) }), + ); + + expect(without.split("\n").some((line) => line.startsWith(" Errors:"))).toBe(false); + expect(withOne.split("\n")).toContain(" Errors: x.y 1"); + }); + + it("prints the Granted row only when something was granted", () => { + const withGrant = formatUsage( + usageFixture({ + totals: figuresFixture({ bySource: { booted: 0, provisioned: 0, warm: 1 }, granted: 1 }), + }), + ).split("\n"); + const without = formatUsage(usageFixture()).split("\n"); + + expect(withGrant).toContain(" Granted: warm 1, booted 0, provisioned 0"); + expect(without.some((line) => line.startsWith(" Granted:"))).toBe(false); + }); + + it("prints the Rejected row only when something was rejected", () => { + const withReject = formatUsage( + usageFixture({ + totals: figuresFixture({ rejected: { byReason: { timeout: 1 }, total: 1 } }), + }), + ).split("\n"); + const without = formatUsage(usageFixture()).split("\n"); + + expect(withReject).toContain(" Rejected: timeout 1"); + expect(without.some((line) => line.startsWith(" Rejected:"))).toBe(false); + }); + + it("prints the RAM row only when the figures carry RAM", () => { + const withRam = formatUsage( + usageFixture({ + totals: figuresFixture({ + utilisation: { + ram: { limitBytes: 8 * 1024 ** 3, meanBytes: 0, peakBytes: 1024 ** 3 }, + slots: { max: null, mean: null, peak: null }, + }, + }), + }), + ).split("\n"); + const without = formatUsage(usageFixture()).split("\n"); + + expect(withRam).toContain(" RAM: peak 1.0 GiB of 8.0 GiB, mean 0.0 GiB"); + expect(without.some((line) => line.startsWith(" RAM:"))).toBe(false); + }); + + it.each([ + ["peak", { max: 4, mean: 1.5, peak: null }], + ["max", { max: null, mean: 1.5, peak: 3 }], + ["mean", { max: 4, mean: null, peak: 3 }], + ])("says slots are not known when only the slot %s is missing", (_name, slots) => { + const lines = formatUsage( + usageFixture({ totals: figuresFixture({ utilisation: { slots } }) }), + ).split("\n"); + + expect(lines).toContain(" Slots: not known"); + }); + + it.each([ + ["peak depth", { meanDepth: 0.4, peakDepth: null }], + ["mean depth", { meanDepth: null, peakDepth: 2 }], + ])("says the queue is not known when only the queue %s is missing", (_name, queue) => { + const lines = formatUsage(usageFixture({ totals: figuresFixture({ queue }) })).split("\n"); + + expect(lines).toContain(" Queue: not known"); + }); + + it.each([ + ["p50", { count: 1, max: 2, p50: null, p95: 1 }], + ["p95", { count: 1, max: 2, p50: 1, p95: null }], + ["max", { count: 1, max: null, p50: 1, p95: 2 }], + ])("says there are no samples when only the %s is missing", (_name, wait) => { + const lines = formatUsage(usageFixture({ totals: figuresFixture({ wait }) })).split("\n"); + + expect(lines).toContain(" Wait: no samples"); + }); +}); + +describe("formatUsage durations", () => { + const waitP50 = (ms: number): string => { + const lines = formatUsage( + usageFixture({ + totals: figuresFixture({ wait: { count: 1, max: ms, p50: ms, p95: ms } }), + }), + ).split("\n"); + const line = lines.find((candidate) => candidate.startsWith(" Wait:")); + return (line ?? "").replace(/^ {2}Wait: +p50 /, "").replace(/,.*$/, ""); + }; + + it.each([ + [0, "0s"], + [250, "250ms"], + [999, "999ms"], + [1_000, "1s"], + [59_999, "60s"], + [60_000, "1m 0s"], + [3_599_999, "59m 59s"], + [3_600_000, "1h 0m"], + [7_380_000, "2h 3m"], + ])("prints %i ms as %s", (ms, expected) => { + expect(waitP50(ms)).toBe(expected); + }); +}); diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index 10002fc3..7d07da0a 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -763,4 +763,239 @@ describe("computeUsage", () => { "device.purge-failed": 2, }); }); + + it("sorts events supplied out of order and leaves the caller's array exactly as it was", () => { + const events = [ + released(T0 + 13_000, "l1"), + granted(T0 + 3_000, "r1", "l1"), + requested(T0 + 1_000, "r1"), + ]; + const before = events.map((event) => event.id); + + const usage = computeUsage(events, WINDOW, WORKER); + + expect(events.map((event) => event.id)).toEqual(before); + expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + expect(usage.totals.held).toEqual({ count: 1, max: 10_000, p50: 10_000, p95: 10_000 }); + // The oldest event is the earliest one by time, not the first one given. + expect(usage.coversFrom).toBe(T0 + 1_000); + }); + + it("names the one worker `local` when it is given no workers", () => { + const usage = computeUsage([capacity(T0 + MINUTE, figures(1, 0, 2))], WINDOW, { + fleet: false, + }); + + expect(usage.workers.map((worker) => worker.id)).toEqual(["local"]); + }); + + it("matches a fleet request to its worker's grant by the bare requester when no requesterPrefix is given", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + at(T0 + 1_100, "request.dispatched", { + model: "m", + platform: "ios", + queuedMs: 0, + reason: "warm-hit", + requestId: "gw-r1", + requesterId: "agent-a", + stage: "s", + workerId: "w1", + }), + granted(T0 + 4_000, "w-r1", "l1", "warm", "agent-a", { workerId: "w1" }), + ], + WINDOW, + { fleet: true }, + ); + + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait).toEqual({ count: 1, max: 3_000, p50: 3_000, p95: 3_000 }); + }); + + it("counts a grant whose source is none of warm, booted and provisioned under no source", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1"), + granted(T0 + 2_000, "r1", "l1", "teleported"), + requested(T0 + 3_000, "r2"), + granted(T0 + 4_000, "r2", "l2", "warm"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.granted).toBe(2); + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 0, warm: 1 }); + }); + + it("keeps durations and incidents out of errors.byCode when device events of every kind are present", () => { + const usage = computeUsage( + [ + at(T0 + 1_000, "device.provisioned", { + deviceId: "d1", + driver: "fake", + duration: 90_000, + spec: { platform: "ios" }, + }), + at(T0 + 2_000, "device.ready", { bootDuration: 20_000, deviceId: "d1" }), + at(T0 + 3_000, "device.quarantined", { deviceId: "d1", reason: "x" }), + at(T0 + 4_000, "device.purge-failed", { + attemptedStrategy: "erase", + deviceId: "d1", + duration: 1, + error: "x", + leaseId: "l", + }), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.errors.byCode).toEqual({ "device.purge-failed": 1 }); + expect(usage.totals.incidents.quarantined).toBe(1); + expect(usage.totals.boot.count).toBe(1); + expect(usage.totals.provisioning.count).toBe(1); + }); + + it("gives each worker only its own capacity steps and each platform only its own entry in them", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(1, 0, 2), undefined, { + android: figures(0, 0, 1), + ios: figures(1, 0, 2), + workerId: "w1", + }), + capacity(T0 - 1_000, figures(3, 0, 6), undefined, { + android: figures(3, 0, 5), + ios: figures(0, 0, 1), + workerId: "w2", + }), + ], + WINDOW, + FLEET, + ); + const byId = Object.fromEntries(usage.workers.map((worker) => [worker.id, worker])); + + expect(byId.w1?.utilisation.slots).toEqual({ max: 2, mean: 1, peak: 1 }); + expect(byId.w2?.utilisation.slots).toEqual({ max: 6, mean: 3, peak: 3 }); + expect(usage.totals.utilisation.slots).toEqual({ max: 8, mean: 4, peak: 4 }); + expect(usage.platforms.ios.utilisation.slots).toEqual({ max: 3, mean: 1, peak: 1 }); + expect(usage.platforms.android.utilisation.slots).toEqual({ max: 6, mean: 3, peak: 3 }); + }); + + it("reports slots.max as the highest total across the window, from stretches that last, and null when no step is known", () => { + const usage = computeUsage( + [ + capacity(T0 + 10 * MINUTE, figures(0, 0, 2)), + capacity(T0 + 20 * MINUTE, figures(0, 0, 9)), + // Two steps at one instant: the 9 holds for no time at all. + capacity(T0 + 30 * MINUTE, figures(0, 0, 4)), + capacity(T0 + 30 * MINUTE, figures(0, 0, 4)), + capacity(T0 + 30 * MINUTE, figures(0, 0, 3)), + ], + { from: T0, to: T0 + 40 * MINUTE }, + WORKER, + ); + const none = computeUsage([], WINDOW, WORKER); + + expect(usage.totals.utilisation.slots.max).toBe(9); + expect(none.totals.utilisation.slots).toEqual({ max: null, mean: null, peak: null }); + }); + + it("does not let a zero-length stretch between two steps at one instant set slots.max", () => { + const usage = computeUsage( + [ + capacity(T0 - 1_000, figures(0, 0, 2)), + capacity(T0 + 10 * MINUTE, figures(0, 0, 50)), + capacity(T0 + 10 * MINUTE, figures(0, 0, 3)), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.utilisation.slots.max).toBe(3); + }); + + it("orders requesters by requests descending, then by id ascending whichever order they arrive in", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "r1", "b"), + requested(T0 + 2_000, "r2", "a"), + requested(T0 + 3_000, "r3", "c"), + requested(T0 + 4_000, "r4", "c"), + requested(T0 + 5_000, "r5", "d"), + requested(T0 + 6_000, "r6", "d"), + requested(T0 + 7_000, "r7", "e"), + ], + WINDOW, + WORKER, + ); + + expect(usage.requesters.map((entry) => [entry.id, entry.requests])).toEqual([ + ["c", 2], + ["d", 2], + ["a", 1], + ["b", 1], + ["e", 1], + ]); + }); + + it("counts a rejection refused before admission under rejected for its requester and not under requests", () => { + const usage = computeUsage( + [ + rejected(T0 + 1_000, "r1", "already-leased", "gw-tok"), + requested(T0 + 2_000, "r2", "gw-tok"), + ], + WINDOW, + { ...WORKER, labels: { "gw-tok": "ci" } }, + ); + + expect(usage.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: "gw-tok", label: "ci", rejected: 1, requests: 1 }, + ]); + expect(usage.totals.requests).toBe(1); + expect(usage.totals.rejected).toEqual({ byReason: { "already-leased": 1 }, total: 1 }); + }); + + it("carries queue depth on a worker's own row and none on a gateway's worker rows", () => { + const events = [ + queueChanged(T0 - 1_000, 2), + capacity(T0 - 1_000, figures(0, 0, 2), undefined, { workerId: "w1" }), + ]; + + const worker = computeUsage(events, WINDOW, WORKER); + const fleet = computeUsage(events, WINDOW, { ...FLEET, workers: [{ id: "w1" }] }); + + expect(worker.workers[0]?.queue).toEqual({ meanDepth: 2, peakDepth: 2 }); + expect(fleet.workers[0]?.queue).toEqual({ meanDepth: null, peakDepth: null }); + }); + + it("counts a request as waiting from the instant it is made until the instant it settles, per series point", () => { + const usage = computeUsage( + [ + // Refused before admission: no request, so it never waits. + rejected(T0 + 10_000, "x", "killed", "z"), + // Made first of all, settled last. + requested(T0 + 20_000, "c", "c"), + // Made, never settled. + requested(T0 + 30_000, "open", "o"), + // Made on a bucket end, settled on a bucket end. + requested(T0 + MINUTE, "a", "a"), + granted(T0 + 2 * MINUTE, "a", "la", "warm", "a"), + // Made after a, settled before it. + requested(T0 + 70_000, "b", "b"), + granted(T0 + 100_000, "b", "lb", "warm", "b"), + rejected(T0 + 5 * MINUTE + 30_000, "c", "timeout", "c"), + ], + WINDOW, + WORKER, + ); + const waiting = usage.series.slice(0, 7).map((point) => point.waiting); + + // Ends at 1m, 2m, ... 7m: c, open and a (made at exactly 1m) wait at 1m; b is made, and b and + // a settle (a at exactly 2m), by 2m; c settles at 5m30s. + expect(waiting).toEqual([3, 2, 2, 2, 2, 1, 1]); + expect(usage.series.at(-1)?.waiting).toBe(1); + }); }); diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts index cc64296f..516cb39c 100644 --- a/src/core/usage/compute-usage.ts +++ b/src/core/usage/compute-usage.ts @@ -112,24 +112,22 @@ function figuresFor(read: ReadEvents, window: UsageWindow, scope: Scope): UsageF const requests = read.requests.filter((fact) => inScope(fact, scope)); const devices = read.devices.filter((fact) => inScope(fact, scope)); const asked = requests.filter((fact) => fact.requestedAt !== undefined); - const granted = requests.filter((fact) => fact.outcome?.kind === "granted"); + const grants = requests.flatMap((fact) => + fact.outcome?.kind === "granted" ? [{ ...fact, outcome: fact.outcome }] : [], + ); const rejected = requests.flatMap((fact) => fact.outcome?.kind === "rejected" ? [fact.outcome.reason] : [], ); return { ...deviceFigures(devices), - bySource: sourcesOf(granted), - granted: granted.length, - held: summarise(granted.flatMap((fact) => spanOf(fact.outcome?.at, fact.endedAt))), + bySource: sourcesOf(grants.map((fact) => fact.outcome.source)), + granted: grants.length, + held: summarise(grants.flatMap((fact) => spanOf(fact.outcome.at, fact.endedAt))), queue: scope.queue === true ? queueOf(read.queue, window) : { meanDepth: null, peakDepth: null }, rejected: { byReason: countBy(rejected), total: rejected.length }, requests: asked.length, - turnaround: summarise( - asked.flatMap((fact) => - fact.outcome?.kind === "granted" ? spanOf(fact.requestedAt, fact.endedAt) : [], - ), - ), + turnaround: summarise(grants.flatMap((fact) => spanOf(fact.requestedAt, fact.endedAt))), utilisation: utilisationOf(read.capacity, window, scope), wait: summarise(asked.flatMap((fact) => spanOf(fact.requestedAt, fact.outcome?.at))), }; @@ -140,10 +138,9 @@ function spanOf(start: number | undefined, end: number | undefined): number[] { return start === undefined || end === undefined ? [] : [end - start]; } -function sourcesOf(granted: readonly RequestFact[]): UsageFigures["bySource"] { +function sourcesOf(grantedSources: readonly string[]): UsageFigures["bySource"] { const sources = { booted: 0, provisioned: 0, warm: 0 }; - for (const fact of granted) { - const source = fact.outcome?.kind === "granted" ? fact.outcome.source : undefined; + for (const source of grantedSources) { if (source === "warm" || source === "booted" || source === "provisioned") sources[source] += 1; } return sources; diff --git a/src/core/usage/read-events.test.ts b/src/core/usage/read-events.test.ts new file mode 100644 index 00000000..47fcadeb --- /dev/null +++ b/src/core/usage/read-events.test.ts @@ -0,0 +1,943 @@ +import { describe, expect, it } from "vitest"; + +import type { EventEnvelope } from "../../bus/index.js"; +import { readEvents, type ReadOptions } from "./read-events.js"; + +const WINDOW = { from: 100, to: 200 }; +const WORKER: ReadOptions = { fleet: false, requesterPrefix: "", self: "self" }; +const FLEET: ReadOptions = { fleet: true, requesterPrefix: "gw:", self: "gateway" }; + +let sequence = 0; + +function at(timestamp: number, event: string, payload: Record): EventEnvelope { + sequence += 1; + return { + event, + id: `evt_${sequence}`, + module: "test", + payload, + seq: sequence, + timestamp, + } as unknown as EventEnvelope; +} + +const read = (events: EventEnvelope[], options: ReadOptions = WORKER) => + readEvents(events, WINDOW, options); + +const entry = (running: number, reserved: number, maxRunning: number) => ({ + maxRunning, + reserved, + running, +}); +const capacityPayload = (extra: Record = {}) => ({ + android: entry(0, 0, 2), + global: entry(1, 2, 5), + ios: entry(1, 0, 3), + ...extra, +}); +const request = (ts: number, requestId: string, requester = "a", extra = {}) => + at(ts, "lease.requested", { requestId, requestSpec: { platform: "ios" }, requester, ...extra }); +const dispatch = (ts: number, requestId: string, requesterId = "a", workerId = "w1") => + at(ts, "request.dispatched", { requestId, requesterId, workerId }); +const relayedGrant = (ts: number, extra: Record = {}) => + at(ts, "lease.granted", { + leaseId: "L", + requester: "gw:a", + source: "warm", + workerId: "w1", + ...extra, + }); +const relayedReject = (ts: number, extra: Record = {}) => + at(ts, "lease.rejected", { + reason: "no-capacity", + requester: "gw:a", + requestId: "r", + workerId: "w1", + ...extra, + }); + +describe("payload guards", () => { + it("ignores a deviceId, spec platform and duration of the wrong types when provisioning", () => { + const result = read([ + at(150, "device.provisioned", { deviceId: 5, duration: "12", spec: { platform: "ios" } }), + at(151, "device.provisioned", { duration: Number.NaN, spec: { platform: "ios" } }), + at(152, "device.provisioned", { duration: Infinity, spec: { platform: "ios" } }), + at(153, "device.provisioned", { duration: -Infinity, spec: { platform: "ios" } }), + at(154, "device.provisioned", { duration: 7, spec: ["ios"] }), + at(155, "device.provisioned", { duration: 8, spec: null }), + at(156, "device.provisioned", { duration: 9, spec: { platform: "windows" } }), + at(157, "device.provisioned", { duration: 0, spec: { platform: "android" } }), + ]); + expect(result.devices).toEqual([ + { kind: "provisioning", platform: undefined, value: 7, worker: "self" }, + { kind: "provisioning", platform: undefined, value: 8, worker: "self" }, + { kind: "provisioning", platform: undefined, value: 9, worker: "self" }, + { kind: "provisioning", platform: "android", value: 0, worker: "self" }, + ]); + }); + + it("does not count a boot whose duration is not a finite number", () => { + const result = read([ + at(150, "device.ready", { bootDuration: "5", deviceId: "d" }), + at(151, "device.ready", { bootDuration: Number.NaN, deviceId: "d" }), + at(152, "device.ready", { bootDuration: Infinity, deviceId: "d" }), + at(153, "device.ready", { deviceId: "d" }), + at(154, "device.ready", { bootDuration: 4, deviceId: "d" }), + ]); + expect(result.devices).toEqual([ + { kind: "boot", platform: undefined, value: 4, worker: "self" }, + ]); + }); +}); + +describe("capacity steps", () => { + const stepFor = (payload: Record) => + read([at(150, "capacity.changed", payload)]).capacity; + + it("turns a complete capacity.changed into a step with used = running + reserved", () => { + expect(stepFor(capacityPayload())).toEqual([ + { + at: 150, + key: "self", + value: { + max: 5, + platforms: { + android: { max: 2, ram: undefined, used: 0 }, + ios: { max: 3, ram: undefined, used: 1 }, + }, + ram: undefined, + used: 3, + }, + }, + ]); + }); + + it.each(["global", "ios", "android"])("ignores a step when %s is missing", (missing) => { + const payload: Record = capacityPayload(); + delete payload[missing]; + expect(stepFor(payload)).toEqual([]); + }); + + it.each(["global", "ios", "android"])("ignores a step when %s is not an object", (name) => { + expect(stepFor(capacityPayload({ [name]: null }))).toEqual([]); + expect(stepFor(capacityPayload({ [name]: 3 }))).toEqual([]); + }); + + it.each(["running", "reserved", "maxRunning"])( + "ignores a step when %s is missing in an entry", + (field) => { + const bad: Record = entry(1, 1, 1); + delete bad[field]; + for (const name of ["global", "ios", "android"]) { + expect(stepFor(capacityPayload({ [name]: bad }))).toEqual([]); + } + }, + ); + + it("ignores a step whose entry figure is not a finite number", () => { + expect(stepFor(capacityPayload({ global: { ...entry(1, 1, 1), running: "1" } }))).toEqual([]); + expect( + stepFor(capacityPayload({ global: { ...entry(1, 1, 1), reserved: Number.NaN } })), + ).toEqual([]); + expect( + stepFor(capacityPayload({ global: { ...entry(1, 1, 1), maxRunning: Infinity } })), + ).toEqual([]); + }); + + it("reads the ram figure from ramBudget and drops it when either side is missing", () => { + const ramOf = (budget: unknown) => stepFor(capacityPayload({ ramBudget: budget }))[0]?.value; + expect(ramOf({ limitBytes: 100, usedBytes: 40 })?.ram).toEqual({ limit: 100, used: 40 }); + expect(ramOf({ limitBytes: 100, usedBytes: 40 })?.platforms.ios.ram).toEqual({ + limit: 100, + used: 40, + }); + expect(ramOf({ limitBytes: 100 })).toMatchObject({ ram: undefined, used: 3 }); + expect(ramOf({ usedBytes: 40 })).toMatchObject({ ram: undefined, used: 3 }); + expect(ramOf({ limitBytes: "100", usedBytes: 40 })).toMatchObject({ ram: undefined }); + expect(ramOf({ limitBytes: 100, usedBytes: Number.NaN })).toMatchObject({ ram: undefined }); + expect(ramOf(null)).toMatchObject({ ram: undefined }); + expect(ramOf([1])).toMatchObject({ ram: undefined, used: 3 }); + }); + + it("keeps a step at exactly the end of the window, ignores one after it", () => { + const result = read([ + at(200, "capacity.changed", capacityPayload()), + at(201, "capacity.changed", capacityPayload()), + ]); + expect(result.capacity.map((s) => s.at)).toEqual([200]); + }); + + it("keeps a step from before the window start so the opening level is known", () => { + expect(read([at(50, "capacity.changed", capacityPayload())]).capacity).toHaveLength(1); + }); + + it("keys a relayed step by its worker and records the worker", () => { + const result = read([at(150, "capacity.changed", capacityPayload({ workerId: "w1" }))], FLEET); + expect(result.capacity.map((s) => s.key)).toEqual(["w1"]); + expect([...result.workers]).toEqual(["w1"]); + }); + + it("ignores a gateway capacity.changed with no workerId", () => { + const result = read([at(150, "capacity.changed", capacityPayload())], FLEET); + expect(result.capacity).toEqual([]); + expect([...result.workers]).toEqual([]); + }); + + it("records the worker of a malformed capacity.changed without adding a step", () => { + const result = read([at(150, "capacity.changed", { workerId: "w1" })], FLEET); + expect(result.capacity).toEqual([]); + expect([...result.workers]).toEqual(["w1"]); + }); + + it("records the worker of a worker's own malformed capacity.changed", () => { + const result = read([at(150, "capacity.changed", {})]); + expect([...result.workers]).toEqual(["self"]); + }); +}); + +describe("queue steps", () => { + it("records a depth at exactly the window end and ignores one after it", () => { + const result = read([ + at(50, "queue.changed", { depth: 1 }), + at(200, "queue.changed", { depth: 2 }), + at(201, "queue.changed", { depth: 3 }), + ]); + expect(result.queue).toEqual([ + { at: 50, key: "queue", value: 1 }, + { at: 200, key: "queue", value: 2 }, + ]); + }); + + it("ignores a depth that is not a finite number", () => { + const result = read([ + at(150, "queue.changed", { depth: "3" }), + at(151, "queue.changed", { depth: Number.NaN }), + at(152, "queue.changed", { depth: Infinity }), + at(153, "queue.changed", {}), + at(154, "queue.changed", { depth: 0 }), + ]); + expect(result.queue).toEqual([{ at: 154, key: "queue", value: 0 }]); + }); + + it("ignores a queue.changed a worker relayed to the gateway", () => { + const result = read( + [ + at(150, "queue.changed", { depth: 4, workerId: "w1" }), + at(151, "queue.changed", { depth: 5 }), + ], + FLEET, + ); + expect(result.queue).toEqual([{ at: 151, key: "queue", value: 5 }]); + }); +}); + +describe("requests on a worker", () => { + it("makes no request of an event at exactly the window start and one of an event at its end", () => { + const result = read([request(100, "r0"), request(200, "r1"), request(201, "r2")]); + expect(result.requests.map((r) => r.requestedAt)).toEqual([200]); + }); + + it("is not a request when requestId or requester is missing or not a string", () => { + const result = read([ + at(150, "lease.requested", { requester: "a" }), + at(151, "lease.requested", { requestId: "r" }), + at(152, "lease.requested", { requestId: 1, requester: "a" }), + at(153, "lease.requested", { requestId: "r", requester: 2 }), + request(154, "ok"), + ]); + expect(result.requests).toEqual([ + { platform: "ios", requestedAt: 154, requester: "a", worker: "self" }, + ]); + }); + + it("reads the platform from requestSpec and leaves it undefined otherwise", () => { + const result = read([ + at(150, "lease.requested", { + requestId: "a", + requestSpec: { platform: "android" }, + requester: "x", + }), + at(151, "lease.requested", { requestId: "b", requestSpec: ["ios"], requester: "x" }), + at(152, "lease.requested", { requestId: "c", requestSpec: null, requester: "x" }), + at(153, "lease.requested", { + requestId: "d", + requestSpec: { platform: "tv" }, + requester: "x", + }), + ]); + expect(result.requests.map((r) => r.platform)).toEqual([ + "android", + undefined, + undefined, + undefined, + ]); + }); + + it("joins a grant with its end by lease id and defaults a missing source to empty", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(130, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests).toEqual([ + { + endedAt: 130, + outcome: { at: 120, kind: "granted", leaseId: "L", source: "" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: "self", + }, + ]); + }); + + it("keeps the source of a grant and joins an expiry as an end", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r", source: "warm" }), + at(130, "lease.expired", { leaseId: "L" }), + ]); + expect(result.requests[0]).toMatchObject({ endedAt: 130, outcome: { source: "warm" } }); + }); + + it("does not join an end of another lease id", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(130, "lease.released", { leaseId: "other" }), + ]); + expect(result.requests[0]).not.toHaveProperty("endedAt"); + }); + + it("ignores a grant missing requestId or leaseId, and one outside the window", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L" }), + at(121, "lease.granted", { requestId: "r" }), + at(100, "lease.granted", { leaseId: "L", requestId: "r" }), + at(201, "lease.granted", { leaseId: "L", requestId: "r" }), + ]); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("takes a grant at exactly the window end", () => { + const result = read([ + request(110, "r"), + at(200, "lease.granted", { leaseId: "L", requestId: "r", source: "cold" }), + ]); + expect(result.requests[0]?.outcome).toEqual({ + at: 200, + kind: "granted", + leaseId: "L", + source: "cold", + }); + }); + + it("ignores an end with no lease id, and one outside the window", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(130, "lease.released", {}), + at(100, "lease.released", { leaseId: "L" }), + at(201, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests[0]).not.toHaveProperty("endedAt"); + }); + + it("takes an end at exactly the window end", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "L", requestId: "r" }), + at(200, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests[0]?.endedAt).toBe(200); + }); + + it("outcome of a rejection of a seen request is the rejection with its reason", () => { + const result = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "no-capacity", requestId: "r", requester: "a" }), + ]); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason: "no-capacity" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: "self", + }, + ]); + }); + + it("ignores a rejection missing requestId, reason or requester, or outside the window", () => { + const result = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "killed", requester: "a" }), + at(121, "lease.rejected", { requestId: "r", requester: "a" }), + at(122, "lease.rejected", { reason: "killed", requestId: "r" }), + at(123, "lease.rejected", { reason: 4, requestId: "r", requester: "a" }), + at(100, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + at(201, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(result.requests).toHaveLength(1); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("takes a rejection at exactly the window end", () => { + const result = read([ + request(110, "r"), + at(200, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(result.requests[0]?.outcome).toEqual({ at: 200, kind: "rejected", reason: "killed" }); + }); + + it("counts a refused-at-admission rejection of an unseen request, with its platform", () => { + const result = read([ + at(150, "lease.rejected", { + reason: "already-leased", + requestId: "x", + requestSpec: { platform: "android" }, + requester: "b", + }), + at(151, "lease.rejected", { reason: "killed", requestId: "y", requester: "b" }), + at(152, "lease.rejected", { reason: "no-capacity", requestId: "z", requester: "b" }), + ]); + expect(result.requests).toEqual([ + { + outcome: { at: 150, kind: "rejected", reason: "already-leased" }, + platform: "android", + requestedAt: undefined, + requester: "b", + worker: "self", + }, + { + outcome: { at: 151, kind: "rejected", reason: "killed" }, + platform: undefined, + requestedAt: undefined, + requester: "b", + worker: "self", + }, + ]); + }); + + it("does not double count a refused-at-admission rejection whose request was seen", () => { + const result = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(result.requests).toHaveLength(1); + }); + + it("an empty payload is read as no facts instead of throwing", () => { + const envelope = { event: "lease.requested", id: "e", module: "t", seq: 1, timestamp: 150 }; + expect(read([envelope as unknown as EventEnvelope]).requests).toEqual([]); + }); +}); + +describe("device facts", () => { + it("counts a provisioning at exactly the window end and not at its start or after", () => { + const result = read([ + at(100, "device.provisioned", { duration: 1 }), + at(200, "device.provisioned", { duration: 2 }), + at(201, "device.provisioned", { duration: 3 }), + ]); + expect(result.devices.map((d) => d.value)).toEqual([2]); + }); + + it("takes the boot platform from the earlier provisioning of the same device on the same worker", () => { + const result = read([ + at(110, "device.provisioned", { deviceId: "d", duration: 1, spec: { platform: "android" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d" }), + at(121, "device.ready", { bootDuration: 6, deviceId: "other" }), + at(122, "device.ready", { bootDuration: 7 }), + ]); + expect(result.devices.map((d) => [d.kind, d.platform, d.value])).toEqual([ + ["provisioning", "android", 1], + ["boot", "android", 5], + ["boot", undefined, 6], + ["boot", undefined, 7], + ]); + }); + + it("remembers the platform even when the provisioning is before the window or has no duration", () => { + const result = read([ + at(50, "device.provisioned", { deviceId: "d", duration: 1, spec: { platform: "ios" } }), + at(60, "device.provisioned", { deviceId: "e", spec: { platform: "android" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d" }), + at(121, "device.ready", { bootDuration: 6, deviceId: "e" }), + ]); + expect(result.devices.map((d) => d.platform)).toEqual(["ios", "android"]); + }); + + it("does not remember a platform for a provisioning with no device id or no platform", () => { + const result = read([ + at(110, "device.provisioned", { duration: 1, spec: { platform: "ios" } }), + at(111, "device.provisioned", { deviceId: "d", duration: 1 }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d" }), + ]); + expect(result.devices.at(-1)).toEqual({ + kind: "boot", + platform: undefined, + value: 5, + worker: "self", + }); + }); + + it("counts each incident event under the name the figure uses, and each error under its event name", () => { + const names: Record = { + "device.quarantine-abandoned": "lost", + "device.quarantine-recovered": "quarantineRecovered", + "device.quarantine-stranded": "lost", + "device.quarantined": "quarantined", + "device.recovered": "crashRecovered", + "device.recovery-failed": "lost", + }; + const incidents = Object.keys(names).map((name, i) => at(110 + i, name, {})); + const errors = [at(130, "device.purge-failed", {}), at(131, "component.install-failed", {})]; + const result = read([ + ...incidents, + ...errors, + at(132, "device.created", {}), + at(133, "lease.other", {}), + ]); + expect(result.devices.map((d) => [d.kind, d.value])).toEqual([ + ...Object.values(names).map((v) => ["incident", v]), + ["error", "device.purge-failed"], + ["error", "component.install-failed"], + ]); + }); + + it("does not count incidents outside the window", () => { + const result = read([at(100, "device.quarantined", {}), at(201, "device.quarantined", {})]); + expect(result.devices).toEqual([]); + }); + + it("takes an incident's platform from its payload, else from the provisioned device", () => { + const result = read([ + at(110, "device.provisioned", { deviceId: "d", spec: { platform: "android" } }), + at(120, "device.quarantined", { deviceId: "d", platform: "ios" }), + at(121, "device.quarantined", { deviceId: "d" }), + at(122, "device.quarantined", { deviceId: "d", platform: "tv" }), + at(123, "device.quarantined", {}), + ]); + expect(result.devices.map((d) => d.platform)).toEqual(["ios", "android", "android", undefined]); + }); + + it("adds the worker to the workers set from a device fact", () => { + expect([...read([at(110, "device.quarantined", {})]).workers]).toEqual(["self"]); + }); +}); + +describe("a gateway", () => { + it("counts a relayed device fact against its worker and ignores one with no worker", () => { + const result = read( + [ + at(110, "device.provisioned", { + deviceId: "d", + duration: 1, + spec: { platform: "ios" }, + workerId: "w1", + }), + at(111, "device.provisioned", { + deviceId: "d", + duration: 2, + spec: { platform: "android" }, + workerId: "w2", + }), + at(112, "device.provisioned", { deviceId: "d", duration: 3, spec: { platform: "ios" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d", workerId: "w2" }), + at(121, "device.ready", { bootDuration: 6, deviceId: "d", workerId: "w3" }), + at(122, "device.ready", { bootDuration: 7, deviceId: "d" }), + at(123, "device.quarantined", { workerId: "w1" }), + at(124, "device.quarantined", {}), + ], + FLEET, + ); + expect(result.devices.map((d) => [d.kind, d.worker, d.platform, d.value])).toEqual([ + ["provisioning", "w1", "ios", 1], + ["provisioning", "w2", "android", 2], + ["boot", "w2", "android", 5], + ["boot", "w3", undefined, 6], + ["incident", "w1", undefined, "quarantined"], + ]); + expect([...result.workers].sort()).toEqual(["w1", "w2", "w3"]); + }); + + it("ignores a lease end that no worker relayed", () => { + const result = read( + [ + request(110, "r"), + dispatch(111, "r"), + relayedGrant(112), + at(130, "lease.released", { leaseId: "L" }), + ], + FLEET, + ); + expect(result.requests[0]).not.toHaveProperty("endedAt"); + }); + + it("joins a relayed grant with the end of the same worker and lease id only", () => { + const events = (endWorker: string, endLease: string) => [ + request(110, "r"), + dispatch(111, "r"), + relayedGrant(112), + at(130, "lease.released", { leaseId: endLease, workerId: endWorker }), + ]; + expect(read(events("w1", "L"), FLEET).requests[0]?.endedAt).toBe(130); + expect(read(events("w2", "L"), FLEET).requests[0]).not.toHaveProperty("endedAt"); + expect(read(events("w1", "M"), FLEET).requests[0]).not.toHaveProperty("endedAt"); + }); + + it("gives a request with no dispatch no worker and no outcome, and does not count its grant", () => { + const result = read([request(110, "r"), relayedGrant(112)], FLEET); + expect(result.requests).toEqual([ + { platform: "ios", requestedAt: 110, requester: "a", worker: undefined }, + ]); + expect([...result.workers]).toEqual([]); + }); + + it("serves a request through its dispatch, and records the worker it was dispatched to", () => { + const result = read([request(110, "r"), dispatch(111, "r", "a", "w9")], FLEET); + expect(result.requests).toEqual([ + { platform: "ios", requestedAt: 110, requester: "a", worker: "w9" }, + ]); + expect([...result.workers]).toEqual(["w9"]); + }); + + it("ignores a dispatch missing requestId, requesterId or workerId, or outside the window", () => { + const result = read( + [ + request(110, "r"), + at(111, "request.dispatched", { requesterId: "a", workerId: "w1" }), + at(112, "request.dispatched", { requestId: "r", workerId: "w1" }), + at(113, "request.dispatched", { requestId: "r", requesterId: "a" }), + dispatch(100, "r"), + dispatch(201, "r"), + ], + FLEET, + ); + expect(result.requests[0]?.worker).toBeUndefined(); + }); + + it("takes a dispatch at exactly the window end", () => { + const result = read([request(110, "r"), dispatch(200, "r")], FLEET); + expect(result.requests[0]?.worker).toBe("w1"); + }); + + it("ignores a dispatch on a worker", () => { + const result = read([request(110, "r"), dispatch(111, "r")]); + expect(result.requests[0]?.worker).toBe("self"); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("counts a relayed grant with its source and lease id, and defaults both to empty", () => { + const withEnd = read( + [ + request(110, "r"), + dispatch(111, "r"), + relayedGrant(112, { leaseId: undefined, source: undefined }), + at(130, "lease.released", { leaseId: "", workerId: "w1" }), + ], + FLEET, + ); + expect(withEnd.requests[0]).toMatchObject({ + endedAt: 130, + outcome: { at: 112, kind: "granted", source: "" }, + }); + const plain = read( + [request(110, "r"), dispatch(111, "r"), relayedGrant(112, { source: "cold" })], + FLEET, + ); + expect(plain.requests[0]?.outcome).toEqual({ at: 112, kind: "granted", source: "cold" }); + }); + + it("counts a relayed rejection with its reason, defaulting a missing reason to empty is impossible so it is ignored", () => { + const rejection = read([request(110, "r"), dispatch(111, "r"), relayedReject(112)], FLEET); + expect(rejection.requests[0]?.outcome).toEqual({ + at: 112, + kind: "rejected", + reason: "no-capacity", + }); + const reasonless = read( + [request(110, "r"), dispatch(111, "r"), relayedReject(112, { reason: undefined })], + FLEET, + ); + expect(reasonless.requests[0]).not.toHaveProperty("outcome"); + }); + + it("ignores a relayed grant or rejection without a requester, and one outside the window", () => { + const result = read( + [ + request(110, "r"), + dispatch(111, "r"), + relayedGrant(112, { requester: undefined }), + relayedReject(113, { requester: undefined }), + relayedGrant(100), + relayedGrant(201), + relayedReject(100), + relayedReject(201), + ], + FLEET, + ); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("ignores a relayed grant of another worker or another requester", () => { + const result = read( + [ + request(110, "r"), + dispatch(111, "r"), + relayedGrant(112, { workerId: "w2" }), + relayedGrant(113, { requester: "gw:b" }), + relayedGrant(114, { requester: "a" }), + ], + FLEET, + ); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("takes the first relayed answer, in event order", () => { + const result = read( + [ + request(110, "r"), + dispatch(111, "r"), + relayedGrant(112, { source: "first" }), + relayedGrant(113, { source: "second" }), + ], + FLEET, + ); + expect(result.requests[0]?.outcome).toMatchObject({ source: "first" }); + }); + + it("lets a grant that came before the dispatch count, but not a rejection", () => { + const grant = read([request(110, "r"), relayedGrant(111), dispatch(112, "r")], FLEET); + expect(grant.requests[0]?.outcome).toMatchObject({ kind: "granted" }); + const rejection = read([request(110, "r"), relayedReject(111), dispatch(112, "r")], FLEET); + expect(rejection.requests[0]).not.toHaveProperty("outcome"); + }); + + it("does not let a grant from before the request count", () => { + const result = read([relayedGrant(105), request(110, "r"), dispatch(111, "r")], FLEET); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("counts a rejection arriving right after the dispatch", () => { + const result = read([request(110, "r"), dispatch(111, "r"), relayedReject(112)], FLEET); + expect(result.requests[0]?.outcome).toMatchObject({ kind: "rejected" }); + }); + + it("cuts off relayed answers at the requester's next request", () => { + const result = read( + [ + request(110, "r1"), + dispatch(111, "r1"), + request(112, "r2"), + dispatch(113, "r2"), + relayedGrant(114, { leaseId: "L2", source: "late" }), + ], + FLEET, + ); + const [first, second] = result.requests; + expect(first).not.toHaveProperty("outcome"); + expect(second?.outcome).toMatchObject({ source: "late" }); + }); + + it("cuts off at a next dispatch of the requester even when its request was not seen", () => { + const result = read( + [request(110, "r1"), dispatch(111, "r1"), dispatch(112, "r2"), relayedGrant(113)], + FLEET, + ); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("does not cut off at another requester's request", () => { + const result = read( + [request(110, "r1"), dispatch(111, "r1"), request(112, "r2", "b"), relayedGrant(113)], + FLEET, + ); + expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); + }); + + it("does not cut off at a request with no requester or a dispatch with no requesterId", () => { + const result = read( + [ + request(110, "r1"), + dispatch(111, "r1"), + at(112, "lease.requested", { requestId: "x" }), + at(113, "request.dispatched", { requestId: "x", workerId: "w1" }), + relayedGrant(114), + ], + FLEET, + ); + expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); + }); + + it("does not cut off at an earlier request of the same requester", () => { + const result = read( + [request(105, "r0"), request(110, "r1"), dispatch(111, "r1"), relayedGrant(112)], + FLEET, + ); + expect(result.requests.map((r) => r.outcome?.kind)).toEqual([undefined, "granted"]); + }); + + it("does not treat a relayed lease.requested as a request or a boundary", () => { + const result = read( + [ + request(110, "r1"), + dispatch(111, "r1"), + request(112, "r9", "a", { workerId: "w1" }), + relayedGrant(113), + ], + FLEET, + ); + expect(result.requests).toHaveLength(1); + expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); + }); + + it("settles a gateway's own rejection of a seen request without a worker", () => { + const result = read( + [ + request(110, "r"), + at(120, "lease.rejected", { reason: "no-capacity", requestId: "r", requester: "a" }), + ], + FLEET, + ); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason: "no-capacity" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: undefined, + }, + ]); + }); + + it("counts a gateway's own refused-at-admission rejection with no worker", () => { + const result = read( + [at(120, "lease.rejected", { reason: "killed", requestId: "x", requester: "a" })], + FLEET, + ); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason: "killed" }, + platform: undefined, + requestedAt: undefined, + requester: "a", + worker: undefined, + }, + ]); + }); + + it("does not count a relayed rejection with a refused-at-admission reason as the gateway's own", () => { + const result = read([relayedReject(120, { reason: "killed", requestId: "x" })], FLEET); + expect(result.requests).toEqual([]); + }); + + it("does not take a relayed lease.requested as the gateway's own request", () => { + const result = read([request(110, "r", "a", { workerId: "w1" })], FLEET); + expect(result.requests).toEqual([]); + }); + + it("ignores a dispatch missing requesterId or workerId, recording no worker for it", () => { + const noRequester = read( + [request(110, "r"), at(111, "request.dispatched", { requestId: "r", workerId: "w1" })], + FLEET, + ); + expect(noRequester.requests[0]?.worker).toBeUndefined(); + expect([...noRequester.workers]).toEqual([]); + const noWorker = read( + [request(110, "r"), at(111, "request.dispatched", { requestId: "r", requesterId: "a" })], + FLEET, + ); + expect(noWorker.requests[0]?.worker).toBeUndefined(); + expect([...noWorker.workers]).toEqual([]); + }); + + it("keeps a grant answering a request before the requester's next request", () => { + const result = read( + [ + request(110, "r1"), + dispatch(111, "r1"), + relayedGrant(112, { source: "mine" }), + request(113, "r2"), + dispatch(114, "r2"), + ], + FLEET, + ); + expect(result.requests[0]?.outcome).toMatchObject({ source: "mine" }); + expect(result.requests[1]).not.toHaveProperty("outcome"); + }); + + it("does not match a worker-less relayed grant to a dispatch to a worker named undefined", () => { + const result = read( + [ + request(110, "r"), + dispatch(111, "r", "a", "undefined"), + at(112, "lease.granted", { leaseId: "L", requester: "gw:a" }), + ], + FLEET, + ); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); + + it("does not match a relayed grant with no requester to a requester named undefined", () => { + const result = read( + [ + request(110, "r", "undefined"), + dispatch(111, "r", "undefined"), + relayedGrant(112, { requester: undefined }), + ], + { ...FLEET, requesterPrefix: "" }, + ); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); +}); + +describe("keys that look like missing ids", () => { + it("does not join an end with no lease id to a grant of a lease named undefined", () => { + const result = read([ + request(110, "r"), + at(120, "lease.granted", { leaseId: "undefined", requestId: "r" }), + at(130, "lease.released", {}), + ]); + expect(result.requests[0]).not.toHaveProperty("endedAt"); + }); + + it("does not remember a platform for a provisioning with no device id under the name undefined", () => { + const result = read([ + at(110, "device.provisioned", { duration: 1, spec: { platform: "android" } }), + at(120, "device.ready", { bootDuration: 5, deviceId: "undefined" }), + ]); + expect(result.devices.at(-1)?.platform).toBeUndefined(); + }); + + it("gives a boot with no device id no platform even if a device was named undefined", () => { + const result = read([ + at(110, "device.provisioned", { + deviceId: "undefined", + duration: 1, + spec: { platform: "android" }, + }), + at(120, "device.ready", { bootDuration: 5 }), + ]); + expect(result.devices.at(-1)?.platform).toBeUndefined(); + }); + + it("does not share a platform between a worker-less provisioning and a worker named undefined", () => { + const result = read( + [ + at(110, "device.provisioned", { + deviceId: "d", + duration: 1, + spec: { platform: "android" }, + }), + at(120, "device.ready", { bootDuration: 5, deviceId: "d", workerId: "undefined" }), + ], + FLEET, + ); + expect(result.devices).toEqual([ + { kind: "boot", platform: undefined, value: 5, worker: "undefined" }, + ]); + }); +}); diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index 2eab2751..3666f112 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -110,12 +110,10 @@ const objectOf = (value: unknown): Payload | undefined => typeof value === "object" && value !== null ? (value as Payload) : undefined; /** A relayed grant or rejection, kept for the request that will claim it. */ -interface Relayed { - readonly index: number; - readonly kind: "granted" | "rejected"; - readonly at: number; - readonly payload: Payload; -} +type RelayedAnswer = + | { readonly kind: "granted"; readonly source: string; readonly leaseId: string } + | { readonly kind: "rejected"; readonly reason: string }; +type Relayed = { readonly index: number; readonly at: number } & RelayedAnswer; interface Dispatch { readonly index: number; @@ -162,7 +160,10 @@ class Reader { readonly #requested = new Map(); readonly #ownRejections = new Map< string, - Rejected & { readonly requester: string; readonly platform: Platform | undefined } + Omit & { + readonly requester: string; + readonly platform: Platform | undefined; + } >(); readonly #grants = new Map(); readonly #ends = new Map(); @@ -193,7 +194,7 @@ class Reader { const seen: Seen = { event, index, - own: !this.options.fleet || event.event === "request.dispatched" || relayedFrom === undefined, + own: !this.options.fleet || relayedFrom === undefined, payload, within: event.timestamp > this.window.from && event.timestamp <= this.window.to, worker: this.options.fleet ? relayedFrom : this.options.self, @@ -211,21 +212,21 @@ class Reader { }; } - #key(worker: string | undefined, id: string): string { - return `${worker ?? ""}\u0000${id}`; + #key(worker: string, id: string): string { + return `${worker}\u0000${id}`; } /** On a gateway a device fact needs a worker behind it; on a worker it is always its own. */ - #counts(seen: Seen): boolean { + #counts(seen: Seen): seen is Seen & { readonly worker: string } { return seen.within && (!this.options.fleet || seen.worker !== undefined); } #capacityChanged(seen: Seen): void { const { event, payload, worker } = seen; - if (event.timestamp > this.window.to || (this.options.fleet && worker === undefined)) return; - const step = capacityStep(payload, event.timestamp, worker ?? ""); + if (event.timestamp > this.window.to || worker === undefined) return; + const step = capacityStep(payload, event.timestamp, worker); if (step !== undefined) this.#capacity.push(step); - if (worker !== undefined) this.#workers.add(worker); + this.#workers.add(worker); } #queueChanged(seen: Seen): void { @@ -236,7 +237,11 @@ class Reader { #boundary(requester: string | undefined, index: number): void { if (requester === undefined) return; - this.#boundaries.set(requester, [...(this.#boundaries.get(requester) ?? []), index]); + this.#boundaries.set(requester, [...this.#boundariesOf(requester), index]); + } + + #boundariesOf(requester: string): readonly number[] { + return this.#boundaries.get(requester) ?? []; } #leaseRequested(seen: Seen): void { @@ -263,12 +268,12 @@ class Reader { this.#dispatches.set(requestId, { index: seen.index, requestId, requester, worker }); } - #relay(seen: Seen, kind: Relayed["kind"]): void { + #relay(seen: Seen, relayed: RelayedAnswer): void { const requester = text(seen.payload, "requester"); if (requester === undefined || seen.worker === undefined) return; const key = this.#key(seen.worker, requester); - const relayed = { at: seen.event.timestamp, index: seen.index, kind, payload: seen.payload }; - this.#relayed.set(key, [...(this.#relayed.get(key) ?? []), relayed]); + const entry = { ...relayed, at: seen.event.timestamp, index: seen.index } as Relayed; + this.#relayed.set(key, [...(this.#relayed.get(key) ?? []), entry]); } #rejected(seen: Seen): void { @@ -280,20 +285,23 @@ class Reader { if (requester === undefined) return; this.#ownRejections.set(requestId, { at: seen.event.timestamp, - kind: "rejected", platform: requestPlatform(seen.payload), reason, requester, }); } else { - this.#relay(seen, "rejected"); + this.#relay(seen, { kind: "rejected", reason }); } } #granted(seen: Seen): void { if (!seen.within) return; if (this.options.fleet) { - this.#relay(seen, "granted"); + this.#relay(seen, { + kind: "granted", + leaseId: text(seen.payload, "leaseId") ?? "", + source: text(seen.payload, "source") ?? "", + }); return; } const requestId = text(seen.payload, "requestId"); @@ -331,7 +339,7 @@ class Reader { if (!this.#counts(seen) || duration === undefined) return; this.#addDevice(seen, { kind: "boot", - platform: this.#platformOfDevice(seen, deviceId), + platform: this.#platformOfDevice(seen.worker, deviceId), value: duration, }); } @@ -342,7 +350,7 @@ class Reader { return; const platform = platformOf(seen.payload.platform) ?? - this.#platformOfDevice(seen, text(seen.payload, "deviceId")); + this.#platformOfDevice(seen.worker, text(seen.payload, "deviceId")); this.#addDevice(seen, { kind: incident === undefined ? "error" : "incident", platform, @@ -350,10 +358,10 @@ class Reader { }); } - #platformOfDevice(seen: Seen, deviceId: string | undefined): Platform | undefined { + #platformOfDevice(worker: string, deviceId: string | undefined): Platform | undefined { return deviceId === undefined ? undefined - : this.#devicePlatform.get(this.#key(seen.worker, deviceId)); + : this.#devicePlatform.get(this.#key(worker, deviceId)); } #addDevice(seen: Seen, fact: Omit): void { @@ -386,10 +394,10 @@ class Reader { } return this.options.fleet ? this.#fleetFact(base, requestId, request) - : this.#workerFact(base, requestId, own); + : this.#workerFact(base, requestId, this.options.self); } - #workerFact(base: RequestFactBase, requestId: string, own: string | undefined): RequestFact { + #workerFact(base: RequestFactBase, requestId: string, own: string): RequestFact { const grant = this.#grants.get(requestId); return grant === undefined ? { ...base, worker: own } @@ -405,18 +413,14 @@ class Reader { const relayed = this.#fleetOutcome(dispatch, request.index); if (relayed === undefined) return { ...base, worker }; if (relayed.kind === "rejected") { - const reason = text(relayed.payload, "reason") ?? ""; - return { ...base, outcome: { at: relayed.at, kind: "rejected", reason }, worker }; + const outcome: Rejected = { at: relayed.at, kind: "rejected", reason: relayed.reason }; + return { ...base, outcome, worker }; } - const source = text(relayed.payload, "source") ?? ""; - return this.#withEnd( - { ...base, outcome: { at: relayed.at, kind: "granted", source }, worker }, - text(relayed.payload, "leaseId") ?? "", - worker, - ); + const outcome: Granted = { at: relayed.at, kind: "granted", source: relayed.source }; + return this.#withEnd({ ...base, outcome, worker }, relayed.leaseId, worker); } - #withEnd(fact: RequestFact, leaseId: string, worker: string | undefined): RequestFact { + #withEnd(fact: RequestFact, leaseId: string, worker: string): RequestFact { const endedAt = this.#ends.get(this.#key(worker, leaseId)); return endedAt === undefined ? fact : { ...fact, endedAt }; } @@ -429,9 +433,7 @@ class Reader { * request. */ #fleetOutcome(dispatch: Dispatch, requestIndex: number): Relayed | undefined { - const next = (this.#boundaries.get(dispatch.requester) ?? []).find( - (index) => index > dispatch.index, - ); + const next = this.#boundariesOf(dispatch.requester).find((index) => index > dispatch.index); const key = this.#key(dispatch.worker, `${this.options.requesterPrefix}${dispatch.requester}`); return (this.#relayed.get(key) ?? []).find( (candidate) => From 8f9c4d84c3c29deb33ec01165eadc546635c4722 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 19:15:37 +0200 Subject: [PATCH 09/23] test: a requester without a token label carries no label key --- src/core/usage/compute-usage.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index 7d07da0a..4a185538 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -708,7 +708,7 @@ describe("computeUsage", () => { { ...WORKER, labels: { tok_b: "ci-bot" } }, ); - expect(usage.requesters).toEqual([ + expect(usage.requesters).toStrictEqual([ { granted: 0, heldTotalMs: 0, id: "tok_a", rejected: 1, requests: 3 }, { granted: 1, heldTotalMs: 5_000, id: "tok_b", label: "ci-bot", rejected: 0, requests: 1 }, ]); From c73df7a27580056c90dd3e4abef0eb2f6fe0adbd Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 19:26:47 +0200 Subject: [PATCH 10/23] fix: round the usage window down at both ends, count a rejection by its request, give core/usage an index window.to ceil -> floor; a killed rejection of a request made before the window is in no count (requestedBefore carried by the history read); core/usage/index.ts and a COMPONENTS row; a test for the unreadable oldestTs message. --- docs/CLI.md | 7 ++--- docs/CLIENT.md | 2 +- docs/internal/ARCHITECTURE.md | 7 ++--- docs/internal/COMPONENTS.md | 1 + src/bus/event-file.test.ts | 36 +++++++++++++++++++++++++- src/bus/event-file.ts | 40 ++++++++++++++++++++++++----- src/core/usage/compute-usage.ts | 7 ++++- src/core/usage/index.ts | 2 ++ src/core/usage/read-events.test.ts | 29 +++++++++++++++++++-- src/core/usage/read-events.ts | 11 +++++--- src/core/usage/usage-reader.test.ts | 6 ++--- src/core/usage/usage-reader.ts | 22 +++++++++------- src/daemon/dispatcher.test.ts | 22 +++++++++++++--- src/daemon/dispatcher.ts | 2 +- src/gateway/dispatcher.test.ts | 4 +-- src/gateway/dispatcher.ts | 2 +- 16 files changed, 160 insertions(+), 40 deletions(-) create mode 100644 src/core/usage/index.ts diff --git a/docs/CLI.md b/docs/CLI.md index c6ecef57..fa457e18 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -1930,9 +1930,10 @@ What the figures count: installs that failed. - A requester that is a token shows the token's label beside its id. -The window is widened to whole steps of the time series `--json` carries, which -is 1 minute for a window of a few hours and grows to 1 day for 90 days; the -window in the answer is the widened one. +The window is rounded down to whole steps of the time series `--json` carries, +which is 1 minute for a window of a few hours and grows to 1 day for 90 days, so +it never ends at a time that has not come yet; the window in the answer is the +rounded one. Against a **gateway** the totals are the fleet's and there is a row for each worker, with the grants, hold times and device figures of that worker. Requests, diff --git a/docs/CLIENT.md b/docs/CLIENT.md index 386d221f..c94fe3ec 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -554,7 +554,7 @@ usage.series; // one point per bucket, for a chart The daemon computes the figures from its event history, so they cover only what the history holds: `partial` is `true` and `coversFrom` says where they start when it does not reach the start of the window. `window` in the answer is the window -asked for, widened to a whole number of `bucketMs`, and the series never has more +asked for, rounded down to a whole number of `bucketMs` at both ends, and the series never has more than 200 points. A window that ends before the oldest event the history holds rejects with `HISTORY_NOT_KEPT`; its `details.oldestTs` is the oldest time the history reaches. Against a gateway the totals are the fleet's and `workers` has diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index bf7fb570..3daa2194 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -1947,14 +1947,15 @@ pushes, so the join neither loses nor repeats an event. turns it into numbers. `computeUsage` (`src/core/usage/`) is pure: it takes the envelopes of a window, with the latest `capacity.changed` and `queue.changed` at or before the window's start that `EventHistory.read`'s `carry` adds, and -returns the figures. `readEvents` reads them into one fact per request, joined +the ids of the requests made before it (`requestedBefore`, so a rejection of +one is in no count), and returns the figures. `readEvents` reads them into one fact per request, joined by `requestId` and `leaseId`, and one per device event; `compute-usage.ts` adds them up by platform, by worker and by requester. On a gateway (`fleet`) request facts come from its own events and device facts from the events its workers relayed, and a request is joined to its worker's grant or rejection through `request.dispatched`; that rule is in `readEvents` and nowhere else. One -`UsageReader` serves both dispatchers: it rounds the window out to the series -bucket, keeps its last answer by that window and the newest event id, and joins +`UsageReader` serves both dispatchers: it rounds the window down to the series +bucket at both ends, keeps its last answer by that window and the newest event id, and joins token labels, so the daemon's two handlers differ only in `fleet`. `simlock stats` prints what the operation returns and computes nothing. diff --git a/docs/internal/COMPONENTS.md b/docs/internal/COMPONENTS.md index 83ad512b..bc4029b7 100644 --- a/docs/internal/COMPONENTS.md +++ b/docs/internal/COMPONENTS.md @@ -55,6 +55,7 @@ coordinator into the components below. | `Registry` | `src/core/registry.ts` | Devices, leases, lease requests and component records, written through one commit to `state.json`; the device transition function's only caller; emits the post-commit device and lease facts. | Decide a transition: callers do, inside a decision section. | | `SerializedDecision` | `src/core/serialized-decision.ts` | Serialising short read-decide-commit sections. | Hold driver work or other long I/O. | | `DeviceOperationClaims` | `src/core/device-operation-claims.ts` | Exclusive per-device operation claims: boot, eviction, cleanup, nuke, reclaim. | Any lifecycle, cleanup or leasing policy. | +| `UsageReader`, `computeUsage` | `src/core/usage/` (surface: `index.ts`) | `usage.get`'s one answerer for a worker and a gateway: reads the event history for a window rounded down to the series bucket, turns it into figures (`computeUsage`, `readEvents`, the timelines), joins token labels, keeps the last answer by window and newest event. | Write anything, or read the registry, capacity or lifecycle engine: events in, numbers out. | | `LeaseRequestBook` | `src/core/lease-request-book.ts` | The lease-request rules: store before queueing, answer a repeat under the same idempotency key, write a result once. Shared with the gateway over an in-memory store. | Queue or serve the request. | | `WaitQueue` | `src/core/wait-queue.ts` | FIFO membership, timeouts, cancellation, progress, and settlement of pending requests. | Decide whether capacity exists or perform lease work. | | `LeaseAcquisitionCoordinator` | `src/core/lease-acquisition-coordinator.ts` | Admission, resolving a request into a requirement and a create spec (class, OS range, default mode, image tag), driving plans to a grant, eviction by demand, maintenance fencing for nuke. Exposes the queue head's spec and the waiting requests, read only. | Choose which device: the planner does. Reclaim: the release side does. | diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index fcd615ca..32b732f8 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -228,7 +228,7 @@ describe("EventHistory", () => { const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_000 }); - expect(read).toEqual({ events: [early, late], oldestTs: 1_000 }); + expect(read).toEqual({ events: [early, late], oldestTs: 1_000, requestedBefore: new Set() }); expect(await events.replay({ carry: ["queue.changed"], sinceTs: 2_000 })).toEqual([ early, late, @@ -401,6 +401,40 @@ describe("readEventFile", () => { expect(empty.oldestTs).toBeUndefined(); }); + it("reports the requestId of each lease.requested at or before sinceTs, and none from after it or from other events", async () => { + const requested = (seq: number, timestamp: number, requestId: string): EventEnvelope => + ({ + ...envelope(seq, timestamp), + event: "lease.requested", + payload: { requestId }, + }) as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + requested(1, 100, "old"), + envelope(2, 150), + requested(3, 200, "edge"), + requested(4, 300, "new"), + ), + }); + + const read = await readEventHistory(filesystem, "/data/events.jsonl", { sinceTs: 200 }); + + expect([...read.requestedBefore].sort()).toEqual(["edge", "old"]); + }); + + it("reports the requests made before sinceTs from the ring too", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + bus.emit("lease.requested", { requestId: "old", requester: "a" } as never, "lease"); + clock.advance(2_000); + bus.emit("lease.requested", { requestId: "new", requester: "a" } as never, "lease"); + + const read = await events.read({ carry: [], sinceTs: 2_000 }); + + expect([...read.requestedBefore]).toEqual(["old"]); + }); + it("returns the generations when the current file is missing", async () => { const filesystem = await filesystemWith({ "/data/events.jsonl.1": lines(envelope(1, 100)), diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index f20b15ea..bec8dae6 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -37,11 +37,12 @@ export async function readEventHistory( filesystem: Filesystem, path: string, { sinceTs, carry = [] }: { readonly sinceTs: number; readonly carry?: readonly EventName[] }, -): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { +): Promise { const generations = await readGenerations(filesystem, path); const seen = new Set(); const events: EventEnvelope[] = []; const carried = new Map(); + const requestedBefore = new Set(); let oldestTs: number | undefined; // Oldest generation first, so events that tie on time and seq keep the order they were written. for (const line of generations.reverse().flat()) { @@ -50,13 +51,33 @@ export async function readEventHistory( oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); if (envelope.timestamp <= sinceTs) { keepIfCarried(carried, envelope, carry); + noteRequest(requestedBefore, envelope); continue; } if (seen.has(eventKey(envelope))) continue; seen.add(eventKey(envelope)); events.push(envelope); } - return { events: [...carried.values(), ...events].sort(byTimeThenSeq), oldestTs }; + return { + events: [...carried.values(), ...events].sort(byTimeThenSeq), + oldestTs, + requestedBefore, + }; +} + +/** What a history read answers: the events, how far back it reaches, and which requests were made + * before `sinceTs` (a rejection in the window follows its request, ADR 0016 §4). */ +export interface EventHistoryRead { + readonly events: EventEnvelope[]; + readonly oldestTs: number | undefined; + /** The `requestId` of every `lease.requested` at or before `sinceTs`. */ + readonly requestedBefore: ReadonlySet; +} + +function noteRequest(requestedBefore: Set, envelope: EventEnvelope): void { + if (envelope.event !== "lease.requested") return; + const requestId = (envelope.payload as { readonly requestId?: unknown }).requestId; + if (typeof requestId === "string") requestedBefore.add(requestId); } /** @@ -184,7 +205,7 @@ export class EventHistory { async read(input: { readonly sinceTs: number; readonly carry: readonly EventName[]; - }): Promise<{ readonly events: EventEnvelope[]; readonly oldestTs: number | undefined }> { + }): Promise { const { filesystem, logger, path } = this.#options; if (!this.#writing) return this.#readRing(input); try { @@ -209,15 +230,22 @@ export class EventHistory { }: { readonly sinceTs: number; readonly carry: readonly EventName[]; - }): { readonly events: EventEnvelope[]; readonly oldestTs: number | undefined } { + }): EventHistoryRead { const ring = this.#options.bus.replay(); const carried = new Map(); + const requestedBefore = new Set(); let oldestTs: number | undefined; for (const envelope of ring) { oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); - if (envelope.timestamp <= sinceTs) keepIfCarried(carried, envelope, carry); + if (envelope.timestamp > sinceTs) continue; + keepIfCarried(carried, envelope, carry); + noteRequest(requestedBefore, envelope); } const newer = ring.filter((envelope) => envelope.timestamp > sinceTs); - return { events: [...carried.values(), ...newer].sort(byTimeThenSeq), oldestTs }; + return { + events: [...carried.values(), ...newer].sort(byTimeThenSeq), + oldestTs, + requestedBefore, + }; } } diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts index 516cb39c..507fdfef 100644 --- a/src/core/usage/compute-usage.ts +++ b/src/core/usage/compute-usage.ts @@ -16,6 +16,10 @@ import { peakAndMean, segmentsOf, type Step, valuesAt } from "./timeline.js"; export type { UsageWindow } from "./read-events.js"; export interface UsageOptions { + /** The series bucket, picked once from the window asked for; the window's own span when absent. */ + readonly bucketMs?: number; + /** The `requestId` of every request made before the window; a rejection of one is in no count. */ + readonly requestedBefore?: ReadonlySet; /** The gateway rule (ADR 0016 §6): request facts from the gateway's own events, device facts * from the events its workers relayed. */ readonly fleet: boolean; @@ -61,10 +65,11 @@ export function computeUsage( const workers = options.workers ?? []; const read = readEvents(sorted, window, { fleet: options.fleet, + requestedBefore: options.requestedBefore ?? new Set(), requesterPrefix: options.requesterPrefix ?? "", self: workers[0]?.id ?? "local", }); - const bucketMs = seriesBucketMs(window.to - window.from); + const bucketMs = options.bucketMs ?? seriesBucketMs(window.to - window.from); const oldest = options.oldestTs ?? sorted[0]?.timestamp; const coversFrom = oldest === undefined ? window.from : Math.max(window.from, oldest); const figures = (scope: Scope) => figuresFor(read, window, scope); diff --git a/src/core/usage/index.ts b/src/core/usage/index.ts new file mode 100644 index 00000000..f14f039f --- /dev/null +++ b/src/core/usage/index.ts @@ -0,0 +1,2 @@ +export { tokenLabelMap, UsageReader } from "./usage-reader.js"; +export type { UsageHistory, UsageReaderOptions, UsageResult } from "./usage-reader.js"; diff --git a/src/core/usage/read-events.test.ts b/src/core/usage/read-events.test.ts index 47fcadeb..d03aa75a 100644 --- a/src/core/usage/read-events.test.ts +++ b/src/core/usage/read-events.test.ts @@ -4,8 +4,18 @@ import type { EventEnvelope } from "../../bus/index.js"; import { readEvents, type ReadOptions } from "./read-events.js"; const WINDOW = { from: 100, to: 200 }; -const WORKER: ReadOptions = { fleet: false, requesterPrefix: "", self: "self" }; -const FLEET: ReadOptions = { fleet: true, requesterPrefix: "gw:", self: "gateway" }; +const WORKER: ReadOptions = { + fleet: false, + requestedBefore: new Set(), + requesterPrefix: "", + self: "self", +}; +const FLEET: ReadOptions = { + fleet: true, + requestedBefore: new Set(), + requesterPrefix: "gw:", + self: "gateway", +}; let sequence = 0; @@ -428,6 +438,21 @@ describe("requests on a worker", () => { expect(result.requests).toHaveLength(1); }); + it("counts a killed rejection of a request made before the window in no count, and one made inside it with that request", () => { + const before = read( + [at(120, "lease.rejected", { reason: "killed", requestId: "old", requester: "a" })], + { ...WORKER, requestedBefore: new Set(["old"]) }, + ); + expect(before.requests).toEqual([]); + + const inside = read([ + request(110, "r"), + at(120, "lease.rejected", { reason: "killed", requestId: "r", requester: "a" }), + ]); + expect(inside.requests).toHaveLength(1); + expect(inside.requests[0]).toMatchObject({ requestedAt: 110 }); + }); + it("an empty payload is read as no facts instead of throwing", () => { const envelope = { event: "lease.requested", id: "e", module: "t", seq: 1, timestamp: 150 }; expect(read([envelope as unknown as EventEnvelope]).requests).toEqual([]); diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index 3666f112..41e51c7d 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -64,6 +64,8 @@ export interface ReadEvents { export interface ReadOptions { readonly fleet: boolean; + /** The `requestId` of every request made before the window. */ + readonly requestedBefore: ReadonlySet; /** The id a worker's own events are attributed to. */ readonly self: string; /** What a gateway puts in front of a requester it forwards. */ @@ -78,8 +80,10 @@ export interface UsageWindow { type Payload = Readonly>; type Incident = keyof UsageFigures["incidents"]; -/** A rejection whose request the events do not show was refused before it was stored, or belongs - * to a request from before the window; only these two reasons are ever given before admission. */ +/** A rejection with no `lease.requested` anywhere in the history read was refused before the + * request was stored; only these two reasons are ever given before admission. One whose request + * is in the history follows that request: counted with it, or in no count when it was made before + * the window. */ const REFUSED_AT_ADMISSION = new Set(["already-leased", "killed"]); /** The device events that count as an incident, by what the figure calls them. */ const INCIDENT_OF: Readonly> = { @@ -447,7 +451,8 @@ class Reader { #refusedAtAdmission(): RequestFact[] { const facts: RequestFact[] = []; for (const [requestId, rejection] of this.#ownRejections) { - if (this.#requested.has(requestId) || !REFUSED_AT_ADMISSION.has(rejection.reason)) continue; + if (this.#requested.has(requestId) || this.options.requestedBefore.has(requestId)) continue; + if (!REFUSED_AT_ADMISSION.has(rejection.reason)) continue; facts.push({ outcome: { at: rejection.at, kind: "rejected", reason: rejection.reason }, platform: rejection.platform, diff --git a/src/core/usage/usage-reader.test.ts b/src/core/usage/usage-reader.test.ts index 871aadf8..e4a2413c 100644 --- a/src/core/usage/usage-reader.test.ts +++ b/src/core/usage/usage-reader.test.ts @@ -34,7 +34,7 @@ function history(events: readonly EventEnvelope[] = [], oldestTs: number | undef latestId: () => state.newest, read: async (input) => { state.asked.push(input); - return { events: state.events, oldestTs: state.oldestTs }; + return { events: state.events, oldestTs: state.oldestTs, requestedBefore: new Set() }; }, }; return { source, state }; @@ -63,13 +63,13 @@ const answer = (result: Awaited>) => { }; describe("UsageReader", () => { - it("widens the window out to the bucket on both sides and reads the history from its start, with the two step events carried", async () => { + it("rounds the window down to the bucket on both sides, so it never ends in the future, and reads the history from its start, with the two step events carried", async () => { const { source, state } = history(); const { usage } = reader(source); const result = answer(await usage.get({ from: T0 + 10_000, to: T0 + HOUR + 10_000 })); - expect(result.window).toEqual({ from: T0, to: T0 + HOUR + MINUTE }); + expect(result.window).toEqual({ from: T0, to: T0 + HOUR }); expect(result.bucketMs).toBe(MINUTE); expect(state.asked).toEqual([{ carry: ["capacity.changed", "queue.changed"], sinceTs: T0 }]); }); diff --git a/src/core/usage/usage-reader.ts b/src/core/usage/usage-reader.ts index aef1822d..4543c195 100644 --- a/src/core/usage/usage-reader.ts +++ b/src/core/usage/usage-reader.ts @@ -7,10 +7,12 @@ const STEP_EVENTS: readonly EventName[] = ["capacity.changed", "queue.changed"]; /** The part of the event history `usage.get` reads (`EventHistory` satisfies it). */ export interface UsageHistory { - read(input: { - readonly sinceTs: number; - readonly carry: readonly EventName[]; - }): Promise<{ readonly events: readonly EventEnvelope[]; readonly oldestTs: number | undefined }>; + read(input: { readonly sinceTs: number; readonly carry: readonly EventName[] }): Promise<{ + readonly events: readonly EventEnvelope[]; + readonly oldestTs: number | undefined; + /** The `requestId` of every `lease.requested` at or before `sinceTs`. */ + readonly requestedBefore: ReadonlySet; + }>; /** The id of the event published last; the memo is good while it has not moved. */ latestId(): string | undefined; } @@ -31,8 +33,8 @@ export type UsageResult = { readonly usage: UsageOutput } | { readonly oldestTs: /** * Answers `usage.get` for a worker or a gateway from the same code (ADR 0016): reads the history - * for the window, hands it to `computeUsage`, joins token labels. The window is rounded out to the - * series bucket, and the last answer is kept by that window and the newest event, so a caller that + * for the window, hands it to `computeUsage`, joins token labels. The window is rounded down to the + * series bucket, both ends, so it never reaches a time that has not come, and the last answer is kept by that window and the newest event, so a caller that * asks again within the bucket while nothing happened reads nothing. */ export class UsageReader { @@ -42,23 +44,25 @@ export class UsageReader { async get(asked: UsageWindow): Promise { const bucketMs = seriesBucketMs(asked.to - asked.from); - // Out to the bucket on both sides, so the window asked for is inside the one answered. + // Down to the bucket on both sides (ADR 0016): the answer never reaches a time not yet come. const window = { from: Math.floor(asked.from / bucketMs) * bucketMs, - to: Math.ceil(asked.to / bucketMs) * bucketMs, + to: Math.floor(asked.to / bucketMs) * bucketMs, }; const key = `${window.from}:${window.to}:${String(this.options.history.latestId())}`; if (this.#memo?.key === key) return { usage: this.#memo.usage }; - const { events, oldestTs } = await this.options.history.read({ + const { events, oldestTs, requestedBefore } = await this.options.history.read({ carry: STEP_EVENTS, sinceTs: window.from, }); if (oldestTs !== undefined && oldestTs > asked.to) return { oldestTs }; const usage = computeUsage(events, window, { + bucketMs, fleet: this.options.fleet !== undefined, labels: await this.#labelsFor(events), oldestTs, + requestedBefore, requesterPrefix: this.options.fleet?.requesterPrefix ?? "", workers: this.options.workers(), }); diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 512d0510..60ad118a 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -1121,7 +1121,7 @@ describe("Dispatcher: events.replay", () => { /** The parts of the history `usage.get` reads, for a test that is not about it. */ const noUsageHistory = { latestId: () => undefined, - read: async () => ({ events: [], oldestTs: undefined }), + read: async () => ({ events: [], oldestTs: undefined, requestedBefore: new Set() }), }; describe("Dispatcher: usage.get", () => { @@ -1135,7 +1135,7 @@ describe("Dispatcher: usage.get", () => { latestId: () => state.newest, read: async () => { state.reads += 1; - return { events: [], oldestTs }; + return { events: [], oldestTs, requestedBefore: new Set() }; }, replay: async () => [], }; @@ -1155,8 +1155,8 @@ describe("Dispatcher: usage.get", () => { expect(state.reads).toBe(1); expect(second).toEqual(first); - // The answer's window is the window asked for rounded out to the bucket, not the one asked for. - expect(second.window).toEqual({ from: WINDOW.from, to: WINDOW.to + minute }); + // The answer's window is the window asked for rounded down to the bucket, never into the future. + expect(second.window).toEqual({ from: WINDOW.from, to: WINDOW.to }); state.newest = "evt_2"; await call(asked); @@ -1178,6 +1178,20 @@ describe("Dispatcher: usage.get", () => { }); }); + it("the worker handler names an oldest timestamp no date can hold as the number in HISTORY_NOT_KEPT's message", async () => { + const unreadable = 9e15; + const { history } = countingHistory(unreadable); + const { dispatcher } = await buildDispatcher({ eventHistory: history }); + + await expect( + dispatcher.dispatch("usage.get", WINDOW, session({ role: "admin" })), + ).rejects.toMatchObject({ + code: "HISTORY_NOT_KEPT", + details: { oldestTs: unreadable }, + message: expect.stringContaining("its oldest event is from 9000000000000000."), + }); + }); + it("the worker handler answers for itself as one worker, with the token label beside a requester it knows", async () => { const { dispatcher, eventBus, tokens } = await buildDispatcher(); const { record } = await tokens.create("agent", "ci-bot"); diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index dae58a6f..7eeca41f 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -57,7 +57,7 @@ import { type ErasedHandler, usageAnswer, } from "./dispatch.js"; -import { tokenLabelMap, UsageReader } from "../core/usage/usage-reader.js"; +import { tokenLabelMap, UsageReader } from "../core/usage/index.js"; export { DispatchError, type ContractDispatcher, type DispatchSession } from "./dispatch.js"; diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index bcdf1271..8ea9faa6 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -564,7 +564,7 @@ describe("GatewayDispatcher", () => { const { dispatcher } = harness({ eventHistory: { latestId: () => undefined, - read: async () => ({ events: [], oldestTs: undefined }), + read: async () => ({ events: [], oldestTs: undefined, requestedBefore: new Set() }), replay: async (input) => { asked.push(input); return fromHistory; @@ -684,7 +684,7 @@ describe("GatewayDispatcher", () => { latestId: () => "evt_1", read: async () => { reads += 1; - return { events: [], oldestTs: 99 * 3_600_000 }; + return { events: [], oldestTs: 99 * 3_600_000, requestedBefore: new Set() }; }, replay: async () => [], }, diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 52180268..6f915384 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -50,7 +50,7 @@ import { } from "../daemon/dispatch.js"; import type { Clock, Logger } from "../ports/index.js"; import { NoopLogger } from "../ports/index.js"; -import { tokenLabelMap, UsageReader } from "../core/usage/usage-reader.js"; +import { tokenLabelMap, UsageReader } from "../core/usage/index.js"; import { aggregateCatalog, aggregateStatus, type AggregateStatusOptions } from "./aggregate.js"; import { relayComponentInstall } from "./component-relay.js"; import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; From 30a52808345b996a897f2304debd1a9235cd5576 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 19:36:03 +0200 Subject: [PATCH 11/23] test: the stats e2e waits for the window to close over the newest event --- e2e/usage-stats.test.ts | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index 7e19789f..06b13ae5 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -48,6 +48,29 @@ async function stats( return result.json as Usage; } +/** + * The figures once the window has closed over the newest event: the window is rounded down to the + * series bucket, so what happened in the minute now running is counted when the minute is over. + */ +async function settledStats( + env: { + cli: (args: string[]) => Promise<{ code: number | null; json?: unknown }>; + events: () => Promise; + }, + until: (usage: Usage) => boolean = () => true, +): Promise { + let usage: Usage | undefined; + await waitFor( + async () => { + const newest = Math.max(...(await env.events()).map((event) => event.timestamp)); + usage = await stats(env); + return usage.window.to >= newest && until(usage); + }, + { interval: 1_000, label: "the window closed over the newest event", timeout: 90_000 }, + ); + return usage as Usage; +} + function countBy(values: readonly string[]): Record { const counts: Record = {}; for (const value of values) counts[value] = (counts[value] ?? 0) + 1; @@ -93,7 +116,7 @@ describe("simlock stats", () => { (await env.cli(["release", (second.json as { lease: { id: string } }).lease.id])).code, ).toBe(0); - const usage = await stats(env); + const usage = await settledStats(env); const everything = await env.events(); const history = inWindow(everything, usage.window); const named = (name: string) => history.filter((event) => event.event === name); @@ -161,11 +184,11 @@ describe("simlock stats", () => { expect( (await env.cli(["release", (lease.json as { lease: { id: string } }).lease.id])).code, ).toBe(0); - const before = await stats(env); + const before = await settledStats(env); expect(before.totals.granted).toBe(1); await env.restartDaemon(); - const after = await stats(env); + const after = await settledStats(env); expect(after.totals).toMatchObject({ boot: before.totals.boot, @@ -219,19 +242,16 @@ describe("simlock stats", () => { } let fleet: Usage | undefined; - await waitFor( - async () => { - fleet = await stats(gateway); - return fleet.totals.granted === 2 && fleet.workers.length === 2; - }, - { label: "the gateway's figures show both grants", timeout: 30_000 }, + fleet = await settledStats( + gateway, + (usage) => usage.totals.granted === 2 && usage.workers.length === 2, ); expect(fleet?.totals.requests).toBe(2); expect(fleet?.workers.map((worker) => worker.label).sort()).toEqual(["worker-a", "worker-b"]); expect(fleet?.workers.map((worker) => worker.granted)).toEqual([1, 1]); for (const worker of [workerA, workerB]) { - const own = await stats(worker); + const own = await settledStats(worker); expect(own.workers).toHaveLength(1); expect(own.totals.granted).toBe(1); expect(own.workers[0]?.granted).toBe(1); From a98bfc01ddf131e5eb8ca90268e87d5636c15544 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 20:12:06 +0200 Subject: [PATCH 12/23] feat(usage): a fleet request joins its requester's first relayed answer, failures count by event name, waiting carries requests open at the window's start (#347) --- docs/CLI.md | 7 +- docs/CLIENT.md | 4 +- docs/internal/ARCHITECTURE.md | 9 +- ...-derived-on-read-from-the-event-history.md | 25 ++-- e2e/usage-stats.test.ts | 25 ++++ src/bus/event-file.test.ts | 33 ++++++ src/bus/event-file.ts | 16 ++- src/cli/index.test.ts | 2 +- src/cli/stats.test.ts | 20 ++-- src/cli/stats.ts | 4 +- src/contract/operations.test.ts | 2 +- src/contract/schemas.ts | 2 +- src/core/usage/compute-usage.test.ts | 112 ++++++++++++++---- src/core/usage/compute-usage.ts | 11 +- src/core/usage/read-events.test.ts | 101 +++++++++++++--- src/core/usage/read-events.ts | 111 +++++++++++------ src/core/usage/usage-reader.test.ts | 15 ++- src/core/usage/usage-reader.ts | 10 +- src/simlock-client/client.test.ts | 2 +- 19 files changed, 391 insertions(+), 120 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index fa457e18..bcb7df80 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -1925,9 +1925,10 @@ What the figures count: time is left out of the figures rather than counted as zero. - **Incidents** count devices quarantined, devices recovered after a crash, devices recovered from quarantine, and devices lost (a recovery that failed, or - a quarantined device given up on). **Errors** counts failures that carry no - reason of their own among the rejections: purges that failed and component - installs that failed. + a quarantined device given up on). **Failures** counts the failure events in + the window by name: `device.purge-failed`, `device.recovery-failed`, + `component.install-failed`, and any other event whose name ends in + `-failed`. The row is left out when there were none. - A requester that is a token shows the token's label beside its id. The window is rounded down to whole steps of the time series `--json` carries, diff --git a/docs/CLIENT.md b/docs/CLIENT.md index c94fe3ec..61a1d29d 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -547,8 +547,10 @@ const usage = await admin.usage({ from: Date.now() - 6 * 3_600_000, to: Date.now usage.totals.requests; // lease requests made in the window usage.totals.wait.p95; // milliseconds, or null when nothing waited +usage.totals.failures.byEvent; // a count per failure event, e.g. "device.purge-failed" usage.workers[0]?.label; // one entry for each worker; a worker lists itself -usage.series; // one point per bucket, for a chart +usage.series; // one point per bucket, for a chart; `waiting` counts every + // request open at the bucket's end ``` The daemon computes the figures from its event history, so they cover only what diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index 3daa2194..3ffae5eb 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -1948,12 +1948,15 @@ turns it into numbers. `computeUsage` (`src/core/usage/`) is pure: it takes the envelopes of a window, with the latest `capacity.changed` and `queue.changed` at or before the window's start that `EventHistory.read`'s `carry` adds, and the ids of the requests made before it (`requestedBefore`, so a rejection of -one is in no count), and returns the figures. `readEvents` reads them into one fact per request, joined +one is in no count), and the latest `lease.requested`, `lease.granted` and +`lease.rejected` of each requester at or before it (so a request still waiting when +the window opens counts in the series' `waiting`), and returns the figures. `readEvents` reads them into one fact per request, joined by `requestId` and `leaseId`, and one per device event; `compute-usage.ts` adds them up by platform, by worker and by requester. On a gateway (`fleet`) request facts come from its own events and device facts from the events its workers -relayed, and a request is joined to its worker's grant or rejection through -`request.dispatched`; that rule is in `readEvents` and nowhere else. One +relayed, and a request's outcome is the first relayed grant or rejection for its +namespaced requester before the requester's next request, the worker being the one +`request.dispatched` names, else the one that relayed it; that rule is in `readEvents` and nowhere else. One `UsageReader` serves both dispatchers: it rounds the window down to the series bucket at both ends, keeps its last answer by that window and the newest event id, and joins token labels, so the daemon's two handlers differ only in `fleet`. `simlock stats` diff --git a/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md b/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md index b16332ce..47bfec8a 100644 --- a/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md +++ b/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md @@ -137,19 +137,18 @@ grants, held time, provisioning, boots, capacity and device incidents, each attributed to its worker. A relayed `lease.requested`, `lease.queued` or `queue.changed` is not counted; the gateway forwards every request as `noWait`, so a worker's queue never holds a fleet request. A relayed -`lease.rejected` counts only when it settles a dispatched request: it comes -from the worker that request went to, for its namespaced requester, at or -after the dispatch, and carries the worker's reason. The gateway emits no -rejection of its own for a request that failed on its worker. - -A fleet request is joined to its outcome through `request.dispatched`: the -gateway's request id names the worker and the requester, and the first -relayed `lease.granted` or `lease.rejected` from that worker for the -namespaced requester at or after the dispatch, and before the gateway's -next `request.dispatched` or `lease.requested` for the same requester, is -that request's outcome. The core allows one open request per requester, so -within those bounds the match is unique, and a request with no outcome in -them is open. +`lease.rejected` counts only when it settles a request by the rule below. The +gateway emits no rejection of its own for a request that failed on its worker. + +A fleet request's outcome is the first relayed `lease.granted` or +`lease.rejected` for its namespaced requester at or after the gateway's own +`lease.requested` for it and before the gateway's next `lease.requested` for +that requester. `request.dispatched` names the worker when it exists, and +otherwise the relayed event's `workerId` does. A rejection carries the +worker's reason. The grant may arrive before `request.dispatched`: a warm +device is granted before the dispatch answer reaches the gateway. The core +allows one open request per requester, so within those bounds the match is +unique, and a request with no outcome in them is open. The answer has fleet totals and one entry per worker. A worker answers the same shape as a fleet of one (ADR 0012): totals, and one entry for itself. diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index 06b13ae5..cfe7e5b4 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -175,6 +175,31 @@ describe("simlock stats", () => { expect(table.stdout).toContain("Requests:"); }); + it("after a --no-wait refusal, requests and rejected.byReason.no-wait each rise by one and granted is unchanged", async () => { + const env = await withDaemon({ + configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }, + }); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + expect((await env.cli([...LEASE_ARGS, "--agent-id", "agent-a", "--detach"])).code).toBe(0); + const before = await settledStats(env); + + expect( + (await env.cli([...LEASE_ARGS, "--agent-id", "agent-b", "--no-wait", "--detach"])).code, + ).toBe(11); + const after = await settledStats( + env, + (usage) => usage.totals.requests > before.totals.requests, + ); + + expect(after.totals.requests).toBe(before.totals.requests + 1); + expect(after.totals.rejected.byReason["no-wait"]).toBe( + (before.totals.rejected.byReason["no-wait"] ?? 0) + 1, + ); + expect(after.totals.granted).toBe(before.totals.granted); + }); + it("simlock stats --since 1h returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { const env = await withDaemon(); await env.driverScript.set({ diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index 32b732f8..0bcb1e42 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -565,6 +565,39 @@ describe("readEventFile", () => { expect(await read(step("evt_5", 5, 200), step("evt_6", 5, 200))).toEqual(["evt_6"]); }); + it("readEventFile with carry keeps the latest of a named event for each requester, so a request still open when the window opens is carried", async () => { + const forRequester = ( + seq: number, + timestamp: number, + event: "lease.requested" | "lease.granted", + requester: string, + ): EventEnvelope => + ({ + event, + id: `evt_${seq}`, + module: "test", + payload: { requester }, + seq, + timestamp, + }) as unknown as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + forRequester(1, 100, "lease.requested", "a"), + forRequester(2, 110, "lease.requested", "b"), + forRequester(3, 120, "lease.granted", "a"), + forRequester(4, 130, "lease.requested", "a"), + forRequester(5, 400, "lease.requested", "c"), + ), + }); + + const read = await readEventFile(filesystem, "/data/events.jsonl", { + carry: ["lease.requested", "lease.granted"], + sinceTs: 300, + }); + + expect(read.map((entry) => entry.seq)).toEqual([2, 3, 4, 5]); + }); + it("readEventFile with carry returns the latest capacity.changed and queue.changed at or before sinceTs, one per worker id, and none when there is none", async () => { const step = ( seq: number, diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index bec8dae6..a7a157b3 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -9,7 +9,7 @@ export const EVENT_FILE_NAME = "events.jsonl"; /** * Every envelope in the event file newer than `sinceTs`, by `timestamp` then `seq` (ADR 0014 §5). * With `carry`, also the latest envelope of each named event at or before `sinceTs` -- one per - * `workerId` where the payload names one -- so a reader of a step function (ADR 0016 §3) is told + * `workerId` and one per `requester` where the payload names one -- so a reader of a step function (ADR 0016 §3) is told * the step in force when its window opens. * The current file is read first, then its generations `.1`, `.2` and so on until * one is missing (ADR 0016 §4). Newest first means a rotation landing mid-read can only make a @@ -81,8 +81,9 @@ function noteRequest(requestedBefore: Set, envelope: EventEnvelope): voi } /** - * Keeps `envelope` as the step in force for its event and worker when it is a named event and no - * later one is kept already. Ties on time and seq go to the one met last, the one written last. + * Keeps `envelope` as the step in force for its event, worker and requester when it is a named + * event and no later one is kept already. Ties on time and seq go to the one met last, the one + * written last. */ function keepIfCarried( carried: Map, @@ -90,8 +91,13 @@ function keepIfCarried( carry: readonly EventName[], ): void { if (!carry.includes(envelope.event)) return; - const workerId = (envelope.payload as { readonly workerId?: unknown }).workerId; - const key = `${envelope.event}\u0000${typeof workerId === "string" ? workerId : ""}`; + const { workerId, requester } = envelope.payload as { + readonly workerId?: unknown; + readonly requester?: unknown; + }; + const key = [envelope.event, workerId, requester] + .map((part) => (typeof part === "string" ? part : "")) + .join("\u0000"); const kept = carried.get(key); if (kept === undefined || byTimeThenSeq(kept, envelope) <= 0) carried.set(key, envelope); } diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index acad4ac9..122e0904 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -5380,7 +5380,7 @@ function usageAnswer( const figures = { boot: samples, bySource: { booted: 0, provisioned: 0, warm: 0 }, - errors: { byCode: {} }, + failures: { byEvent: {} }, granted: 0, held: samples, incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, diff --git a/src/cli/stats.test.ts b/src/cli/stats.test.ts index 7e0fae4a..613dd1b7 100644 --- a/src/cli/stats.test.ts +++ b/src/cli/stats.test.ts @@ -9,7 +9,7 @@ function figuresFixture(overrides: Partial = {}): UsageFigures { return { boot: NONE, bySource: { booted: 0, provisioned: 0, warm: 0 }, - errors: { byCode: {} }, + failures: { byEvent: {} }, granted: 0, held: NONE, incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, @@ -45,7 +45,7 @@ function usageFixture(overrides: Partial = {}): UsageOutput { const BUSY = figuresFixture({ boot: { count: 2, max: 40_000, p50: 20_000, p95: 40_000 }, bySource: { booted: 3, provisioned: 1, warm: 6 }, - errors: { byCode: { "device.purge-failed": 2 } }, + failures: { byEvent: { "device.purge-failed": 2 } }, granted: 10, held: { count: 9, max: 3_725_000, p50: 300_000, p95: 1_800_000 }, incidents: { crashRecovered: 1, lost: 4, quarantineRecovered: 3, quarantined: 2 }, @@ -93,7 +93,7 @@ describe("formatUsage", () => { expect(lines).toContain( " Incidents: 2 quarantined, 1 recovered after a crash, 3 recovered from quarantine, 4 lost", ); - expect(lines).toContain(" Errors: device.purge-failed 2"); + expect(lines).toContain(" Failures: device.purge-failed 2"); }); it("says plainly what it has no figure for, and leaves out rows it has nothing to say in", () => { @@ -104,7 +104,7 @@ describe("formatUsage", () => { expect(text).toContain(" Slots: not known"); expect(text).toContain(" Queue: not known"); expect(text).not.toContain("RAM:"); - expect(text).not.toContain("Errors:"); + expect(text).not.toContain("Failures:"); expect(text).not.toContain("Rejected:"); expect(text).not.toContain("Granted:"); expect(text).not.toContain("Workers"); @@ -219,24 +219,24 @@ describe("formatUsage rows that depend on the figures", () => { const lines = formatUsage( usageFixture({ totals: figuresFixture({ - errors: { byCode: { "c.zed": 1, "a.alpha": 2, "b.mid": 3 } }, + failures: { byEvent: { "c.zed": 1, "a.alpha": 2, "b.mid": 3 } }, rejected: { byReason: { timeout: 1, "no-wait": 2, capacity: 3 }, total: 6 }, }), }), ).split("\n"); - expect(lines).toContain(" Errors: a.alpha 2, b.mid 3, c.zed 1"); + expect(lines).toContain(" Failures: a.alpha 2, b.mid 3, c.zed 1"); expect(lines).toContain(" Rejected: capacity 3, no-wait 2, timeout 1"); }); - it("prints the Errors row only when there are error codes", () => { + it("prints the Failures row only when there are failure events", () => { const without = formatUsage(usageFixture()); const withOne = formatUsage( - usageFixture({ totals: figuresFixture({ errors: { byCode: { "x.y": 1 } } }) }), + usageFixture({ totals: figuresFixture({ failures: { byEvent: { "x.y": 1 } } }) }), ); - expect(without.split("\n").some((line) => line.startsWith(" Errors:"))).toBe(false); - expect(withOne.split("\n")).toContain(" Errors: x.y 1"); + expect(without.split("\n").some((line) => line.startsWith(" Failures:"))).toBe(false); + expect(withOne.split("\n")).toContain(" Failures: x.y 1"); }); it("prints the Granted row only when something was granted", () => { diff --git a/src/cli/stats.ts b/src/cli/stats.ts index 3f9d56df..29849241 100644 --- a/src/cli/stats.ts +++ b/src/cli/stats.ts @@ -91,9 +91,9 @@ function totalRows(totals: UsageFigures): string[] { `${totals.incidents.quarantined} quarantined, ${totals.incidents.crashRecovered} recovered after a crash, ` + `${totals.incidents.quarantineRecovered} recovered from quarantine, ${totals.incidents.lost} lost`, ), - ...(Object.keys(totals.errors.byCode).length === 0 + ...(Object.keys(totals.failures.byEvent).length === 0 ? [] - : [row("Errors:", list(totals.errors.byCode))]), + : [row("Failures:", list(totals.failures.byEvent))]), ]; } diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 7fe77ec2..1bd89ea7 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -983,7 +983,7 @@ describe("usage.get contract details", () => { const figures = { boot: samples, bySource: { booted: 0, provisioned: 0, warm: 0 }, - errors: { byCode: {} }, + failures: { byEvent: {} }, granted: 0, held: samples, incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 3199e0d5..62b98899 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -1064,7 +1064,7 @@ const usageFiguresSchema = z.object({ warm: z.number().int().nonnegative(), }), boot: usageSamplesSchema, - errors: z.object({ byCode: usageCountsSchema }), + failures: z.object({ byEvent: usageCountsSchema }), granted: z.number().int().nonnegative(), held: usageSamplesSchema, incidents: z.object({ diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index 4a185538..8f1a3568 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -562,25 +562,21 @@ describe("computeUsage", () => { }); it("with fleet: true settles a dispatched request as rejected, with the worker's reason, from the relayed lease.rejected of the worker it was dispatched to", () => { - const dispatched = (workerId: string) => - at(T0 + 2_000, "request.dispatched", { - model: "m", - platform: "ios", - queuedMs: 0, - reason: "free-capacity", - requestId: "gw-r1", - requesterId: "agent-a", - stage: "s", - workerId, - }); - const base = [requested(T0 + 1_000, "gw-r1")]; - const usage = computeUsage( [ - ...base, - dispatched("w1"), - // Another worker's rejection for the same requester is not this request's outcome. - rejected(T0 + 2_500, "w2-r", "no-wait", "gw:g1:agent-a", { workerId: "w2" }), + requested(T0 + 1_000, "gw-r1"), + at(T0 + 2_000, "request.dispatched", { + model: "m", + platform: "ios", + queuedMs: 0, + reason: "free-capacity", + requestId: "gw-r1", + requesterId: "agent-a", + stage: "s", + workerId: "w1", + }), + // Another requester's rejection is not this request's outcome. + rejected(T0 + 2_500, "w1-x", "no-wait", "gw:g1:agent-b", { workerId: "w1" }), rejected(T0 + 3_000, "w1-r", "boot-timeout", "gw:g1:agent-a", { workerId: "w1" }), ], WINDOW, @@ -649,6 +645,39 @@ describe("computeUsage", () => { expect(usage.totals.wait.p50).toBe(400); }); + it("with fleet: true does not credit a requester's next request with a grant that came before it", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + // The grant for the first request reaches the gateway ahead of its dispatch, and the + // requester then asks again and nothing answers the second request. + granted(T0 + 1_400, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), + requested(T0 + 10_000, "gw-r2"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(2); + expect(usage.totals.granted).toBe(1); + expect(usage.totals.wait).toEqual({ count: 1, max: 400, p50: 400, p95: 400 }); + }); + + it("with fleet: true names the worker of the relayed grant when the request has no request.dispatched", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gw-r1"), + granted(T0 + 1_400, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w7" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.granted).toBe(1); + expect(usage.workers).toHaveLength(1); + expect(usage.workers[0]).toMatchObject({ granted: 1, id: "w7", requests: 1 }); + }); + it("sets partial and coversFrom when the oldest event is inside the window", () => { const partial = computeUsage( [requested(T0 + 5 * MINUTE, "r1"), requested(T0 + 9 * MINUTE, "r2", "b")], @@ -730,7 +759,7 @@ describe("computeUsage", () => { }); }); - it("counts failures that carry no reason of their own by event name under errors.byCode", () => { + it("counts every `*-failed` event in the window by its event name under failures.byEvent, by the event's timestamp", () => { const usage = computeUsage( [ at(T0 + 1_000, "device.purge-failed", { @@ -753,15 +782,22 @@ describe("computeUsage", () => { error: "x", platform: "ios", }), + at(T0 + 4_000, "device.recovery-failed", { deviceId: "d3", leaseId: "l" }), + at(T0 + 5_000, "device.some-new-failed" as EventName, { deviceId: "d3" }), + at(WINDOW.from - 1, "device.purge-failed", { deviceId: "d9" }), + at(WINDOW.to + 1, "device.purge-failed", { deviceId: "d9" }), ], WINDOW, WORKER, ); - expect(usage.totals.errors.byCode).toEqual({ + expect(usage.totals.failures.byEvent).toEqual({ "component.install-failed": 1, "device.purge-failed": 2, + "device.recovery-failed": 1, + "device.some-new-failed": 1, }); + expect(usage.totals.incidents.lost).toBe(1); }); it("sorts events supplied out of order and leaves the caller's array exactly as it was", () => { @@ -829,7 +865,7 @@ describe("computeUsage", () => { expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 0, warm: 1 }); }); - it("keeps durations and incidents out of errors.byCode when device events of every kind are present", () => { + it("keeps durations and incidents out of failures.byEvent when device events of every kind are present", () => { const usage = computeUsage( [ at(T0 + 1_000, "device.provisioned", { @@ -852,7 +888,7 @@ describe("computeUsage", () => { WORKER, ); - expect(usage.totals.errors.byCode).toEqual({ "device.purge-failed": 1 }); + expect(usage.totals.failures.byEvent).toEqual({ "device.purge-failed": 1 }); expect(usage.totals.incidents.quarantined).toBe(1); expect(usage.totals.boot.count).toBe(1); expect(usage.totals.provisioning.count).toBe(1); @@ -971,6 +1007,40 @@ describe("computeUsage", () => { expect(fleet.workers[0]?.queue).toEqual({ meanDepth: null, peakDepth: null }); }); + it("counts a request made before the window and still waiting at its start in the series' waiting, though in no total", () => { + const usage = computeUsage( + [ + requested(T0 - 30_000, "old", "agent-a"), + // Answered before the window opened: not waiting at its start. + requested(T0 - 40_000, "done", "agent-b"), + granted(T0 - 35_000, "done", "l0", "warm", "agent-b"), + queueChanged(T0 - 30_000, 1), + granted(T0 + 90_000, "old", "l1"), + ], + WINDOW, + WORKER, + ); + + expect(usage.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 0, 0]); + expect(usage.series[0]?.queueDepth).toBe(1); + expect(usage.totals.requests).toBe(0); + expect(usage.totals.granted).toBe(0); + }); + + it("with fleet: true counts a request made before the window until a worker's relayed answer for its requester", () => { + const usage = computeUsage( + [ + requested(T0 - 30_000, "gw-old", "agent-a"), + granted(T0 + 90_000, "w-1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 0, 0]); + expect(usage.totals.granted).toBe(0); + }); + it("counts a request as waiting from the instant it is made until the instant it settles, per series point", () => { const usage = computeUsage( [ diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts index 507fdfef..f00a4627 100644 --- a/src/core/usage/compute-usage.ts +++ b/src/core/usage/compute-usage.ts @@ -154,7 +154,7 @@ function sourcesOf(grantedSources: readonly string[]): UsageFigures["bySource"] /** The figures a device event brings: durations, incidents, and failures by name. */ function deviceFigures( devices: readonly DeviceFact[], -): Pick { +): Pick { const durations = (kind: DeviceFact["kind"]) => summarise(devices.filter((fact) => fact.kind === kind).map((fact) => fact.value as number)); const named = (kind: DeviceFact["kind"]) => @@ -162,7 +162,7 @@ function deviceFigures( const incidents = named("incident"); return { boot: durations("boot"), - errors: { byCode: named("error") }, + failures: { byEvent: named("failure") }, incidents: { crashRecovered: incidents.crashRecovered ?? 0, lost: incidents.lost ?? 0, @@ -283,7 +283,7 @@ function seriesOf(read: ReadEvents, window: UsageWindow, bucketMs: number): Usag ); const capacity = valuesAt(scopedSteps(read.capacity, {}), ends); const depths = valuesAt(read.queue, ends); - const waiting = waitingAt(read.requests, ends); + const waiting = waitingAt([...read.requests, ...read.carried], ends); return ends.map((at, index) => { const values = capacity[index] as readonly CapacityValue[]; const ram = ramOf(values, "used"); @@ -298,7 +298,10 @@ function seriesOf(read: ReadEvents, window: UsageWindow, bucketMs: number): Usag }); } -/** How many requests have been made and not yet settled at each of `times`, which ascend. */ +/** + * How many requests have been made and not yet settled at each of `times`, which ascend: every + * request open at that time, those made before the window and still open when it began included. + */ function waitingAt(requests: readonly RequestFact[], times: readonly number[]): number[] { const ascending = (values: number[]) => values.sort((left, right) => left - right); const made = ascending(requests.flatMap((fact) => fact.requestedAt ?? [])); diff --git a/src/core/usage/read-events.test.ts b/src/core/usage/read-events.test.ts index d03aa75a..ffc9463f 100644 --- a/src/core/usage/read-events.test.ts +++ b/src/core/usage/read-events.test.ts @@ -508,7 +508,7 @@ describe("device facts", () => { }); }); - it("counts each incident event under the name the figure uses, and each error under its event name", () => { + it("counts each incident event under the name the figure uses, and each `*-failed` event under its event name, a recovery failure under both", () => { const names: Record = { "device.quarantine-abandoned": "lost", "device.quarantine-recovered": "quarantineRecovered", @@ -518,17 +518,23 @@ describe("device facts", () => { "device.recovery-failed": "lost", }; const incidents = Object.keys(names).map((name, i) => at(110 + i, name, {})); - const errors = [at(130, "device.purge-failed", {}), at(131, "component.install-failed", {})]; + const failures = [ + at(130, "device.purge-failed", {}), + at(131, "component.install-failed", {}), + at(132, "device.some-new-failed", {}), + ]; const result = read([ ...incidents, - ...errors, - at(132, "device.created", {}), - at(133, "lease.other", {}), + ...failures, + at(133, "device.created", {}), + at(134, "lease.other", {}), ]); expect(result.devices.map((d) => [d.kind, d.value])).toEqual([ ...Object.values(names).map((v) => ["incident", v]), - ["error", "device.purge-failed"], - ["error", "component.install-failed"], + ["failure", "device.recovery-failed"], + ["failure", "device.purge-failed"], + ["failure", "component.install-failed"], + ["failure", "device.some-new-failed"], ]); }); @@ -553,6 +559,51 @@ describe("device facts", () => { }); }); +describe("requests carried into the window", () => { + const grant = (ts: number, requestId: string, requester = "a") => + at(ts, "lease.granted", { leaseId: `L${requestId}`, requestId, requester, source: "warm" }); + + it("carries a request made before the window that nothing had answered, with its outcome inside it", () => { + const result = read([request(90, "old"), grant(150, "old")]); + expect(result.requests).toEqual([]); + expect(result.carried).toHaveLength(1); + expect(result.carried[0]).toMatchObject({ + outcome: { at: 150, kind: "granted", source: "warm" }, + requestedAt: 90, + requester: "a", + worker: "self", + }); + }); + + it("does not carry a request answered before the window, or one made inside it", () => { + const result = read([ + request(80, "done", "b"), + grant(90, "done", "b"), + request(85, "refused", "c"), + at(95, "lease.rejected", { reason: "timeout", requestId: "refused", requester: "c" }), + request(110, "inside"), + ]); + expect(result.carried).toEqual([]); + expect(result.requests).toHaveLength(1); + }); + + it("carries only a requester's latest request made before the window", () => { + const result = read([request(80, "first"), grant(85, "first"), request(90, "second")]); + expect(result.carried.map((fact) => fact.requestedAt)).toEqual([90]); + }); + + it("carries a gateway request answered only by a relayed answer inside the window, matched by the namespaced requester", () => { + const answered = read([request(90, "old"), relayedGrant(150)], FLEET); + expect(answered.carried).toEqual([ + expect.objectContaining({ outcome: expect.objectContaining({ at: 150 }), worker: "w1" }), + ]); + const before = read([request(80, "old"), relayedGrant(90), request(95, "later")], FLEET); + expect(before.carried.map((fact) => fact.requestedAt)).toEqual([95]); + const first = read([request(80, "old"), relayedGrant(90)], FLEET); + expect(first.carried).toEqual([]); + }); +}); + describe("a gateway", () => { it("counts a relayed device fact against its worker and ignores one with no worker", () => { const result = read( @@ -613,14 +664,34 @@ describe("a gateway", () => { expect(read(events("w1", "M"), FLEET).requests[0]).not.toHaveProperty("endedAt"); }); - it("gives a request with no dispatch no worker and no outcome, and does not count its grant", () => { - const result = read([request(110, "r"), relayedGrant(112)], FLEET); + it("gives a request with no dispatch and no relayed answer no worker and no outcome", () => { + const result = read([request(110, "r")], FLEET); expect(result.requests).toEqual([ { platform: "ios", requestedAt: 110, requester: "a", worker: undefined }, ]); expect([...result.workers]).toEqual([]); }); + it("takes the worker of the relayed answer for a request with no dispatch", () => { + const grant = read([request(110, "r"), relayedGrant(112, { workerId: "w5" })], FLEET); + expect(grant.requests[0]).toMatchObject({ + outcome: { kind: "granted" }, + requestedAt: 110, + worker: "w5", + }); + const rejection = read([request(110, "r"), relayedReject(112, { workerId: "w6" })], FLEET); + expect(rejection.requests[0]).toMatchObject({ outcome: { kind: "rejected" }, worker: "w6" }); + expect([...grant.workers]).toEqual(["w5"]); + }); + + it("names the worker the request was dispatched to, not the one that relayed the answer", () => { + const result = read( + [request(110, "r"), dispatch(111, "r", "a", "w1"), relayedGrant(112, { workerId: "w2" })], + FLEET, + ); + expect(result.requests[0]?.worker).toBe("w1"); + }); + it("serves a request through its dispatch, and records the worker it was dispatched to", () => { const result = read([request(110, "r"), dispatch(111, "r", "a", "w9")], FLEET); expect(result.requests).toEqual([ @@ -707,12 +778,11 @@ describe("a gateway", () => { expect(result.requests[0]).not.toHaveProperty("outcome"); }); - it("ignores a relayed grant of another worker or another requester", () => { + it("ignores a relayed grant of another requester, or of one without the gateway's prefix", () => { const result = read( [ request(110, "r"), dispatch(111, "r"), - relayedGrant(112, { workerId: "w2" }), relayedGrant(113, { requester: "gw:b" }), relayedGrant(114, { requester: "a" }), ], @@ -734,11 +804,12 @@ describe("a gateway", () => { expect(result.requests[0]?.outcome).toMatchObject({ source: "first" }); }); - it("lets a grant that came before the dispatch count, but not a rejection", () => { + it("counts a grant or a rejection that came before the dispatch", () => { const grant = read([request(110, "r"), relayedGrant(111), dispatch(112, "r")], FLEET); expect(grant.requests[0]?.outcome).toMatchObject({ kind: "granted" }); + expect(grant.requests[0]?.worker).toBe("w1"); const rejection = read([request(110, "r"), relayedReject(111), dispatch(112, "r")], FLEET); - expect(rejection.requests[0]).not.toHaveProperty("outcome"); + expect(rejection.requests[0]?.outcome).toMatchObject({ kind: "rejected" }); }); it("does not let a grant from before the request count", () => { @@ -767,12 +838,12 @@ describe("a gateway", () => { expect(second?.outcome).toMatchObject({ source: "late" }); }); - it("cuts off at a next dispatch of the requester even when its request was not seen", () => { + it("does not cut off at a next dispatch of the requester, only at its next request", () => { const result = read( [request(110, "r1"), dispatch(111, "r1"), dispatch(112, "r2"), relayedGrant(113)], FLEET, ); - expect(result.requests[0]).not.toHaveProperty("outcome"); + expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); }); it("does not cut off at another requester's request", () => { diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index 41e51c7d..b2e2356e 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -37,10 +37,10 @@ interface Rejected { } export interface DeviceFact { - readonly kind: "provisioning" | "boot" | "incident" | "error"; + readonly kind: "provisioning" | "boot" | "incident" | "failure"; readonly worker: string | undefined; readonly platform: Platform | undefined; - /** The duration for the first two, the incident or error name for the others. */ + /** The duration for the first two, the incident name or the failure event name for the others. */ readonly value: number | string; } @@ -56,6 +56,8 @@ export type CapacityStep = Step< export interface ReadEvents { readonly requests: readonly RequestFact[]; + /** Requests made before the window and still open when it began; only the series counts them. */ + readonly carried: readonly RequestFact[]; readonly devices: readonly DeviceFact[]; readonly capacity: readonly CapacityStep[]; readonly queue: readonly Step[]; @@ -94,8 +96,8 @@ const INCIDENT_OF: Readonly> = { "device.recovered": "crashRecovered", "device.recovery-failed": "lost", }; -/** Failures that carry no reason of their own among the rejections, counted by their event name. */ -const ERROR_EVENTS = new Set(["device.purge-failed", "component.install-failed"]); +/** A failure is any event named `*-failed`, counted by its name. */ +const isFailure = (event: string): boolean => event.endsWith("-failed"); const text = (payload: Payload, key: string): string | undefined => { const value = payload[key]; @@ -117,7 +119,11 @@ const objectOf = (value: unknown): Payload | undefined => type RelayedAnswer = | { readonly kind: "granted"; readonly source: string; readonly leaseId: string } | { readonly kind: "rejected"; readonly reason: string }; -type Relayed = { readonly index: number; readonly at: number } & RelayedAnswer; +type Relayed = { + readonly index: number; + readonly at: number; + readonly worker: string; +} & RelayedAnswer; interface Dispatch { readonly index: number; @@ -162,6 +168,9 @@ class Reader { readonly #workers = new Set(); readonly #devicePlatform = new Map(); readonly #requested = new Map(); + /** The latest request of each requester made before the window, and where each was answered. */ + readonly #carriedRequests = new Map(); + readonly #answeredBefore = new Map(); readonly #ownRejections = new Map< string, Omit & { @@ -173,6 +182,7 @@ class Reader { readonly #ends = new Map(); readonly #dispatches = new Map(); readonly #boundaries = new Map(); + /** Relayed grants and rejections by the namespaced requester they are for. */ readonly #relayed = new Map(); readonly #handlers: Record void> = { "capacity.changed": (seen) => this.#capacityChanged(seen), @@ -211,6 +221,7 @@ class Reader { capacity: this.#capacity, devices: this.#devices, queue: this.#queue, + carried: this.#carriedOpen(), requests: [...this.#requests(), ...this.#refusedAtAdmission()], workers: this.#workers, }; @@ -253,7 +264,17 @@ class Reader { const requestId = text(seen.payload, "requestId"); const requester = text(seen.payload, "requester"); this.#boundary(requester, seen.index); - if (!seen.within || requestId === undefined || requester === undefined) return; + if (requestId === undefined || requester === undefined) return; + if (seen.event.timestamp <= this.window.from) { + this.#carriedRequests.set(requester, { + at: seen.event.timestamp, + index: seen.index, + platform: requestPlatform(seen.payload), + requestId, + requester, + }); + } + if (!seen.within) return; this.#requested.set(requestId, { at: seen.event.timestamp, index: seen.index, @@ -266,7 +287,6 @@ class Reader { const requestId = text(seen.payload, "requestId"); const requester = text(seen.payload, "requesterId"); const worker = text(seen.payload, "workerId"); - this.#boundary(requester, seen.index); if (!this.options.fleet || !seen.within) return; if (requestId === undefined || requester === undefined || worker === undefined) return; this.#dispatches.set(requestId, { index: seen.index, requestId, requester, worker }); @@ -275,14 +295,19 @@ class Reader { #relay(seen: Seen, relayed: RelayedAnswer): void { const requester = text(seen.payload, "requester"); if (requester === undefined || seen.worker === undefined) return; - const key = this.#key(seen.worker, requester); - const entry = { ...relayed, at: seen.event.timestamp, index: seen.index } as Relayed; - this.#relayed.set(key, [...(this.#relayed.get(key) ?? []), entry]); + const entry = { + ...relayed, + at: seen.event.timestamp, + index: seen.index, + worker: seen.worker, + } as Relayed; + this.#relayed.set(requester, [...(this.#relayed.get(requester) ?? []), entry]); } #rejected(seen: Seen): void { const requestId = text(seen.payload, "requestId"); const reason = text(seen.payload, "reason"); + if (this.#answeredEarlier(seen)) return; if (!seen.within || requestId === undefined || reason === undefined) return; const requester = text(seen.payload, "requester"); if (seen.own) { @@ -298,8 +323,16 @@ class Reader { } } + /** Notes where a requester was last answered before the window; true for such an answer. */ + #answeredEarlier(seen: Seen): boolean { + if (seen.event.timestamp > this.window.from) return false; + const requester = text(seen.payload, "requester"); + if (requester !== undefined) this.#answeredBefore.set(requester, seen.index); + return true; + } + #granted(seen: Seen): void { - if (!seen.within) return; + if (this.#answeredEarlier(seen) || !seen.within) return; if (this.options.fleet) { this.#relay(seen, { kind: "granted", @@ -350,16 +383,14 @@ class Reader { #deviceEvent(seen: Seen): void { const incident = INCIDENT_OF[seen.event.event]; - if (!this.#counts(seen) || (incident === undefined && !ERROR_EVENTS.has(seen.event.event))) - return; + const failure = isFailure(seen.event.event); + if (!this.#counts(seen) || (incident === undefined && !failure)) return; const platform = platformOf(seen.payload.platform) ?? this.#platformOfDevice(seen.worker, text(seen.payload, "deviceId")); - this.#addDevice(seen, { - kind: incident === undefined ? "error" : "incident", - platform, - value: incident ?? seen.event.event, - }); + if (incident !== undefined) + this.#addDevice(seen, { kind: "incident", platform, value: incident }); + if (failure) this.#addDevice(seen, { kind: "failure", platform, value: seen.event.event }); } #platformOfDevice(worker: string, deviceId: string | undefined): Platform | undefined { @@ -384,6 +415,18 @@ class Reader { return facts; } + /** The requests made before the window that nothing had answered when it began. */ + #carriedOpen(): RequestFact[] { + const facts: RequestFact[] = []; + for (const [requester, request] of this.#carriedRequests) { + const answered = [requester, `${this.options.requesterPrefix}${requester}`].some( + (key) => (this.#answeredBefore.get(key) ?? -1) > request.index, + ); + if (!answered) facts.push(this.#factFor(request.requestId, request)); + } + return facts; + } + #factFor(requestId: string, request: Request): RequestFact { const rejection = this.#ownRejections.get(requestId); const own = this.options.fleet ? undefined : this.options.self; @@ -408,20 +451,19 @@ class Reader { : this.#withEnd({ ...base, outcome: grant, worker: own }, grant.leaseId, own); } - /** A fleet request's outcome is the worker's, found through its dispatch (ADR 0016 §6). */ + /** A fleet request's outcome is the worker's relayed answer for its requester (ADR 0016 §6). */ #fleetFact(base: RequestFactBase, requestId: string, request: Request): RequestFact { - const dispatch = this.#dispatches.get(requestId); - if (dispatch === undefined) return { ...base, worker: undefined }; - this.#workers.add(dispatch.worker); - const worker = dispatch.worker; - const relayed = this.#fleetOutcome(dispatch, request.index); + const dispatched = this.#dispatches.get(requestId)?.worker; + const relayed = this.#fleetOutcome(request); + const worker = dispatched ?? relayed?.worker; + if (worker !== undefined) this.#workers.add(worker); if (relayed === undefined) return { ...base, worker }; if (relayed.kind === "rejected") { const outcome: Rejected = { at: relayed.at, kind: "rejected", reason: relayed.reason }; return { ...base, outcome, worker }; } const outcome: Granted = { at: relayed.at, kind: "granted", source: relayed.source }; - return this.#withEnd({ ...base, outcome, worker }, relayed.leaseId, worker); + return this.#withEnd({ ...base, outcome, worker }, relayed.leaseId, relayed.worker); } #withEnd(fact: RequestFact, leaseId: string, worker: string): RequestFact { @@ -430,19 +472,16 @@ class Reader { } /** - * The first relayed grant or rejection from the worker a request was dispatched to, for its - * namespaced requester, that falls before the requester's next request (ADR 0016 §6). A rejection - * must come at or after the dispatch. A grant may come before it: a warm device is granted - * before the answer that announces the dispatch reaches the gateway, so a grant counts from the - * request. + * The first relayed grant or rejection for the request's namespaced requester at or after the + * gateway's own `lease.requested` for it and before that requester's next one (ADR 0016 §6). A + * grant may come before `request.dispatched`: a warm device is granted before the dispatch + * answer reaches the gateway. */ - #fleetOutcome(dispatch: Dispatch, requestIndex: number): Relayed | undefined { - const next = this.#boundariesOf(dispatch.requester).find((index) => index > dispatch.index); - const key = this.#key(dispatch.worker, `${this.options.requesterPrefix}${dispatch.requester}`); - return (this.#relayed.get(key) ?? []).find( + #fleetOutcome(request: Request): Relayed | undefined { + const next = this.#boundariesOf(request.requester).find((index) => index > request.index); + return (this.#relayed.get(`${this.options.requesterPrefix}${request.requester}`) ?? []).find( (candidate) => - candidate.index >= (candidate.kind === "granted" ? requestIndex : dispatch.index) && - (next === undefined || candidate.index < next), + candidate.index >= request.index && (next === undefined || candidate.index < next), ); } diff --git a/src/core/usage/usage-reader.test.ts b/src/core/usage/usage-reader.test.ts index e4a2413c..6bdb3069 100644 --- a/src/core/usage/usage-reader.test.ts +++ b/src/core/usage/usage-reader.test.ts @@ -63,7 +63,7 @@ const answer = (result: Awaited>) => { }; describe("UsageReader", () => { - it("rounds the window down to the bucket on both sides, so it never ends in the future, and reads the history from its start, with the two step events carried", async () => { + it("rounds the window down to the bucket on both sides, so it never ends in the future, and reads the history from its start, with the step and the request events carried", async () => { const { source, state } = history(); const { usage } = reader(source); @@ -71,7 +71,18 @@ describe("UsageReader", () => { expect(result.window).toEqual({ from: T0, to: T0 + HOUR }); expect(result.bucketMs).toBe(MINUTE); - expect(state.asked).toEqual([{ carry: ["capacity.changed", "queue.changed"], sinceTs: T0 }]); + expect(state.asked).toEqual([ + { + carry: [ + "capacity.changed", + "lease.granted", + "lease.rejected", + "lease.requested", + "queue.changed", + ], + sinceTs: T0, + }, + ]); }); it("leaves a window that is already on bucket edges as it is", async () => { diff --git a/src/core/usage/usage-reader.ts b/src/core/usage/usage-reader.ts index 4543c195..7076dc6e 100644 --- a/src/core/usage/usage-reader.ts +++ b/src/core/usage/usage-reader.ts @@ -3,7 +3,15 @@ import type { UsageOutput } from "../../contract/index.js"; import { computeUsage, seriesBucketMs, type UsageWindow } from "./compute-usage.js"; /** The two timelines whose step in force at a window's start is read with it (ADR 0016 §3). */ -const STEP_EVENTS: readonly EventName[] = ["capacity.changed", "queue.changed"]; +/** The events whose latest envelope before the window the figures need: the two steps, and the + * latest request and answer of each requester, which tell what was still waiting at its start. */ +const STEP_EVENTS: readonly EventName[] = [ + "capacity.changed", + "lease.granted", + "lease.rejected", + "lease.requested", + "queue.changed", +]; /** The part of the event history `usage.get` reads (`EventHistory` satisfies it). */ export interface UsageHistory { diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index 62dbc4f5..da5f128e 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -231,7 +231,7 @@ describe("connectSimlock: handshake", () => { const figures = { boot: none, bySource: { booted: 0, provisioned: 0, warm: 0 }, - errors: { byCode: {} }, + failures: { byEvent: {} }, granted: 0, held: none, incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, From 3858185b2f3ef5aea4bb7fa4338ff2187792ed41 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 20:29:57 +0200 Subject: [PATCH 13/23] test: kill the carry and join mutants; a dispatch names only its worker --- src/bus/event-file.test.ts | 42 ++++++++++++++++++++++++++++++ src/core/usage/read-events.test.ts | 33 +++++++++++++++++++---- src/core/usage/read-events.ts | 17 ++++-------- 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index 0bcb1e42..9de4a5f0 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -422,6 +422,48 @@ describe("readEventFile", () => { expect([...read.requestedBefore].sort()).toEqual(["edge", "old"]); }); + it("reports no requestId from a lease.requested that has none, or from another event that has one", async () => { + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + { ...envelope(1, 100), event: "lease.requested", payload: {} } as EventEnvelope, + { + ...envelope(2, 110), + event: "lease.granted", + payload: { requestId: "g" }, + } as EventEnvelope, + ), + }); + + const read = await readEventHistory(filesystem, "/data/events.jsonl", { sinceTs: 200 }); + + expect([...read.requestedBefore]).toEqual([]); + }); + + it("carries events whose worker and requester run together into the same text as two different ones", async () => { + const withPayload = (seq: number, payload: Record): EventEnvelope => + ({ + event: "lease.requested", + id: `evt_${seq}`, + module: "test", + payload, + seq, + timestamp: 100 + seq, + }) as unknown as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + withPayload(1, { requester: "w1x" }), + withPayload(2, { requester: "x", workerId: "w1" }), + ), + }); + + const read = await readEventFile(filesystem, "/data/events.jsonl", { + carry: ["lease.requested"], + sinceTs: 300, + }); + + expect(read.map((entry) => entry.seq)).toEqual([1, 2]); + }); + it("reports the requests made before sinceTs from the ring too", async () => { const clock = new FakeClock(1_000); const bus = new EventBus(clock); diff --git a/src/core/usage/read-events.test.ts b/src/core/usage/read-events.test.ts index ffc9463f..93a568aa 100644 --- a/src/core/usage/read-events.test.ts +++ b/src/core/usage/read-events.test.ts @@ -242,6 +242,22 @@ describe("queue steps", () => { }); describe("requests on a worker", () => { + it("counts a grant with no source as an empty source, and takes no grant without a leaseId", () => { + const result = read([ + request(110, "r1"), + at(120, "lease.granted", { leaseId: "L1", requestId: "r1", requester: "a" }), + request(130, "r2", "b"), + at(140, "lease.granted", { requestId: "r2", requester: "b", source: "warm" }), + ]); + expect(result.requests[0]?.outcome).toEqual({ + at: 120, + kind: "granted", + leaseId: "L1", + source: "", + }); + expect(result.requests[1]).not.toHaveProperty("outcome"); + }); + it("makes no request of an event at exactly the window start and one of an event at its end", () => { const result = read([request(100, "r0"), request(200, "r1"), request(201, "r2")]); expect(result.requests.map((r) => r.requestedAt)).toEqual([200]); @@ -560,6 +576,15 @@ describe("device facts", () => { }); describe("requests carried into the window", () => { + it("carries a request made exactly at the window's start, and treats an answer exactly then as answering it", () => { + expect(read([request(100, "edge")]).carried).toHaveLength(1); + const answered = read([ + request(90, "old"), + at(100, "lease.granted", { leaseId: "L", requestId: "old", requester: "a", source: "warm" }), + ]); + expect(answered.carried).toEqual([]); + }); + const grant = (ts: number, requestId: string, requester = "a") => at(ts, "lease.granted", { leaseId: `L${requestId}`, requestId, requester, source: "warm" }); @@ -700,12 +725,11 @@ describe("a gateway", () => { expect([...result.workers]).toEqual(["w9"]); }); - it("ignores a dispatch missing requestId, requesterId or workerId, or outside the window", () => { + it("ignores a dispatch missing requestId or workerId, or outside the window", () => { const result = read( [ request(110, "r"), at(111, "request.dispatched", { requesterId: "a", workerId: "w1" }), - at(112, "request.dispatched", { requestId: "r", workerId: "w1" }), at(113, "request.dispatched", { requestId: "r", requesterId: "a" }), dispatch(100, "r"), dispatch(201, "r"), @@ -935,13 +959,12 @@ describe("a gateway", () => { expect(result.requests).toEqual([]); }); - it("ignores a dispatch missing requesterId or workerId, recording no worker for it", () => { + it("names the worker from a dispatch that carries no requesterId, and records none for one with no workerId", () => { const noRequester = read( [request(110, "r"), at(111, "request.dispatched", { requestId: "r", workerId: "w1" })], FLEET, ); - expect(noRequester.requests[0]?.worker).toBeUndefined(); - expect([...noRequester.workers]).toEqual([]); + expect(noRequester.requests[0]?.worker).toBe("w1"); const noWorker = read( [request(110, "r"), at(111, "request.dispatched", { requestId: "r", requesterId: "a" })], FLEET, diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index b2e2356e..cb2ad3bd 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -125,13 +125,6 @@ type Relayed = { readonly worker: string; } & RelayedAnswer; -interface Dispatch { - readonly index: number; - readonly requestId: string; - readonly requester: string; - readonly worker: string; -} - interface Request { readonly index: number; readonly at: number; @@ -180,7 +173,8 @@ class Reader { >(); readonly #grants = new Map(); readonly #ends = new Map(); - readonly #dispatches = new Map(); + /** The worker each request was dispatched to, by request id. */ + readonly #dispatches = new Map(); readonly #boundaries = new Map(); /** Relayed grants and rejections by the namespaced requester they are for. */ readonly #relayed = new Map(); @@ -285,11 +279,10 @@ class Reader { #dispatched(seen: Seen): void { const requestId = text(seen.payload, "requestId"); - const requester = text(seen.payload, "requesterId"); const worker = text(seen.payload, "workerId"); if (!this.options.fleet || !seen.within) return; - if (requestId === undefined || requester === undefined || worker === undefined) return; - this.#dispatches.set(requestId, { index: seen.index, requestId, requester, worker }); + if (requestId === undefined || worker === undefined) return; + this.#dispatches.set(requestId, worker); } #relay(seen: Seen, relayed: RelayedAnswer): void { @@ -453,7 +446,7 @@ class Reader { /** A fleet request's outcome is the worker's relayed answer for its requester (ADR 0016 §6). */ #fleetFact(base: RequestFactBase, requestId: string, request: Request): RequestFact { - const dispatched = this.#dispatches.get(requestId)?.worker; + const dispatched = this.#dispatches.get(requestId); const relayed = this.#fleetOutcome(request); const worker = dispatched ?? relayed?.worker; if (worker !== undefined) this.#workers.add(worker); From cb0fef0ce9cf04dd2d8c89addfae76df7e1a818b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 20:51:04 +0200 Subject: [PATCH 14/23] test: the stats e2e tests wait out a bucket, so they get the time to --- e2e/usage-stats.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index cfe7e5b4..dac37bba 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -173,7 +173,7 @@ describe("simlock stats", () => { expect(table.code).toBe(0); expect(table.stdout).toContain("Totals"); expect(table.stdout).toContain("Requests:"); - }); + }, 240_000); it("after a --no-wait refusal, requests and rejected.byReason.no-wait each rise by one and granted is unchanged", async () => { const env = await withDaemon({ @@ -198,7 +198,7 @@ describe("simlock stats", () => { (before.totals.rejected.byReason["no-wait"] ?? 0) + 1, ); expect(after.totals.granted).toBe(before.totals.granted); - }); + }, 240_000); it("simlock stats --since 1h returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { const env = await withDaemon(); @@ -228,7 +228,7 @@ describe("simlock stats", () => { wait: before.totals.wait, }); expect(after.requesters).toEqual(before.requesters); - }); + }, 240_000); it("on a gateway with two workers, simlock stats has fleet totals and one row per worker, and each worker's own simlock stats covers itself only", async () => { const port = await freeLoopbackPort(); @@ -281,5 +281,5 @@ describe("simlock stats", () => { expect(own.totals.granted).toBe(1); expect(own.workers[0]?.granted).toBe(1); } - }); + }, 240_000); }); From 821dac341f06ff90a5ec34b4ec88f034631c1311 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 5 Oct 2026 20:51:26 +0200 Subject: [PATCH 15/23] refactor: core/usage's index exports only what its callers import --- src/core/usage/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/usage/index.ts b/src/core/usage/index.ts index f14f039f..530547ce 100644 --- a/src/core/usage/index.ts +++ b/src/core/usage/index.ts @@ -1,2 +1 @@ export { tokenLabelMap, UsageReader } from "./usage-reader.js"; -export type { UsageHistory, UsageReaderOptions, UsageResult } from "./usage-reader.js"; From 0fcc5917d9c880003f4c216c679f11f70b862f87 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 8 Oct 2026 21:09:38 +0200 Subject: [PATCH 16/23] test: the stats e2e names its window's end instead of waiting out a bucket Whole file 4 tests, 6 s instead of minutes; no test timeout override. 0 failing -> 0 failing --- e2e/usage-stats.test.ts | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index dac37bba..40e4fb7b 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -39,18 +39,33 @@ interface Usage { readonly requesters: readonly { readonly id: string; readonly requests: number }[]; } +const HOUR_MS = 3_600_000; +const MINUTE_MS = 60_000; + +/** + * The last hour up to `to`, asked with `--from` and `--to`. The window is rounded down to the + * series bucket (a minute here), so what happened in the minute now running is counted once the + * minute is over; a test that names the end itself does not wait the minute out. + */ async function stats( env: { cli: (args: string[]) => Promise<{ code: number | null; json?: unknown }> }, - since = "1h", + to: number, ): Promise { - const result = await env.cli(["stats", "--since", since, "--json"]); + const result = await env.cli([ + "stats", + "--from", + new Date(to - HOUR_MS).toISOString(), + "--to", + new Date(to).toISOString(), + "--json", + ]); expect(result.code).toBe(0); return result.json as Usage; } /** - * The figures once the window has closed over the newest event: the window is rounded down to the - * series bucket, so what happened in the minute now running is counted when the minute is over. + * The figures for a window that closes over the newest event: its end is the first minute + * boundary after that event, or `at` when given (so two calls compare the same window). */ async function settledStats( env: { @@ -58,15 +73,17 @@ async function settledStats( events: () => Promise; }, until: (usage: Usage) => boolean = () => true, + at?: number, ): Promise { let usage: Usage | undefined; await waitFor( async () => { const newest = Math.max(...(await env.events()).map((event) => event.timestamp)); - usage = await stats(env); - return usage.window.to >= newest && until(usage); + const to = at ?? (Math.floor(newest / MINUTE_MS) + 1) * MINUTE_MS; + usage = await stats(env, to); + return (at !== undefined || usage.window.to >= newest) && until(usage); }, - { interval: 1_000, label: "the window closed over the newest event", timeout: 90_000 }, + { interval: 500, label: "the window closed over the newest event", timeout: 30_000 }, ); return usage as Usage; } @@ -173,7 +190,7 @@ describe("simlock stats", () => { expect(table.code).toBe(0); expect(table.stdout).toContain("Totals"); expect(table.stdout).toContain("Requests:"); - }, 240_000); + }); it("after a --no-wait refusal, requests and rejected.byReason.no-wait each rise by one and granted is unchanged", async () => { const env = await withDaemon({ @@ -198,7 +215,7 @@ describe("simlock stats", () => { (before.totals.rejected.byReason["no-wait"] ?? 0) + 1, ); expect(after.totals.granted).toBe(before.totals.granted); - }, 240_000); + }); it("simlock stats --since 1h returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { const env = await withDaemon(); @@ -213,7 +230,7 @@ describe("simlock stats", () => { expect(before.totals.granted).toBe(1); await env.restartDaemon(); - const after = await settledStats(env); + const after = await settledStats(env, () => true, before.window.to); expect(after.totals).toMatchObject({ boot: before.totals.boot, @@ -228,7 +245,7 @@ describe("simlock stats", () => { wait: before.totals.wait, }); expect(after.requesters).toEqual(before.requesters); - }, 240_000); + }); it("on a gateway with two workers, simlock stats has fleet totals and one row per worker, and each worker's own simlock stats covers itself only", async () => { const port = await freeLoopbackPort(); @@ -281,5 +298,5 @@ describe("simlock stats", () => { expect(own.totals.granted).toBe(1); expect(own.workers[0]?.granted).toBe(1); } - }, 240_000); + }); }); From b0ddcfcb029c87040088c96d8015587b94c30929 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 9 Oct 2026 22:41:40 +0200 Subject: [PATCH 17/23] =?UTF-8?q?docs:=20leave=20ADR=200016=20=C2=A76=20as?= =?UTF-8?q?=20written;=20ADR=200021=20replaced=20its=20join=20(#347)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...-derived-on-read-from-the-event-history.md | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md b/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md index 5f55a9da..35cb5fdb 100644 --- a/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md +++ b/docs/internal/adr/0016-usage-figures-are-derived-on-read-from-the-event-history.md @@ -140,18 +140,19 @@ grants, held time, provisioning, boots, capacity and device incidents, each attributed to its worker. A relayed `lease.requested`, `lease.queued` or `queue.changed` is not counted; the gateway forwards every request as `noWait`, so a worker's queue never holds a fleet request. A relayed -`lease.rejected` counts only when it settles a request by the rule below. The -gateway emits no rejection of its own for a request that failed on its worker. - -A fleet request's outcome is the first relayed `lease.granted` or -`lease.rejected` for its namespaced requester at or after the gateway's own -`lease.requested` for it and before the gateway's next `lease.requested` for -that requester. `request.dispatched` names the worker when it exists, and -otherwise the relayed event's `workerId` does. A rejection carries the -worker's reason. The grant may arrive before `request.dispatched`: a warm -device is granted before the dispatch answer reaches the gateway. The core -allows one open request per requester, so within those bounds the match is -unique, and a request with no outcome in them is open. +`lease.rejected` counts only when it settles a dispatched request: it comes +from the worker that request went to, for its namespaced requester, at or +after the dispatch, and carries the worker's reason. The gateway emits no +rejection of its own for a request that failed on its worker. + +A fleet request is joined to its outcome through `request.dispatched`: the +gateway's request id names the worker and the requester, and the first +relayed `lease.granted` or `lease.rejected` from that worker for the +namespaced requester at or after the dispatch, and before the gateway's +next `request.dispatched` or `lease.requested` for the same requester, is +that request's outcome. The core allows one open request per requester, so +within those bounds the match is unique, and a request with no outcome in +them is open. The answer has fleet totals and one entry per worker. A worker answers the same shape as a fleet of one (ADR 0012): totals, and one entry for itself. From d608467fb1b39f3e9773e229ab5c943508d6db90 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 9 Oct 2026 22:48:53 +0200 Subject: [PATCH 18/23] WIP: tests for the fleet join by request id --- docs/CLI.md | 36 +- docs/CLIENT.md | 5 +- docs/internal/ARCHITECTURE.md | 28 +- e2e/usage-stats.test.ts | 16 +- src/bus/event-file.test.ts | 42 +- src/bus/event-file.ts | 37 +- src/bus/index.ts | 2 + src/cli/index.test.ts | 1 + src/cli/stats.test.ts | 41 ++ src/cli/stats.ts | 16 +- src/contract/operations.test.ts | 1 + src/contract/schemas.ts | 8 + src/core/usage/compute-usage.test.ts | 440 +++++++++++----- src/core/usage/compute-usage.ts | 96 +++- src/core/usage/read-events.test.ts | 745 ++++++++++++++------------- src/core/usage/read-events.ts | 305 +++++------ src/core/usage/usage-reader.test.ts | 58 ++- src/core/usage/usage-reader.ts | 32 +- src/daemon/dispatcher.test.ts | 16 +- src/gateway/dispatcher.test.ts | 14 +- src/simlock-client/client.test.ts | 1 + 21 files changed, 1214 insertions(+), 726 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index 59beb54e..8e0cbb44 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -1966,6 +1966,7 @@ Totals Requests: 12 (10 granted, 2 rejected) Granted: warm 6, booted 3, provisioned 1 Rejected: no-wait 1, timeout 1 + Declined: 2 Wait: p50 1.2s, p95 4s, max 9.1s (11 samples) Held: p50 5m 0s, p95 30m 0s, max 1h 2m (9 samples) Turnaround: p50 5m 10s, p95 31m 40s, max 1h 3m (9 samples) @@ -1999,9 +2000,9 @@ What the figures count: `provisioned` was created for the request. - **Rejected** are the requests that ended without a device, by reason (`timeout`, `no-wait`, `cancelled`, ...), plus the requests refused before - they were stored (a requester that already holds a lease). Those are in the - window their rejection falls in and are not requests, so granted plus - rejected can be more than requests. + they were stored (a requester that already holds a lease, or one that asked + for a lease ID that is taken). Those are in the window their rejection falls + in and are not requests, so granted plus rejected can be more than requests. - **Wait** is the time from the request to its grant or rejection. **Held** is the time from the grant to the release or expiry of the lease. **Turnaround** is the time from the request to that end. **Provisioning** and **Boot** are the @@ -2019,18 +2020,33 @@ What the figures count: the window by name: `device.purge-failed`, `device.recovery-failed`, `component.install-failed`, and any other event whose name ends in `-failed`. The row is left out when there were none. +- **Declined** counts the times a worker refused a request that a gateway sent + it, by the time of the refusal. It is a count of refusals, not of requests: a + gateway tries another worker after a refusal, so one request can be declined + by several workers. The row is left out when there were none. - A requester that is a token shows the token's label beside its id. The window is rounded down to whole steps of the time series `--json` carries, -which is 1 minute for a window of a few hours and grows to 1 day for 90 days, so -it never ends at a time that has not come yet; the window in the answer is the -rounded one. +which is 1 minute for a window of a few hours and grows to 1 day for 90 days. +Neither end moves later than the time you asked for; the window in the answer is +the rounded one. Against a **gateway** the totals are the fleet's and there is a row for each -worker, with the grants, hold times and device figures of that worker. Requests, -waits and rejections come from the gateway's own queue; the queue figure is the -fleet queue's. A worker's own `simlock stats` covers that worker only: its row -is itself, and a request that arrived through a gateway shows under the +worker. Requests, waits and rejections come from the gateway's own record of +each request: it waits from the request until the gateway grants or rejects it, +and a request the gateway was still holding when it stopped counts as rejected +with the reason `daemon-restarted`. How the device came to be ready and how long +the lease was held come from the worker's record of the lease; a grant whose +worker's record the gateway never received counts as `unknown` and has no held +time. A worker's row has the requests granted on it or that failed on it +(`worker-failed`), its declined count and its device figures; a request that +ended any other way, or is still waiting, is in the totals and the platform rows +only. The queue figure is the fleet queue's. + +A worker's own `simlock stats` covers that worker only: its row is itself. A +request that a gateway sent it is not one of the worker's requests: it is counted +under **Probes**, not under requests, and has no wait or turnaround of its own. +Its grant still counts as a grant, with its source and held time, under the requester id the gateway gave it. If the history does not reach back to the start of the window (a new daemon, or diff --git a/docs/CLIENT.md b/docs/CLIENT.md index ca7a199e..ee9e33b8 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -599,7 +599,10 @@ asked for, rounded down to a whole number of `bucketMs` at both ends, and the se than 200 points. A window that ends before the oldest event the history holds rejects with `HISTORY_NOT_KEPT`; its `details.oldestTs` is the oldest time the history reaches. Against a gateway the totals are the fleet's and `workers` has -one entry for each worker. +one entry for each worker; `bySource.unknown` counts grants whose source the +gateway never learned, and there is no `probes`. Against a worker, `probes` +counts the requests a gateway sent it, which `requests` leaves out. `declined` +counts the refusals of such requests. ## One connection, no reconnect, no retry diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index f93697ed..a34548b6 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -2115,18 +2115,22 @@ pushes, so the join neither loses nor repeats an event. `usage.get` (ADR 0016) is the third reader of that history, and the only one that turns it into numbers. `computeUsage` (`src/core/usage/`) is pure: it takes -the envelopes of a window, with the latest `capacity.changed` and `queue.changed` -at or before the window's start that `EventHistory.read`'s `carry` adds, and -the ids of the requests made before it (`requestedBefore`, so a rejection of -one is in no count), and the latest `lease.requested`, `lease.granted` and -`lease.rejected` of each requester at or before it (so a request still waiting when -the window opens counts in the series' `waiting`), and returns the figures. `readEvents` reads them into one fact per request, joined -by `requestId` and `leaseId`, and one per device event; `compute-usage.ts` adds -them up by platform, by worker and by requester. On a gateway (`fleet`) request -facts come from its own events and device facts from the events its workers -relayed, and a request's outcome is the first relayed grant or rejection for its -namespaced requester before the requester's next request, the worker being the one -`request.dispatched` names, else the one that relayed it; that rule is in `readEvents` and nowhere else. One +the envelopes of a window, with the latest `capacity.changed`, `queue.changed` and +`daemon.started` at or before the window's start and the latest `lease.requested` +of each requester, that `EventHistory.read`'s `carry` adds, and the ids of the +requests made before it (`requestedBefore`, so a rejection of one is in no count) +and of those answered by then (`answeredBefore`, so a request still waiting when +the window opens counts in the series' `waiting`), and returns the figures. +`readEvents` reads them into one fact per request, joined by `requestId` and +`leaseId`, one per device event and one per `lease.declined`; `compute-usage.ts` +adds them up by platform, by worker and by requester. On a gateway (`fleet`) +request facts come from its own events and device facts from the events its +workers relayed (ADR 0021 §5): a request's outcome is the gateway's own +`request.granted` or `lease.rejected` for its request id, else its next own +`daemon.started` ends it as rejected `daemon-restarted`, else it is open; the +grant's source and held time are the relayed `lease.granted` of the worker and +lease it names. On a worker a request carrying `fleetRequestId` is a probe, +counted apart from its requests. That rule is in `readEvents` and nowhere else. One `UsageReader` serves both dispatchers: it rounds the window down to the series bucket at both ends, keeps its last answer by that window and the newest event id, and joins token labels, so the daemon's two handlers differ only in `fleet`. `simlock stats` diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index 40e4fb7b..4084953f 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -21,6 +21,8 @@ interface Figures { readonly requests: number; readonly granted: number; readonly bySource: Record; + readonly declined: number; + readonly probes?: number; readonly rejected: { readonly total: number; readonly byReason: Record }; readonly wait: Samples; readonly held: Samples; @@ -103,7 +105,7 @@ function payload(event: RecordedEvent): Record { } describe("simlock stats", () => { - it("after a scripted run of leases against the fake driver, simlock stats --since 1h --json reports the counts a test derives independently from simlock events --since 1h", async () => { + it("after a scripted run of leases against the fake driver, simlock stats --json for the last hour, named with --from and --to, reports the counts a test derives independently from the same hour of simlock events", async () => { const env = await withDaemon({ configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }, }); @@ -217,7 +219,7 @@ describe("simlock stats", () => { expect(after.totals.granted).toBe(before.totals.granted); }); - it("simlock stats --since 1h returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { + it("simlock stats for the last hour, named with --from and --to, returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { const env = await withDaemon(); await env.driverScript.set({ ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, @@ -291,12 +293,22 @@ describe("simlock stats", () => { expect(fleet?.totals.requests).toBe(2); expect(fleet?.workers.map((worker) => worker.label).sort()).toEqual(["worker-a", "worker-b"]); expect(fleet?.workers.map((worker) => worker.granted)).toEqual([1, 1]); + // The gateway knows each grant's source from its worker, counts no probes, and its declined + // figure is its workers'. + expect(fleet?.totals.bySource.unknown).toBe(0); + expect(fleet?.totals).not.toHaveProperty("probes"); + expect(fleet?.totals.declined).toBe( + fleet?.workers.reduce((total, worker) => total + worker.declined, 0), + ); for (const worker of [workerA, workerB]) { const own = await settledStats(worker); expect(own.workers).toHaveLength(1); expect(own.totals.granted).toBe(1); expect(own.workers[0]?.granted).toBe(1); + // What the gateway sent it are probes, not the worker's requests. + expect(own.totals.requests).toBe(0); + expect(own.totals.probes).toBeGreaterThanOrEqual(1); } }); }); diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index 9de4a5f0..99b3830d 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -228,7 +228,12 @@ describe("EventHistory", () => { const read = await events.read({ carry: ["queue.changed"], sinceTs: 2_000 }); - expect(read).toEqual({ events: [early, late], oldestTs: 1_000, requestedBefore: new Set() }); + expect(read).toEqual({ + answeredBefore: new Set(), + events: [early, late], + oldestTs: 1_000, + requestedBefore: new Set(), + }); expect(await events.replay({ carry: ["queue.changed"], sinceTs: 2_000 })).toEqual([ early, late, @@ -439,6 +444,41 @@ describe("readEventFile", () => { expect([...read.requestedBefore]).toEqual([]); }); + it("reports the requestId of each lease.granted, lease.rejected and request.granted at or before sinceTs as answered, and none from after it or from other events", async () => { + const answer = (seq: number, timestamp: number, event: string, requestId: string) => + ({ ...envelope(seq, timestamp), event, payload: { requestId } }) as EventEnvelope; + const filesystem = await filesystemWith({ + "/data/events.jsonl": lines( + answer(1, 100, "lease.granted", "granted"), + answer(2, 110, "lease.rejected", "rejected"), + answer(3, 120, "request.granted", "handed"), + answer(4, 130, "lease.declined", "declined"), + answer(5, 140, "lease.requested", "asked"), + answer(6, 200, "lease.granted", "edge"), + answer(7, 300, "lease.granted", "later"), + { ...envelope(8, 150), event: "lease.granted", payload: {} } as EventEnvelope, + ), + }); + + const read = await readEventHistory(filesystem, "/data/events.jsonl", { sinceTs: 200 }); + + expect([...read.answeredBefore].sort()).toEqual(["edge", "granted", "handed", "rejected"]); + expect([...read.requestedBefore]).toEqual(["asked"]); + }); + + it("reports the requests answered before sinceTs from the ring too", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + bus.emit("lease.rejected", { requestId: "old", requester: "a" } as never, "lease"); + clock.advance(2_000); + bus.emit("lease.rejected", { requestId: "new", requester: "a" } as never, "lease"); + + const read = await events.read({ carry: [], sinceTs: 2_000 }); + + expect([...read.answeredBefore]).toEqual(["old"]); + }); + it("carries events whose worker and requester run together into the same text as two different ones", async () => { const withPayload = (seq: number, payload: Record): EventEnvelope => ({ diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index a7a157b3..acee6d41 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -42,7 +42,7 @@ export async function readEventHistory( const seen = new Set(); const events: EventEnvelope[] = []; const carried = new Map(); - const requestedBefore = new Set(); + const before = { answered: new Set(), requested: new Set() }; let oldestTs: number | undefined; // Oldest generation first, so events that tie on time and seq keep the order they were written. for (const line of generations.reverse().flat()) { @@ -51,7 +51,7 @@ export async function readEventHistory( oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); if (envelope.timestamp <= sinceTs) { keepIfCarried(carried, envelope, carry); - noteRequest(requestedBefore, envelope); + noteRequest(before, envelope); continue; } if (seen.has(eventKey(envelope))) continue; @@ -61,23 +61,39 @@ export async function readEventHistory( return { events: [...carried.values(), ...events].sort(byTimeThenSeq), oldestTs, - requestedBefore, + answeredBefore: before.answered, + requestedBefore: before.requested, }; } /** What a history read answers: the events, how far back it reaches, and which requests were made - * before `sinceTs` (a rejection in the window follows its request, ADR 0016 §4). */ + * before `sinceTs` or answered by then (a rejection in the window follows its request, ADR 0016 + * §2). */ export interface EventHistoryRead { readonly events: EventEnvelope[]; readonly oldestTs: number | undefined; /** The `requestId` of every `lease.requested` at or before `sinceTs`. */ readonly requestedBefore: ReadonlySet; + /** The `requestId` of every `lease.granted`, `lease.rejected` and `request.granted` at or before + * `sinceTs`: a request made before it and in neither set was still waiting when it opened. */ + readonly answeredBefore: ReadonlySet; } -function noteRequest(requestedBefore: Set, envelope: EventEnvelope): void { - if (envelope.event !== "lease.requested") return; +const ANSWERS: ReadonlySet = new Set([ + "lease.granted", + "lease.rejected", + "request.granted", +]); + +/** Notes the request id of a `lease.requested` in `requested`, and of an answer in `answered`. */ +function noteRequest( + { requested, answered }: { readonly requested: Set; readonly answered: Set }, + envelope: EventEnvelope, +): void { const requestId = (envelope.payload as { readonly requestId?: unknown }).requestId; - if (typeof requestId === "string") requestedBefore.add(requestId); + if (typeof requestId !== "string") return; + if (envelope.event === "lease.requested") requested.add(requestId); + if (ANSWERS.has(envelope.event)) answered.add(requestId); } /** @@ -239,19 +255,20 @@ export class EventHistory { }): EventHistoryRead { const ring = this.#options.bus.replay(); const carried = new Map(); - const requestedBefore = new Set(); + const before = { answered: new Set(), requested: new Set() }; let oldestTs: number | undefined; for (const envelope of ring) { oldestTs = Math.min(oldestTs ?? envelope.timestamp, envelope.timestamp); if (envelope.timestamp > sinceTs) continue; keepIfCarried(carried, envelope, carry); - noteRequest(requestedBefore, envelope); + noteRequest(before, envelope); } const newer = ring.filter((envelope) => envelope.timestamp > sinceTs); return { events: [...carried.values(), ...newer].sort(byTimeThenSeq), oldestTs, - requestedBefore, + answeredBefore: before.answered, + requestedBefore: before.requested, }; } } diff --git a/src/bus/index.ts b/src/bus/index.ts index df198f02..1219ee8e 100644 --- a/src/bus/index.ts +++ b/src/bus/index.ts @@ -1,6 +1,8 @@ import { type Clock, CryptoIdGenerator, type IdGenerator } from "../ports/index.js"; import { byTimeThenSeq } from "./order.js"; +export { byTimeThenSeq }; + export interface CapacityFiguresPayload { readonly running: number; readonly maxRunning: number; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index da782651..25872ffc 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -5724,6 +5724,7 @@ function usageAnswer( const figures = { boot: samples, bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, failures: { byEvent: {} }, granted: 0, held: samples, diff --git a/src/cli/stats.test.ts b/src/cli/stats.test.ts index 613dd1b7..876f2e3c 100644 --- a/src/cli/stats.test.ts +++ b/src/cli/stats.test.ts @@ -9,6 +9,7 @@ function figuresFixture(overrides: Partial = {}): UsageFigures { return { boot: NONE, bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, failures: { byEvent: {} }, granted: 0, held: NONE, @@ -251,6 +252,46 @@ describe("formatUsage rows that depend on the figures", () => { expect(without.some((line) => line.startsWith(" Granted:"))).toBe(false); }); + it("prints the unknown grant source only when a gateway counted one", () => { + const unknown = formatUsage( + usageFixture({ + totals: figuresFixture({ + bySource: { booted: 0, provisioned: 0, unknown: 2, warm: 1 }, + granted: 3, + }), + }), + ).split("\n"); + const none = formatUsage( + usageFixture({ + totals: figuresFixture({ + bySource: { booted: 0, provisioned: 0, unknown: 0, warm: 1 }, + granted: 1, + }), + }), + ).split("\n"); + + expect(unknown).toContain(" Granted: warm 1, booted 0, provisioned 0, unknown 2"); + expect(none).toContain(" Granted: warm 1, booted 0, provisioned 0"); + }); + + it("prints the Declined row only when a worker declined something, and the Probes row only when a worker counted a probe", () => { + const both = formatUsage( + usageFixture({ totals: figuresFixture({ declined: 4, probes: 7 }) }), + ).split("\n"); + const without = formatUsage( + usageFixture({ totals: figuresFixture({ declined: 0, probes: 0 }) }), + ).split("\n"); + const gateway = formatUsage(usageFixture({ totals: figuresFixture({ declined: 1 }) })).split( + "\n", + ); + + expect(both).toContain(" Declined: 4"); + expect(both).toContain(" Probes: 7"); + expect(without.some((line) => /^ {2}(Declined|Probes):/.test(line))).toBe(false); + expect(gateway).toContain(" Declined: 1"); + expect(gateway.some((line) => line.startsWith(" Probes:"))).toBe(false); + }); + it("prints the Rejected row only when something was rejected", () => { const withReject = formatUsage( usageFixture({ diff --git a/src/cli/stats.ts b/src/cli/stats.ts index 29849241..2c45e1cc 100644 --- a/src/cli/stats.ts +++ b/src/cli/stats.ts @@ -98,7 +98,14 @@ function totalRows(totals: UsageFigures): string[] { } /** The requests, and what became of them, leaving out the rows with nothing in them. */ -function requestRows({ bySource, granted, rejected, requests }: UsageFigures): string[] { +function requestRows({ + bySource, + declined, + granted, + probes, + rejected, + requests, +}: UsageFigures): string[] { return [ row("Requests:", `${requests} (${granted} granted, ${rejected.total} rejected)`), ...(granted === 0 @@ -106,10 +113,15 @@ function requestRows({ bySource, granted, rejected, requests }: UsageFigures): s : [ row( "Granted:", - `warm ${bySource.warm}, booted ${bySource.booted}, provisioned ${bySource.provisioned}`, + `warm ${bySource.warm}, booted ${bySource.booted}, provisioned ${bySource.provisioned}` + + (bySource.unknown === undefined || bySource.unknown === 0 + ? "" + : `, unknown ${bySource.unknown}`), ), ]), ...(rejected.total === 0 ? [] : [row("Rejected:", list(rejected.byReason))]), + ...(declined === 0 ? [] : [row("Declined:", String(declined))]), + ...(probes === undefined || probes === 0 ? [] : [row("Probes:", String(probes))]), ]; } diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index d7befc1e..4e779631 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -1295,6 +1295,7 @@ describe("usage.get contract details", () => { const figures = { boot: samples, bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, failures: { byEvent: {} }, granted: 0, held: samples, diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 52d81c17..7d20bf29 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -1133,9 +1133,14 @@ const usageFiguresSchema = z.object({ bySource: z.object({ booted: z.number().int().nonnegative(), provisioned: z.number().int().nonnegative(), + /** A gateway only: a grant whose worker's own `lease.granted` was not relayed (ADR 0021 §5). */ + unknown: z.number().int().nonnegative().optional(), warm: z.number().int().nonnegative(), }), boot: usageSamplesSchema, + /** `lease.declined` events by their timestamp: a worker's own, or a gateway's relayed ones. Not + * requests (ADR 0021 §5). */ + declined: z.number().int().nonnegative(), failures: z.object({ byEvent: usageCountsSchema }), granted: z.number().int().nonnegative(), held: usageSamplesSchema, @@ -1145,6 +1150,9 @@ const usageFiguresSchema = z.object({ quarantineRecovered: z.number().int().nonnegative(), quarantined: z.number().int().nonnegative(), }), + /** A worker's own figure, absent from a gateway's answer: the gateway dispatches it received + * (`lease.requested` carrying `fleetRequestId`), which `requests` leaves out (ADR 0021 §5). */ + probes: z.number().int().nonnegative().optional(), provisioning: usageSamplesSchema, queue: z.object({ meanDepth: z.number().nullable(), peakDepth: z.number().nullable() }), rejected: z.object({ diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index 8f1a3568..e320a864 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -10,7 +10,7 @@ const DAY = 24 * HOUR; const T0 = 1_000_000_000; const WINDOW = { from: T0, to: T0 + HOUR }; const WORKER: UsageOptions = { fleet: false, workers: [{ id: "self" }] }; -const FLEET: UsageOptions = { fleet: true, requesterPrefix: "gw:g1:" }; +const FLEET: UsageOptions = { fleet: true }; let sequence = 0; @@ -100,6 +100,34 @@ const capacity = ( const queueChanged = (ts: number, depth: number, extra: Record = {}) => at(ts, "queue.changed", { depth, ...extra }); +/** The gateway's own record that it handed a fleet request's grant to its caller (ADR 0021 §4). */ +const handed = (ts: number, requestId: string, worker: string, workerLeaseId: string) => + at(ts, "request.granted", { leaseId: `gw-${workerLeaseId}`, requestId, worker, workerLeaseId }); + +/** A worker's refusal of a gateway's dispatch, as the worker records it (ADR 0021 §2). */ +const declined = (ts: number, extra: Record = {}) => + at(ts, "lease.declined", { + fleetRequestId: "f", + reason: "no-wait", + requestId: "worker-side", + requestSpec: { platform: "ios" }, + requester: "gw:g1:agent-a", + ...extra, + }); + +const restarted = (ts: number, extra: Record = {}) => + at(ts, "daemon.started", { configSnapshot: {}, version: "0", ...extra }); + +/** A worker's `lease.requested` for a gateway's dispatch: it carries the fleet request id. */ +const probe = (ts: number, requestId: string, requester = "gw:g1:a") => + at(ts, "lease.requested", { + fleetRequestId: `f-${requestId}`, + requestId, + requestSpec: { platform: "ios" }, + requester, + waitPolicy: "noWait", + }); + describe("computeUsage", () => { it("counts a request granted, held and released as one request, one grant, and one sample in wait, held and turnaround with the expected milliseconds", () => { const usage = computeUsage( @@ -452,29 +480,11 @@ describe("computeUsage", () => { const usage = computeUsage( [ requested(T0 + 1_000, "r1", "a", "ios"), - at(T0 + 1_500, "request.dispatched", { - model: "m", - platform: "ios", - queuedMs: 0, - reason: "warm-hit", - requestId: "r1", - requesterId: "a", - stage: "s", - workerId: "w1", - }), granted(T0 + 2_000, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), + handed(T0 + 2_100, "r1", "w1", "l1"), requested(T0 + 3_000, "r2", "b", "android"), - at(T0 + 3_500, "request.dispatched", { - model: "m", - platform: "android", - queuedMs: 0, - reason: "free-capacity", - requestId: "r2", - requesterId: "b", - stage: "s", - workerId: "w2", - }), granted(T0 + 4_000, "w2-r", "l2", "provisioned", "gw:g1:b", { workerId: "w2" }), + handed(T0 + 4_100, "r2", "w2", "l2"), at(T0 + 4_500, "device.provisioned", { deviceId: "dev-l2", driver: "fake", @@ -499,22 +509,13 @@ describe("computeUsage", () => { expect(usage.totals.granted).toBe(2); }); - it("with fleet: true counts a fleet request once, from the gateway's own events, and attributes its grant to the worker named by request.dispatched", () => { + it("with fleet: true counts a fleet request once, from the gateway's own events, and attributes its grant to the worker of its request.granted", () => { const usage = computeUsage( [ requested(T0 + 1_000, "gw-r1"), - at(T0 + 1_100, "request.dispatched", { - model: "m", - platform: "ios", - queuedMs: 100, - reason: "warm-hit", - requestId: "gw-r1", - requesterId: "agent-a", - stage: "s", - workerId: "w1", - }), - // The worker's own copy of the same request. + // The worker's own copy of the same request, relayed. at(T0 + 1_300, "lease.requested", { + fleetRequestId: "gw-r1", requestId: "w-r1", requestSpec: { platform: "ios" }, requester: "gw:g1:agent-a", @@ -522,6 +523,7 @@ describe("computeUsage", () => { workerId: "w1", }), granted(T0 + 4_000, "w-r1", "l1", "booted", "gw:g1:agent-a", { workerId: "w1" }), + handed(T0 + 4_200, "gw-r1", "w1", "l1"), released(T0 + 9_000, "l1", { workerId: "w1" }), ], WINDOW, @@ -531,13 +533,11 @@ describe("computeUsage", () => { expect(usage.totals.requests).toBe(1); expect(usage.totals.granted).toBe(1); expect(usage.totals.bySource.booted).toBe(1); - expect(usage.totals.wait.p50).toBe(3_000); - expect(usage.totals.held.p50).toBe(5_000); expect(usage.workers).toHaveLength(1); expect(usage.workers[0]).toMatchObject({ granted: 1, id: "w1", requests: 1 }); }); - it("with fleet: true ignores relayed lease.requested, lease.queued and queue.changed, and a relayed lease.rejected that matches no dispatched request", () => { + it("with fleet: true ignores relayed lease.requested, lease.queued, queue.changed, lease.rejected and lease.declined as request facts", () => { const usage = computeUsage( [ queueChanged(T0 + 1_000, 1), @@ -551,6 +551,7 @@ describe("computeUsage", () => { }), at(T0 + 2_100, "lease.queued", { queuePosition: 1, requestId: "w-r1", workerId: "w1" }), rejected(T0 + 2_200, "w-r1", "no-wait", "gw:g1:ghost", { workerId: "w1" }), + declined(T0 + 2_300, { requestId: "w-r1", workerId: "w1" }), ], WINDOW, FLEET, @@ -559,123 +560,301 @@ describe("computeUsage", () => { expect(usage.totals.requests).toBe(0); expect(usage.totals.rejected).toEqual({ byReason: {}, total: 0 }); expect(usage.totals.queue.peakDepth).toBe(1); + expect(usage.requesters).toEqual([]); }); - it("with fleet: true settles a dispatched request as rejected, with the worker's reason, from the relayed lease.rejected of the worker it was dispatched to", () => { + it("with fleet: true does not count a relayed lease.granted carrying a request's fleetRequestId as its outcome when the gateway rejected it", () => { const usage = computeUsage( [ requested(T0 + 1_000, "gw-r1"), - at(T0 + 2_000, "request.dispatched", { - model: "m", - platform: "ios", - queuedMs: 0, - reason: "free-capacity", - requestId: "gw-r1", - requesterId: "agent-a", - stage: "s", + granted(T0 + 1_500, "w-r1", "l1", "warm", "gw:g1:agent-a", { + fleetRequestId: "gw-r1", workerId: "w1", }), - // Another requester's rejection is not this request's outcome. - rejected(T0 + 2_500, "w1-x", "no-wait", "gw:g1:agent-b", { workerId: "w1" }), - rejected(T0 + 3_000, "w1-r", "boot-timeout", "gw:g1:agent-a", { workerId: "w1" }), + // The grant came too late: the gateway had settled the request as a timeout. + rejected(T0 + 1_200, "gw-r1", "timeout", "agent-a"), ], WINDOW, FLEET, ); - expect(usage.totals.requests).toBe(1); expect(usage.totals.granted).toBe(0); - expect(usage.totals.rejected).toEqual({ byReason: { "boot-timeout": 1 }, total: 1 }); - expect(usage.workers[0]).toMatchObject({ id: "w1", rejected: { total: 1 } }); - expect(usage.workers).toHaveLength(1); + expect(usage.totals.rejected).toEqual({ byReason: { timeout: 1 }, total: 1 }); + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 0, unknown: 0, warm: 0 }); }); - it("with fleet: true does not join a requester's second dispatch's grant to its first request, which stays open", () => { - const dispatched = (ts: number, requestId: string) => - at(ts, "request.dispatched", { - model: "m", - platform: "ios", - queuedMs: 0, - reason: "free-capacity", - requestId, - requesterId: "agent-a", - stage: "s", - workerId: "w1", - }); - + it("with fleet: true does not let a stale worker's relayed refusal reject a request another worker granted", () => { const usage = computeUsage( [ requested(T0 + 1_000, "gw-r1"), - dispatched(T0 + 1_100, "gw-r1"), - requested(T0 + 10_000, "gw-r2"), - dispatched(T0 + 10_100, "gw-r2"), - granted(T0 + 12_000, "w-r2", "l2", "warm", "gw:g1:agent-a", { workerId: "w1" }), + rejected(T0 + 1_100, "w1-r", "no-wait", "gw:g1:agent-a", { workerId: "w1" }), + granted(T0 + 1_400, "w2-r", "l1", "warm", "gw:g1:agent-a", { workerId: "w2" }), + handed(T0 + 1_500, "gw-r1", "w2", "l1"), ], WINDOW, FLEET, ); - expect(usage.totals.requests).toBe(2); expect(usage.totals.granted).toBe(1); - expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + expect(usage.totals.rejected.total).toBe(0); + expect( + usage.workers.map((worker) => [worker.id, worker.granted, worker.rejected.total]), + ).toEqual([["w2", 1, 0]]); }); - it("with fleet: true joins a grant the worker emitted before the gateway announced the dispatch", () => { + it("with fleet: true does not close an open fleet request at a relayed worker daemon.started", () => { const usage = computeUsage( [ requested(T0 + 1_000, "gw-r1"), - // A warm grant reaches the gateway ahead of the response that announces the dispatch. - granted(T0 + 1_400, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), - at(T0 + 1_500, "request.dispatched", { - model: "m", - platform: "ios", - queuedMs: 0, - reason: "warm-hit", - requestId: "gw-r1", - requesterId: "agent-a", - stage: "s", - workerId: "w1", - }), + at(T0 + 2_000, "daemon.started", { configSnapshot: {}, version: "0", workerId: "w1" }), ], WINDOW, FLEET, ); - expect(usage.totals.granted).toBe(1); - expect(usage.totals.wait.p50).toBe(400); + expect(usage.totals.requests).toBe(1); + expect(usage.totals.rejected.total).toBe(0); + expect(usage.totals.wait.count).toBe(0); + expect(usage.series.at(-1)?.waiting).toBe(1); }); - it("with fleet: true does not credit a requester's next request with a grant that came before it", () => { + it("with fleet: true settles a request as rejected from the gateway's own lease.rejected, and a worker-failed one under its worker", () => { const usage = computeUsage( [ - requested(T0 + 1_000, "gw-r1"), - // The grant for the first request reaches the gateway ahead of its dispatch, and the - // requester then asks again and nothing answers the second request. - granted(T0 + 1_400, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), - requested(T0 + 10_000, "gw-r2"), + requested(T0 + 1_000, "gw-r1", "a"), + rejected(T0 + 2_000, "gw-r1", "timeout", "a"), + requested(T0 + 3_000, "gw-r2", "b"), + rejected(T0 + 3_500, "gw-r2", "worker-failed", "b", { code: "INTERNAL", worker: "w1" }), ], WINDOW, FLEET, ); expect(usage.totals.requests).toBe(2); - expect(usage.totals.granted).toBe(1); - expect(usage.totals.wait).toEqual({ count: 1, max: 400, p50: 400, p95: 400 }); + expect(usage.totals.granted).toBe(0); + expect(usage.totals.rejected).toEqual({ + byReason: { timeout: 1, "worker-failed": 1 }, + total: 2, + }); + expect(usage.workers).toHaveLength(1); + expect(usage.workers[0]).toMatchObject({ + id: "w1", + rejected: { byReason: { "worker-failed": 1 }, total: 1 }, + requests: 1, + wait: { count: 1, p50: 500 }, + }); }); - it("with fleet: true names the worker of the relayed grant when the request has no request.dispatched", () => { + it("with fleet: true counts a fleet request under the worker its request.granted or worker-failed rejection names, with its wait and turnaround, and a timeout rejection under no worker", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "g1", "a"), + granted(T0 + 1_800, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), + handed(T0 + 2_000, "g1", "w1", "l1"), + released(T0 + 7_800, "l1", { workerId: "w1" }), + requested(T0 + 3_000, "g2", "b"), + rejected(T0 + 3_400, "g2", "worker-failed", "b", { + code: "WORKER_UNREACHABLE", + worker: "w2", + }), + requested(T0 + 4_000, "g3", "c"), + rejected(T0 + 9_000, "g3", "timeout", "c"), + ], + WINDOW, + FLEET, + ); + + expect(usage.workers.map((worker) => worker.id)).toEqual(["w1", "w2"]); + expect(usage.workers[0]).toMatchObject({ + granted: 1, + requests: 1, + turnaround: { count: 1, p50: 7_000 }, + wait: { count: 1, p50: 1_000 }, + }); + expect(usage.workers[1]).toMatchObject({ + granted: 0, + rejected: { total: 1 }, + requests: 1, + wait: { count: 1, p50: 400 }, + }); + expect(usage.totals).toMatchObject({ + granted: 1, + rejected: { byReason: { timeout: 1, "worker-failed": 1 }, total: 2 }, + requests: 3, + wait: { count: 3 }, + }); + expect(usage.workers.reduce((total, worker) => total + worker.requests, 0)).toBe(2); + }); + + it("with fleet: true measures a fleet wait from the gateway's lease.requested to its request.granted, ignoring a relayed lease.granted stamped earlier", () => { const usage = computeUsage( [ requested(T0 + 1_000, "gw-r1"), - granted(T0 + 1_400, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w7" }), + granted(T0 + 800, "w-r1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), + handed(T0 + 3_000, "gw-r1", "w1", "l1"), ], WINDOW, FLEET, ); - expect(usage.totals.granted).toBe(1); - expect(usage.workers).toHaveLength(1); - expect(usage.workers[0]).toMatchObject({ granted: 1, id: "w7", requests: 1 }); + expect(usage.totals.wait).toEqual({ count: 1, max: 2_000, p50: 2_000, p95: 2_000 }); + }); + + it("with fleet: true takes the grant source and held time from the relayed lease.granted whose workerId and leaseId match the request.granted's worker and workerLeaseId, and counts source unknown with no held sample when it is missing", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "g1", "a"), + // The same lease id on three workers: only (w2, l1) is the one g1 names. + granted(T0 + 1_100, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), + granted(T0 + 1_200, "w2-r", "l1", "provisioned", "gw:g1:a", { workerId: "w2" }), + handed(T0 + 1_500, "g1", "w2", "l1"), + released(T0 + 6_200, "l1", { workerId: "w2" }), + released(T0 + 9_000, "l1", { workerId: "w1" }), + requested(T0 + 2_000, "g2", "b"), + handed(T0 + 2_400, "g2", "w3", "l9"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.bySource).toEqual({ booted: 0, provisioned: 1, unknown: 1, warm: 0 }); + expect(usage.totals.granted).toBe(2); + expect(usage.totals.held).toEqual({ count: 1, max: 5_000, p50: 5_000, p95: 5_000 }); + expect(usage.totals.turnaround).toEqual({ count: 1, max: 5_500, p50: 5_500, p95: 5_500 }); + expect(usage.totals.wait.count).toBe(2); + expect(usage.requesters.map((requester) => [requester.id, requester.heldTotalMs])).toEqual([ + ["a", 5_000], + ["b", 0], + ]); + }); + + it("with fleet: true counts each relayed lease.declined under its worker's and platform's declined, two declines of one request as two", () => { + const usage = computeUsage( + [ + declined(T0 + 1_000, { workerId: "w1" }), + declined(T0 + 2_000, { workerId: "w1" }), + declined(T0 + 3_000, { requestSpec: { platform: "android" }, workerId: "w2" }), + declined(T0 - 1_000, { workerId: "w1" }), + // The gateway's own events have no decline of this kind: nothing counts for no worker. + declined(T0 + 4_000), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.declined).toBe(3); + expect(usage.platforms.ios.declined).toBe(2); + expect(usage.platforms.android.declined).toBe(1); + expect(usage.workers.map((worker) => [worker.id, worker.declined])).toEqual([ + ["w1", 2], + ["w2", 1], + ]); + expect(usage.totals.requests).toBe(0); + }); + + it("with fleet: true ends a request with no outcome at the gateway's next daemon.started as rejected daemon-restarted, and leaves one with no later start open", () => { + const usage = computeUsage( + [ + requested(T0 + 1_000, "gone", "a"), + restarted(T0 + 5_000), + requested(T0 + 6_000, "open", "b"), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.requests).toBe(2); + expect(usage.totals.rejected).toEqual({ byReason: { "daemon-restarted": 1 }, total: 1 }); + expect(usage.totals.wait).toEqual({ count: 1, max: 4_000, p50: 4_000, p95: 4_000 }); + expect(usage.workers).toEqual([]); + expect(usage.series.slice(0, 7).map((point) => point.waiting)).toEqual([1, 1, 1, 1, 1, 1, 1]); + expect(usage.series.at(-1)?.waiting).toBe(1); + }); + + it("with fleet: true answers no probes field in totals, platforms or worker entries", () => { + const usage = computeUsage( + [capacity(T0 + 1_000, figures(0, 0, 2), undefined, { workerId: "w1" })], + WINDOW, + FLEET, + ); + + expect(usage.totals).not.toHaveProperty("probes"); + expect(usage.platforms.ios).not.toHaveProperty("probes"); + expect(usage.workers[0]).not.toHaveProperty("probes"); + expect(usage.totals.bySource).toHaveProperty("unknown", 0); + }); + + it("on a worker counts a lease.requested carrying fleetRequestId under probes, not requests, its grant under granted, its lease.declined under declined by the decline's timestamp, and gives it no wait sample", () => { + const usage = computeUsage( + [ + probe(T0 + 1_000, "p1"), + granted(T0 + 2_000, "p1", "l1", "booted", "gw:g1:a", { fleetRequestId: "f-p1" }), + released(T0 + 8_000, "l1"), + probe(T0 + 3_000, "p2"), + declined(T0 + 3_100, { requestId: "p2" }), + // A decline of a probe made before the window counts by its own timestamp. + declined(T0 + 4_000, { requestId: "old-probe" }), + requested(T0 + 5_000, "local", "agent-b"), + granted(T0 + 5_500, "local", "l2", "warm", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals).toMatchObject({ + declined: 2, + granted: 2, + probes: 2, + rejected: { total: 0 }, + requests: 1, + }); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, warm: 1 }); + expect(usage.totals.bySource).not.toHaveProperty("unknown"); + expect(usage.totals.wait).toEqual({ count: 1, max: 500, p50: 500, p95: 500 }); + expect(usage.totals.held).toMatchObject({ count: 1, p50: 6_000 }); + expect(usage.workers[0]).toMatchObject({ declined: 2, probes: 2, requests: 1 }); + expect(usage.platforms.ios).toMatchObject({ declined: 2, probes: 2 }); + }); + + it("on a worker gives a probe no turnaround sample and lists its gw: requester with its grant and held time and no request", () => { + const usage = computeUsage( + [ + probe(T0 + 1_000, "p1"), + granted(T0 + 2_000, "p1", "l1", "warm", "gw:g1:a", { fleetRequestId: "f-p1" }), + released(T0 + 8_000, "l1"), + // A probe that was declined is nothing of its requester's. + probe(T0 + 9_000, "p2", "gw:g1:b"), + declined(T0 + 9_100, { requestId: "p2" }), + requested(T0 + 10_000, "local", "agent-b"), + granted(T0 + 10_400, "local", "l2", "warm", "agent-b"), + released(T0 + 12_400, "l2"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.turnaround).toEqual({ count: 1, max: 2_400, p50: 2_400, p95: 2_400 }); + expect(usage.requesters).toEqual([ + { granted: 1, heldTotalMs: 2_000, id: "agent-b", rejected: 0, requests: 1 }, + { granted: 1, heldTotalMs: 6_000, id: "gw:g1:a", rejected: 0, requests: 0 }, + ]); + }); + + it("on a worker leaves a probe between its lease.requested and its grant out of series[].waiting", () => { + const usage = computeUsage( + [ + // A probe made before the window and still open at its start is not waiting either. + probe(T0 - 30_000, "old"), + probe(T0 + 10_000, "p1"), + granted(T0 + 100_000, "p1", "l1", "warm", "gw:g1:a", { fleetRequestId: "f-p1" }), + requested(T0 + 20_000, "local", "agent-b"), + granted(T0 + 130_000, "local", "l2", "warm", "agent-b"), + ], + WINDOW, + WORKER, + ); + + expect(usage.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 1, 0]); }); it("sets partial and coversFrom when the oldest event is inside the window", () => { @@ -825,30 +1004,6 @@ describe("computeUsage", () => { expect(usage.workers.map((worker) => worker.id)).toEqual(["local"]); }); - it("matches a fleet request to its worker's grant by the bare requester when no requesterPrefix is given", () => { - const usage = computeUsage( - [ - requested(T0 + 1_000, "gw-r1"), - at(T0 + 1_100, "request.dispatched", { - model: "m", - platform: "ios", - queuedMs: 0, - reason: "warm-hit", - requestId: "gw-r1", - requesterId: "agent-a", - stage: "s", - workerId: "w1", - }), - granted(T0 + 4_000, "w-r1", "l1", "warm", "agent-a", { workerId: "w1" }), - ], - WINDOW, - { fleet: true }, - ); - - expect(usage.totals.granted).toBe(1); - expect(usage.totals.wait).toEqual({ count: 1, max: 3_000, p50: 3_000, p95: 3_000 }); - }); - it("counts a grant whose source is none of warm, booted and provisioned under no source", () => { const usage = computeUsage( [ @@ -1011,14 +1166,13 @@ describe("computeUsage", () => { const usage = computeUsage( [ requested(T0 - 30_000, "old", "agent-a"), - // Answered before the window opened: not waiting at its start. + // Answered before the window opened (the history says so): not waiting at its start. requested(T0 - 40_000, "done", "agent-b"), - granted(T0 - 35_000, "done", "l0", "warm", "agent-b"), queueChanged(T0 - 30_000, 1), granted(T0 + 90_000, "old", "l1"), ], WINDOW, - WORKER, + { ...WORKER, answeredBefore: new Set(["done"]) }, ); expect(usage.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 0, 0]); @@ -1027,11 +1181,13 @@ describe("computeUsage", () => { expect(usage.totals.granted).toBe(0); }); - it("with fleet: true counts a request made before the window until a worker's relayed answer for its requester", () => { + it("with fleet: true counts a request made before the window until the gateway's request.granted for it", () => { const usage = computeUsage( [ requested(T0 - 30_000, "gw-old", "agent-a"), - granted(T0 + 90_000, "w-1", "l1", "warm", "gw:g1:agent-a", { workerId: "w1" }), + // A relayed grant for the same requester is not the gateway's answer. + granted(T0 + 30_000, "w-0", "l0", "warm", "gw:g1:agent-a", { workerId: "w1" }), + handed(T0 + 90_000, "gw-old", "w1", "l1"), ], WINDOW, FLEET, @@ -1041,6 +1197,34 @@ describe("computeUsage", () => { expect(usage.totals.granted).toBe(0); }); + it("with fleet: true counts a request made before the window as waiting at its start unless the history says it was answered by then", () => { + const events = [ + requested(T0 - 30_000, "gw-waits", "agent-a"), + requested(T0 - 20_000, "gw-done", "agent-b"), + ]; + + const usage = computeUsage(events, WINDOW, { ...FLEET, answeredBefore: new Set(["gw-done"]) }); + + expect(usage.series[0]?.waiting).toBe(1); + }); + + it("with fleet: true stops counting a request the gateway lost at a stop as waiting from the start that followed it, whether before the window or inside it", () => { + const before = computeUsage( + [requested(T0 - 60_000, "gw-lost"), restarted(T0 - 30_000)], + WINDOW, + FLEET, + ); + const inside = computeUsage( + [requested(T0 - 60_000, "gw-lost"), restarted(T0 + 90_000)], + WINDOW, + FLEET, + ); + + expect(before.series.slice(0, 3).map((point) => point.waiting)).toEqual([0, 0, 0]); + expect(inside.series.slice(0, 3).map((point) => point.waiting)).toEqual([1, 0, 0]); + expect(inside.totals.rejected.total).toBe(0); + }); + it("counts a request as waiting from the instant it is made until the instant it settles, per series point", () => { const usage = computeUsage( [ diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts index f00a4627..5377b56c 100644 --- a/src/core/usage/compute-usage.ts +++ b/src/core/usage/compute-usage.ts @@ -1,5 +1,4 @@ -import { byTimeThenSeq } from "../../bus/order.js"; -import type { EventEnvelope } from "../../bus/index.js"; +import { byTimeThenSeq, type EventEnvelope } from "../../bus/index.js"; import type { UsageFigures, UsageOutput } from "../../contract/index.js"; import { type CapacityStep, @@ -20,14 +19,14 @@ export interface UsageOptions { readonly bucketMs?: number; /** The `requestId` of every request made before the window; a rejection of one is in no count. */ readonly requestedBefore?: ReadonlySet; - /** The gateway rule (ADR 0016 §6): request facts from the gateway's own events, device facts - * from the events its workers relayed. */ + /** The `requestId` of every request answered, granted or rejected, before the window; a request + * made before it and not in this set was still waiting when it began. */ + readonly answeredBefore?: ReadonlySet; + /** The gateway rule (ADR 0016 §6 as ADR 0021 §5 amends it): request facts from the gateway's + * own events, device facts from the events its workers relayed. */ readonly fleet: boolean; /** Requester id to label, for the requesters the caller found a label for (ADR 0016 §7). */ readonly labels?: Readonly>; - /** What a gateway puts in front of a requester it forwards (`gw::`). A fleet request - * is matched with its worker's events by the requester carrying it. */ - readonly requesterPrefix?: string; /** The workers there are to report: a worker's own entry, a gateway's registry. A worker the * events name and this list does not is reported too. */ readonly workers?: readonly { readonly id: string; readonly label?: string | undefined }[]; @@ -52,9 +51,11 @@ export function seriesBucketMs(spanMs: number): number { /** * The usage figures for `window`, from the events of a history (ADR 0016). Pure: events in, numbers - * out. `events` are the envelopes the history holds for the window, with the step in force at its - * start for each capacity and queue timeline; any order, and any envelope outside the window is - * left out. + * out. `events` are the envelopes the history holds for the window, with the latest envelope at or + * before its start of `capacity.changed` and `queue.changed` (the step in force), of + * `lease.requested` for each requester (a request still waiting when the window began) and, on a + * gateway, the latest of its own `daemon.started`; any order, and any envelope outside the window + * is left out. */ export function computeUsage( events: readonly EventEnvelope[], @@ -64,15 +65,15 @@ export function computeUsage( const sorted = [...events].sort(byTimeThenSeq); const workers = options.workers ?? []; const read = readEvents(sorted, window, { + answeredBefore: options.answeredBefore ?? new Set(), fleet: options.fleet, requestedBefore: options.requestedBefore ?? new Set(), - requesterPrefix: options.requesterPrefix ?? "", self: workers[0]?.id ?? "local", }); const bucketMs = options.bucketMs ?? seriesBucketMs(window.to - window.from); const oldest = options.oldestTs ?? sorted[0]?.timestamp; const coversFrom = oldest === undefined ? window.from : Math.max(window.from, oldest); - const figures = (scope: Scope) => figuresFor(read, window, scope); + const figures = (scope: Scope) => figuresFor(read, window, scope, options.fleet); const labelOf = new Map(workers.map((worker) => [worker.id, worker.label])); const ids = new Set([...labelOf.keys(), ...read.workers]); return { @@ -113,40 +114,75 @@ function inScope( ); } -function figuresFor(read: ReadEvents, window: UsageWindow, scope: Scope): UsageFigures { +/** + * One scope's figures. On a worker a probe (a gateway's dispatch, ADR 0021 §5) is not a request: + * it is counted under `probes`, gives no wait, turnaround or rejection, and still counts as a grant + * with its source and held time. A gateway's answer has no `probes`, and its sources include + * `unknown`. + */ +function figuresFor( + read: ReadEvents, + window: UsageWindow, + scope: Scope, + fleet: boolean, +): UsageFigures { const requests = read.requests.filter((fact) => inScope(fact, scope)); const devices = read.devices.filter((fact) => inScope(fact, scope)); - const asked = requests.filter((fact) => fact.requestedAt !== undefined); + const asked = requests.filter((fact) => fact.requestedAt !== undefined && fact.probe !== true); + const probes = requests.filter((fact) => fact.requestedAt !== undefined && fact.probe === true); const grants = requests.flatMap((fact) => fact.outcome?.kind === "granted" ? [{ ...fact, outcome: fact.outcome }] : [], ); + const turnarounds = grants.filter((fact) => fact.probe !== true).flatMap(turnaroundOf); const rejected = requests.flatMap((fact) => - fact.outcome?.kind === "rejected" ? [fact.outcome.reason] : [], + fact.outcome?.kind === "rejected" && fact.probe !== true ? [fact.outcome.reason] : [], ); + const sources = sourcesOf(grants.map((fact) => fact.outcome.source)); return { ...deviceFigures(devices), - bySource: sourcesOf(grants.map((fact) => fact.outcome.source)), + bySource: { + booted: sources.booted, + provisioned: sources.provisioned, + ...(fleet ? { unknown: sources.unknown } : {}), + warm: sources.warm, + }, + declined: read.declines.filter((fact) => inScope(fact, scope)).length, granted: grants.length, - held: summarise(grants.flatMap((fact) => spanOf(fact.outcome.at, fact.endedAt))), + held: summarise(grants.flatMap((fact) => fact.heldMs ?? [])), queue: scope.queue === true ? queueOf(read.queue, window) : { meanDepth: null, peakDepth: null }, rejected: { byReason: countBy(rejected), total: rejected.length }, requests: asked.length, - turnaround: summarise(grants.flatMap((fact) => spanOf(fact.requestedAt, fact.endedAt))), + turnaround: summarise(turnarounds), utilisation: utilisationOf(read.capacity, window, scope), wait: summarise(asked.flatMap((fact) => spanOf(fact.requestedAt, fact.outcome?.at))), + ...(fleet ? {} : { probes: probes.length }), }; } +/** Wait plus held time: each by the clock of the host that saw it, so the two never mix. */ +function turnaroundOf(fact: RequestFact): number[] { + const wait = spanOf(fact.requestedAt, fact.outcome?.at)[0]; + return wait === undefined || fact.heldMs === undefined ? [] : [wait + fact.heldMs]; +} + /** The milliseconds from `start` to `end`, when both are known. */ function spanOf(start: number | undefined, end: number | undefined): number[] { return start === undefined || end === undefined ? [] : [end - start]; } -function sourcesOf(grantedSources: readonly string[]): UsageFigures["bySource"] { - const sources = { booted: 0, provisioned: 0, warm: 0 }; +type GrantSource = "booted" | "provisioned" | "unknown" | "warm"; + +function sourcesOf(grantedSources: readonly string[]): Record { + const sources = { booted: 0, provisioned: 0, unknown: 0, warm: 0 }; for (const source of grantedSources) { - if (source === "warm" || source === "booted" || source === "provisioned") sources[source] += 1; + if ( + source === "warm" || + source === "booted" || + source === "provisioned" || + source === "unknown" + ) + sources[source] += 1; } return sources; } @@ -244,6 +280,8 @@ function requestersOf( ): UsageOutput["requesters"] { const byId = new Map(); for (const fact of requests) { + // A probe that was not granted is nothing of its requester's: it was never its request. + if (fact.probe === true && fact.outcome?.kind !== "granted") continue; const entry = byId.get(fact.requester) ?? { granted: 0, heldTotalMs: 0, @@ -263,16 +301,17 @@ function requestersOf( type Requester = UsageOutput["requesters"][number]; -/** `entry` with `fact` counted in it: a request made, a grant and the time it was held, a rejection. */ +/** `entry` with `fact` counted in it: a request made, a grant and the time it was held, a rejection. + * A probe counts only as a grant and its held time. */ function countRequest(entry: Requester, fact: RequestFact): Requester { const { outcome } = fact; - const held = outcome?.kind === "granted" ? (spanOf(outcome.at, fact.endedAt)[0] ?? 0) : 0; + const probe = fact.probe === true; return { ...entry, granted: entry.granted + (outcome?.kind === "granted" ? 1 : 0), - heldTotalMs: entry.heldTotalMs + held, - rejected: entry.rejected + (outcome?.kind === "rejected" ? 1 : 0), - requests: entry.requests + (fact.requestedAt === undefined ? 0 : 1), + heldTotalMs: entry.heldTotalMs + (outcome?.kind === "granted" ? (fact.heldMs ?? 0) : 0), + rejected: entry.rejected + (outcome?.kind === "rejected" && !probe ? 1 : 0), + requests: entry.requests + (fact.requestedAt === undefined || probe ? 0 : 1), }; } @@ -283,7 +322,10 @@ function seriesOf(read: ReadEvents, window: UsageWindow, bucketMs: number): Usag ); const capacity = valuesAt(scopedSteps(read.capacity, {}), ends); const depths = valuesAt(read.queue, ends); - const waiting = waitingAt([...read.requests, ...read.carried], ends); + const waiting = waitingAt( + [...read.requests, ...read.carried].filter((fact) => fact.probe !== true), + ends, + ); return ends.map((at, index) => { const values = capacity[index] as readonly CapacityValue[]; const ram = ramOf(values, "used"); diff --git a/src/core/usage/read-events.test.ts b/src/core/usage/read-events.test.ts index 93a568aa..b68857f5 100644 --- a/src/core/usage/read-events.test.ts +++ b/src/core/usage/read-events.test.ts @@ -5,15 +5,15 @@ import { readEvents, type ReadOptions } from "./read-events.js"; const WINDOW = { from: 100, to: 200 }; const WORKER: ReadOptions = { + answeredBefore: new Set(), fleet: false, requestedBefore: new Set(), - requesterPrefix: "", self: "self", }; const FLEET: ReadOptions = { + answeredBefore: new Set(), fleet: true, requestedBefore: new Set(), - requesterPrefix: "gw:", self: "gateway", }; @@ -47,22 +47,49 @@ const capacityPayload = (extra: Record = {}) => ({ }); const request = (ts: number, requestId: string, requester = "a", extra = {}) => at(ts, "lease.requested", { requestId, requestSpec: { platform: "ios" }, requester, ...extra }); -const dispatch = (ts: number, requestId: string, requesterId = "a", workerId = "w1") => - at(ts, "request.dispatched", { requestId, requesterId, workerId }); +/** What a worker relays of the grant it made for a gateway's dispatch. */ const relayedGrant = (ts: number, extra: Record = {}) => at(ts, "lease.granted", { leaseId: "L", + requestId: "worker-side", requester: "gw:a", source: "warm", workerId: "w1", ...extra, }); -const relayedReject = (ts: number, extra: Record = {}) => +/** The gateway's own record that it handed a fleet request's grant to its caller. */ +const handed = (ts: number, requestId: string, extra: Record = {}) => + at(ts, "request.granted", { + leaseId: "gw-lease", + requestId, + worker: "w1", + workerLeaseId: "L", + ...extra, + }); +/** The gateway's own rejection of a fleet request. */ +const gatewayReject = ( + ts: number, + requestId: string, + reason: string, + extra: Record = {}, +) => at(ts, "lease.rejected", { - reason: "no-capacity", + reason, + requestId, + requestSpec: { platform: "ios" }, + requester: "a", + ...extra, + }); +const restarted = (ts: number, extra: Record = {}) => + at(ts, "daemon.started", { configSnapshot: {}, version: "0", ...extra }); +/** What a worker relays of its refusal of a gateway's dispatch. */ +const declined = (ts: number, extra: Record = {}) => + at(ts, "lease.declined", { + fleetRequestId: "r", + reason: "no-wait", + requestId: "worker-side", + requestSpec: { platform: "ios" }, requester: "gw:a", - requestId: "r", - workerId: "w1", ...extra, }); @@ -249,12 +276,7 @@ describe("requests on a worker", () => { request(130, "r2", "b"), at(140, "lease.granted", { requestId: "r2", requester: "b", source: "warm" }), ]); - expect(result.requests[0]?.outcome).toEqual({ - at: 120, - kind: "granted", - leaseId: "L1", - source: "", - }); + expect(result.requests[0]?.outcome).toEqual({ at: 120, kind: "granted", source: "" }); expect(result.requests[1]).not.toHaveProperty("outcome"); }); @@ -307,8 +329,8 @@ describe("requests on a worker", () => { ]); expect(result.requests).toEqual([ { - endedAt: 130, - outcome: { at: 120, kind: "granted", leaseId: "L", source: "" }, + heldMs: 10, + outcome: { at: 120, kind: "granted", source: "" }, platform: "ios", requestedAt: 110, requester: "a", @@ -323,7 +345,7 @@ describe("requests on a worker", () => { at(120, "lease.granted", { leaseId: "L", requestId: "r", source: "warm" }), at(130, "lease.expired", { leaseId: "L" }), ]); - expect(result.requests[0]).toMatchObject({ endedAt: 130, outcome: { source: "warm" } }); + expect(result.requests[0]).toMatchObject({ heldMs: 10, outcome: { source: "warm" } }); }); it("does not join an end of another lease id", () => { @@ -332,7 +354,7 @@ describe("requests on a worker", () => { at(120, "lease.granted", { leaseId: "L", requestId: "r" }), at(130, "lease.released", { leaseId: "other" }), ]); - expect(result.requests[0]).not.toHaveProperty("endedAt"); + expect(result.requests[0]).not.toHaveProperty("heldMs"); }); it("ignores a grant missing requestId or leaseId, and one outside the window", () => { @@ -351,12 +373,7 @@ describe("requests on a worker", () => { request(110, "r"), at(200, "lease.granted", { leaseId: "L", requestId: "r", source: "cold" }), ]); - expect(result.requests[0]?.outcome).toEqual({ - at: 200, - kind: "granted", - leaseId: "L", - source: "cold", - }); + expect(result.requests[0]?.outcome).toEqual({ at: 200, kind: "granted", source: "cold" }); }); it("ignores an end with no lease id, and one outside the window", () => { @@ -367,7 +384,7 @@ describe("requests on a worker", () => { at(100, "lease.released", { leaseId: "L" }), at(201, "lease.released", { leaseId: "L" }), ]); - expect(result.requests[0]).not.toHaveProperty("endedAt"); + expect(result.requests[0]).not.toHaveProperty("heldMs"); }); it("takes an end at exactly the window end", () => { @@ -376,7 +393,7 @@ describe("requests on a worker", () => { at(120, "lease.granted", { leaseId: "L", requestId: "r" }), at(200, "lease.released", { leaseId: "L" }), ]); - expect(result.requests[0]?.endedAt).toBe(200); + expect(result.requests[0]?.heldMs).toBe(80); }); it("outcome of a rejection of a seen request is the rejection with its reason", () => { @@ -576,18 +593,18 @@ describe("device facts", () => { }); describe("requests carried into the window", () => { - it("carries a request made exactly at the window's start, and treats an answer exactly then as answering it", () => { - expect(read([request(100, "edge")]).carried).toHaveLength(1); - const answered = read([ - request(90, "old"), - at(100, "lease.granted", { leaseId: "L", requestId: "old", requester: "a", source: "warm" }), - ]); - expect(answered.carried).toEqual([]); + const answered = (...ids: string[]): ReadOptions => ({ + ...WORKER, + answeredBefore: new Set(ids), }); - const grant = (ts: number, requestId: string, requester = "a") => at(ts, "lease.granted", { leaseId: `L${requestId}`, requestId, requester, source: "warm" }); + it("carries a request made exactly at the window's start, and treats an answer the history reports as answering it", () => { + expect(read([request(100, "edge")]).carried).toHaveLength(1); + expect(read([request(90, "old")], answered("old")).carried).toEqual([]); + }); + it("carries a request made before the window that nothing had answered, with its outcome inside it", () => { const result = read([request(90, "old"), grant(150, "old")]); expect(result.requests).toEqual([]); @@ -601,31 +618,95 @@ describe("requests carried into the window", () => { }); it("does not carry a request answered before the window, or one made inside it", () => { - const result = read([ - request(80, "done", "b"), - grant(90, "done", "b"), - request(85, "refused", "c"), - at(95, "lease.rejected", { reason: "timeout", requestId: "refused", requester: "c" }), - request(110, "inside"), - ]); + const result = read( + [request(80, "done", "b"), request(85, "refused", "c"), request(110, "inside")], + answered("done", "refused"), + ); expect(result.carried).toEqual([]); expect(result.requests).toHaveLength(1); }); it("carries only a requester's latest request made before the window", () => { - const result = read([request(80, "first"), grant(85, "first"), request(90, "second")]); + const result = read([request(80, "first"), request(90, "second")], answered("first")); expect(result.carried.map((fact) => fact.requestedAt)).toEqual([90]); }); - it("carries a gateway request answered only by a relayed answer inside the window, matched by the namespaced requester", () => { - const answered = read([request(90, "old"), relayedGrant(150)], FLEET); - expect(answered.carried).toEqual([ + it("matches a carried request to its answer by request id, not by requester", () => { + // Another request of the same requester was answered; this one was not. + const result = read([request(90, "waits")], answered("some-other-request")); + expect(result.carried.map((fact) => fact.requestedAt)).toEqual([90]); + }); + + it("does not carry a worker's probe, which never waits there", () => { + const result = read([request(90, "probe", "gw:a", { fleetRequestId: "f" })]); + expect(result.carried).toEqual([]); + }); + + it("carries a gateway request that nothing, on the gateway's own clock, had answered, with its outcome inside the window", () => { + const result = read([request(90, "old"), handed(150, "old")], FLEET); + expect(result.carried).toEqual([ expect.objectContaining({ outcome: expect.objectContaining({ at: 150 }), worker: "w1" }), ]); - const before = read([request(80, "old"), relayedGrant(90), request(95, "later")], FLEET); - expect(before.carried.map((fact) => fact.requestedAt)).toEqual([95]); - const first = read([request(80, "old"), relayedGrant(90)], FLEET); - expect(first.carried).toEqual([]); + expect(read([request(90, "old")], FLEET).carried).toHaveLength(1); + expect( + read([request(90, "old")], { ...FLEET, answeredBefore: new Set(["old"]) }).carried, + ).toEqual([]); + }); + + it("does not carry a gateway request that a restart before the window ended, and carries one a restart inside it ends", () => { + const ended = read([request(80, "old"), restarted(90)], FLEET); + expect(ended.carried).toEqual([]); + const later = read([request(80, "old"), restarted(150)], FLEET); + expect(later.carried).toEqual([ + expect.objectContaining({ + outcome: { at: 150, kind: "rejected", reason: "daemon-restarted" }, + requestedAt: 80, + }), + ]); + const after = read([restarted(70), request(80, "old")], FLEET); + expect(after.carried).toHaveLength(1); + }); +}); + +describe("a worker's probes and declines", () => { + it("marks a request carrying fleetRequestId as a probe and a request without one as no probe", () => { + const result = read([ + request(110, "p", "gw:a", { fleetRequestId: "f" }), + request(111, "l", "b"), + ]); + expect(result.requests[0]).toHaveProperty("probe", true); + expect(result.requests[1]).not.toHaveProperty("probe"); + }); + + it("counts a probe's grant and held time like any grant", () => { + const result = read([ + request(110, "p", "gw:a", { fleetRequestId: "f" }), + at(120, "lease.granted", { leaseId: "L", requestId: "p", requester: "gw:a", source: "warm" }), + at(150, "lease.released", { leaseId: "L" }), + ]); + expect(result.requests[0]).toMatchObject({ + heldMs: 30, + outcome: { at: 120, kind: "granted", source: "warm" }, + probe: true, + }); + }); + + it("counts the worker's own lease.declined under itself with the platform of requestSpec", () => { + const result = read([ + declined(110, { requestSpec: { platform: "android" } }), + declined(111, { requestSpec: {} }), + ]); + expect(result.declines).toEqual([ + { platform: "android", worker: "self" }, + { platform: undefined, worker: "self" }, + ]); + expect(result.requests).toEqual([]); + expect([...result.workers]).toEqual(["self"]); + }); + + it("counts a decline at exactly the window end and not at its start or after", () => { + const result = read([declined(100), declined(200), declined(201)]); + expect(result.declines).toHaveLength(1); }); }); @@ -664,32 +745,7 @@ describe("a gateway", () => { expect([...result.workers].sort()).toEqual(["w1", "w2", "w3"]); }); - it("ignores a lease end that no worker relayed", () => { - const result = read( - [ - request(110, "r"), - dispatch(111, "r"), - relayedGrant(112), - at(130, "lease.released", { leaseId: "L" }), - ], - FLEET, - ); - expect(result.requests[0]).not.toHaveProperty("endedAt"); - }); - - it("joins a relayed grant with the end of the same worker and lease id only", () => { - const events = (endWorker: string, endLease: string) => [ - request(110, "r"), - dispatch(111, "r"), - relayedGrant(112), - at(130, "lease.released", { leaseId: endLease, workerId: endWorker }), - ]; - expect(read(events("w1", "L"), FLEET).requests[0]?.endedAt).toBe(130); - expect(read(events("w2", "L"), FLEET).requests[0]).not.toHaveProperty("endedAt"); - expect(read(events("w1", "M"), FLEET).requests[0]).not.toHaveProperty("endedAt"); - }); - - it("gives a request with no dispatch and no relayed answer no worker and no outcome", () => { + it("gives a request with no outcome no worker and leaves it open", () => { const result = read([request(110, "r")], FLEET); expect(result.requests).toEqual([ { platform: "ios", requestedAt: 110, requester: "a", worker: undefined }, @@ -697,320 +753,297 @@ describe("a gateway", () => { expect([...result.workers]).toEqual([]); }); - it("takes the worker of the relayed answer for a request with no dispatch", () => { - const grant = read([request(110, "r"), relayedGrant(112, { workerId: "w5" })], FLEET); - expect(grant.requests[0]).toMatchObject({ - outcome: { kind: "granted" }, - requestedAt: 110, - worker: "w5", - }); - const rejection = read([request(110, "r"), relayedReject(112, { workerId: "w6" })], FLEET); - expect(rejection.requests[0]).toMatchObject({ outcome: { kind: "rejected" }, worker: "w6" }); - expect([...grant.workers]).toEqual(["w5"]); - }); - - it("names the worker the request was dispatched to, not the one that relayed the answer", () => { - const result = read( - [request(110, "r"), dispatch(111, "r", "a", "w1"), relayedGrant(112, { workerId: "w2" })], - FLEET, - ); - expect(result.requests[0]?.worker).toBe("w1"); - }); - - it("serves a request through its dispatch, and records the worker it was dispatched to", () => { - const result = read([request(110, "r"), dispatch(111, "r", "a", "w9")], FLEET); - expect(result.requests).toEqual([ - { platform: "ios", requestedAt: 110, requester: "a", worker: "w9" }, - ]); - expect([...result.workers]).toEqual(["w9"]); + it("does not take a relayed lease.requested as the gateway's own request", () => { + const result = read([request(110, "r", "a", { workerId: "w1" })], FLEET); + expect(result.requests).toEqual([]); }); - it("ignores a dispatch missing requestId or workerId, or outside the window", () => { - const result = read( - [ - request(110, "r"), - at(111, "request.dispatched", { requesterId: "a", workerId: "w1" }), - at(113, "request.dispatched", { requestId: "r", requesterId: "a" }), - dispatch(100, "r"), - dispatch(201, "r"), - ], - FLEET, - ); - expect(result.requests[0]?.worker).toBeUndefined(); - }); + describe("the outcome of a fleet request, by its request id", () => { + it("is the gateway's request.granted, by the gateway's clock, naming the worker it handed the grant from", () => { + const result = read( + [relayedGrant(105), request(110, "r"), handed(150, "r", { worker: "w1" })], + FLEET, + ); + expect(result.requests[0]).toMatchObject({ + outcome: { at: 150, kind: "granted" }, + requestedAt: 110, + worker: "w1", + }); + expect([...result.workers]).toEqual(["w1"]); + }); - it("takes a dispatch at exactly the window end", () => { - const result = read([request(110, "r"), dispatch(200, "r")], FLEET); - expect(result.requests[0]?.worker).toBe("w1"); - }); + it("is a request.granted at exactly the window end, and none missing a field or outside the window", () => { + expect(read([request(110, "r"), handed(200, "r")], FLEET).requests[0]).toHaveProperty( + "outcome", + ); + const none = read( + [ + request(110, "r"), + handed(120, "r", { requestId: undefined }), + handed(121, "r", { worker: undefined }), + handed(122, "r", { workerLeaseId: undefined }), + handed(100, "r"), + handed(201, "r"), + ], + FLEET, + ); + expect(none.requests[0]).not.toHaveProperty("outcome"); + }); - it("ignores a dispatch on a worker", () => { - const result = read([request(110, "r"), dispatch(111, "r")]); - expect(result.requests[0]?.worker).toBe("self"); - expect(result.requests[0]).not.toHaveProperty("outcome"); - }); + it("is not a request.granted that carries a workerId, which only a worker's relayed event does", () => { + const result = read([request(110, "r"), handed(120, "r", { workerId: "w1" })], FLEET); + expect(result.requests[0]).not.toHaveProperty("outcome"); + }); - it("counts a relayed grant with its source and lease id, and defaults both to empty", () => { - const withEnd = read( - [ - request(110, "r"), - dispatch(111, "r"), - relayedGrant(112, { leaseId: undefined, source: undefined }), - at(130, "lease.released", { leaseId: "", workerId: "w1" }), - ], - FLEET, - ); - expect(withEnd.requests[0]).toMatchObject({ - endedAt: 130, - outcome: { at: 112, kind: "granted", source: "" }, + it("is not a request.granted on a worker", () => { + const result = read([request(110, "r"), handed(120, "r")]); + expect(result.requests[0]).not.toHaveProperty("outcome"); }); - const plain = read( - [request(110, "r"), dispatch(111, "r"), relayedGrant(112, { source: "cold" })], - FLEET, - ); - expect(plain.requests[0]?.outcome).toEqual({ at: 112, kind: "granted", source: "cold" }); - }); - it("counts a relayed rejection with its reason, defaulting a missing reason to empty is impossible so it is ignored", () => { - const rejection = read([request(110, "r"), dispatch(111, "r"), relayedReject(112)], FLEET); - expect(rejection.requests[0]?.outcome).toEqual({ - at: 112, - kind: "rejected", - reason: "no-capacity", + it("is the gateway's own lease.rejected, under no worker, and a worker-failed one under its worker", () => { + const result = read( + [ + request(110, "timed-out"), + gatewayReject(120, "timed-out", "timeout", { worker: "w1" }), + request(111, "failed", "b"), + gatewayReject(121, "failed", "worker-failed", { + code: "INTERNAL", + requester: "b", + worker: "w2", + }), + ], + FLEET, + ); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason: "timeout" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: undefined, + }, + { + outcome: { at: 121, kind: "rejected", reason: "worker-failed" }, + platform: "ios", + requestedAt: 111, + requester: "b", + worker: "w2", + }, + ]); + expect([...result.workers]).toEqual(["w2"]); }); - const reasonless = read( - [request(110, "r"), dispatch(111, "r"), relayedReject(112, { reason: undefined })], - FLEET, - ); - expect(reasonless.requests[0]).not.toHaveProperty("outcome"); - }); - it("ignores a relayed grant or rejection without a requester, and one outside the window", () => { - const result = read( - [ - request(110, "r"), - dispatch(111, "r"), - relayedGrant(112, { requester: undefined }), - relayedReject(113, { requester: undefined }), - relayedGrant(100), - relayedGrant(201), - relayedReject(100), - relayedReject(201), - ], - FLEET, - ); - expect(result.requests[0]).not.toHaveProperty("outcome"); - }); + it("is never a rejection a worker relayed, nor a grant it relayed, nor a decline", () => { + const result = read( + [ + request(110, "r"), + gatewayReject(120, "r", "no-wait", { workerId: "w1" }), + relayedGrant(121, { requestId: "r" }), + declined(122, { requestId: "r" }), + request(130, "r2", "b"), + relayedGrant(131, { requester: "gw:b" }), + ], + FLEET, + ); + expect(result.requests.map((fact) => fact.outcome)).toEqual([undefined, undefined]); + }); - it("ignores a relayed grant of another requester, or of one without the gateway's prefix", () => { - const result = read( - [ - request(110, "r"), - dispatch(111, "r"), - relayedGrant(113, { requester: "gw:b" }), - relayedGrant(114, { requester: "a" }), - ], - FLEET, - ); - expect(result.requests[0]).not.toHaveProperty("outcome"); - }); + it("follows the request id, so another request of the same requester neither takes nor loses the outcome", () => { + const result = read( + [request(110, "r1"), request(112, "r2"), handed(120, "r2"), request(130, "r3")], + FLEET, + ); + expect(result.requests.map((fact) => fact.outcome?.kind)).toEqual([ + undefined, + "granted", + undefined, + ]); + }); - it("takes the first relayed answer, in event order", () => { - const result = read( - [ - request(110, "r"), - dispatch(111, "r"), - relayedGrant(112, { source: "first" }), - relayedGrant(113, { source: "second" }), - ], - FLEET, - ); - expect(result.requests[0]?.outcome).toMatchObject({ source: "first" }); - }); + it("is, with neither, the gateway's next own daemon.started, as a rejection daemon-restarted under no worker", () => { + const result = read([request(110, "r"), restarted(150), restarted(160)], FLEET); + expect(result.requests[0]).toEqual({ + outcome: { at: 150, kind: "rejected", reason: "daemon-restarted" }, + platform: "ios", + requestedAt: 110, + requester: "a", + worker: undefined, + }); + }); - it("counts a grant or a rejection that came before the dispatch", () => { - const grant = read([request(110, "r"), relayedGrant(111), dispatch(112, "r")], FLEET); - expect(grant.requests[0]?.outcome).toMatchObject({ kind: "granted" }); - expect(grant.requests[0]?.worker).toBe("w1"); - const rejection = read([request(110, "r"), relayedReject(111), dispatch(112, "r")], FLEET); - expect(rejection.requests[0]?.outcome).toMatchObject({ kind: "rejected" }); - }); + it("stays open past a daemon.started before the request, a relayed one, one after the window, and one on a worker", () => { + const open = (events: EventEnvelope[], options: ReadOptions = FLEET) => + read(events, options).requests[0]?.outcome; + expect(open([restarted(105), request(110, "r")])).toBeUndefined(); + expect(open([request(110, "r"), restarted(150, { workerId: "w1" })])).toBeUndefined(); + expect(open([request(110, "r"), restarted(201)])).toBeUndefined(); + expect(open([request(110, "r"), restarted(150)], WORKER)).toBeUndefined(); + }); - it("does not let a grant from before the request count", () => { - const result = read([relayedGrant(105), request(110, "r"), dispatch(111, "r")], FLEET); - expect(result.requests[0]).not.toHaveProperty("outcome"); + it("is the request.granted or lease.rejected, not the restart that follows it", () => { + const result = read( + [ + request(110, "g"), + request(111, "j", "b"), + restarted(150), + handed(160, "g"), + gatewayReject(161, "j", "timeout", { requester: "b" }), + ], + FLEET, + ); + expect(result.requests.map((fact) => fact.outcome?.kind)).toEqual(["granted", "rejected"]); + expect(result.requests[1]?.outcome).toMatchObject({ reason: "timeout" }); + }); }); - it("counts a rejection arriving right after the dispatch", () => { - const result = read([request(110, "r"), dispatch(111, "r"), relayedReject(112)], FLEET); - expect(result.requests[0]?.outcome).toMatchObject({ kind: "rejected" }); - }); + describe("the device facts of a fleet grant, from the worker it names", () => { + const events = (endWorker: string, endLease: string) => [ + request(110, "r"), + relayedGrant(112), + handed(115, "r"), + at(130, "lease.released", { leaseId: endLease, workerId: endWorker }), + ]; - it("cuts off relayed answers at the requester's next request", () => { - const result = read( - [ - request(110, "r1"), - dispatch(111, "r1"), - request(112, "r2"), - dispatch(113, "r2"), - relayedGrant(114, { leaseId: "L2", source: "late" }), - ], - FLEET, - ); - const [first, second] = result.requests; - expect(first).not.toHaveProperty("outcome"); - expect(second?.outcome).toMatchObject({ source: "late" }); - }); + it("take the source and the held time, by the worker's clock, from the relayed lease.granted and end of the worker and lease it names", () => { + const result = read(events("w1", "L"), FLEET); + expect(result.requests[0]).toMatchObject({ + heldMs: 18, + outcome: { at: 115, kind: "granted", source: "warm" }, + }); + }); - it("does not cut off at a next dispatch of the requester, only at its next request", () => { - const result = read( - [request(110, "r1"), dispatch(111, "r1"), dispatch(112, "r2"), relayedGrant(113)], - FLEET, - ); - expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); - }); + it("have no held time from an end of another worker or another lease, or one no worker relayed", () => { + expect(read(events("w2", "L"), FLEET).requests[0]).not.toHaveProperty("heldMs"); + expect(read(events("w1", "M"), FLEET).requests[0]).not.toHaveProperty("heldMs"); + const unrelayed = read( + [ + request(110, "r"), + relayedGrant(112), + handed(115, "r"), + at(130, "lease.released", { leaseId: "L" }), + ], + FLEET, + ); + expect(unrelayed.requests[0]).not.toHaveProperty("heldMs"); + }); - it("does not cut off at another requester's request", () => { - const result = read( - [request(110, "r1"), dispatch(111, "r1"), request(112, "r2", "b"), relayedGrant(113)], - FLEET, - ); - expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); - }); + it("look up each grant's end under the worker its own request.granted names, when two workers use one lease id", () => { + const result = read( + [ + request(110, "r1"), + request(111, "r2", "b"), + relayedGrant(112, { workerId: "w1" }), + relayedGrant(115, { requester: "gw:b", workerId: "w2" }), + handed(116, "r1", { worker: "w1" }), + handed(117, "r2", { worker: "w2" }), + at(130, "lease.released", { leaseId: "L", workerId: "w1" }), + at(150, "lease.released", { leaseId: "L", workerId: "w2" }), + ], + FLEET, + ); + expect(result.requests.map((fact) => [fact.worker, fact.heldMs])).toEqual([ + ["w1", 18], + ["w2", 35], + ]); + }); - it("does not cut off at a request with no requester or a dispatch with no requesterId", () => { - const result = read( - [ - request(110, "r1"), - dispatch(111, "r1"), - at(112, "lease.requested", { requestId: "x" }), - at(113, "request.dispatched", { requestId: "x", workerId: "w1" }), - relayedGrant(114), - ], - FLEET, - ); - expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); - }); + it("are the source unknown and no held time when no worker relayed the grant, whichever of its lease id or worker is missing", () => { + const missing = read([request(110, "r"), handed(115, "r")], FLEET); + expect(missing.requests[0]).toMatchObject({ + outcome: { at: 115, kind: "granted", source: "unknown" }, + worker: "w1", + }); + expect(missing.requests[0]).not.toHaveProperty("heldMs"); + const otherLease = read( + [request(110, "r"), relayedGrant(112, { leaseId: "M" }), handed(115, "r")], + FLEET, + ); + expect(otherLease.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + const otherWorker = read( + [request(110, "r"), relayedGrant(112, { workerId: "w2" }), handed(115, "r")], + FLEET, + ); + expect(otherWorker.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + }); - it("does not cut off at an earlier request of the same requester", () => { - const result = read( - [request(105, "r0"), request(110, "r1"), dispatch(111, "r1"), relayedGrant(112)], - FLEET, - ); - expect(result.requests.map((r) => r.outcome?.kind)).toEqual([undefined, "granted"]); - }); + it("default a relayed grant's missing source to empty, and ignore a relayed grant with no lease id or worker", () => { + const noSource = read( + [request(110, "r"), relayedGrant(112, { source: undefined }), handed(115, "r")], + FLEET, + ); + expect(noSource.requests[0]?.outcome).toMatchObject({ source: "" }); + const noLease = read( + [ + request(110, "r"), + relayedGrant(112, { leaseId: undefined }), + handed(115, "r", { workerLeaseId: "undefined" }), + ], + FLEET, + ); + expect(noLease.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + const noWorker = read( + [ + request(110, "r"), + relayedGrant(112, { workerId: undefined }), + handed(115, "r", { worker: "undefined" }), + ], + FLEET, + ); + expect(noWorker.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + }); - it("does not treat a relayed lease.requested as a request or a boundary", () => { - const result = read( - [ - request(110, "r1"), - dispatch(111, "r1"), - request(112, "r9", "a", { workerId: "w1" }), - relayedGrant(113), - ], - FLEET, - ); - expect(result.requests).toHaveLength(1); - expect(result.requests[0]?.outcome).toMatchObject({ kind: "granted" }); + it("ignore a relayed grant outside the window", () => { + const result = read( + [request(110, "r"), relayedGrant(100), relayedGrant(201), handed(115, "r")], + FLEET, + ); + expect(result.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + }); }); - it("settles a gateway's own rejection of a seen request without a worker", () => { + it("counts a relayed lease.declined under its worker and platform, one for each, and ignores a gateway's own and one outside the window", () => { const result = read( [ - request(110, "r"), - at(120, "lease.rejected", { reason: "no-capacity", requestId: "r", requester: "a" }), + declined(110, { workerId: "w1" }), + declined(111, { workerId: "w1" }), + declined(112, { requestSpec: { platform: "android" }, workerId: "w2" }), + declined(113), + declined(100, { workerId: "w1" }), + declined(201, { workerId: "w1" }), ], FLEET, ); - expect(result.requests).toEqual([ - { - outcome: { at: 120, kind: "rejected", reason: "no-capacity" }, - platform: "ios", - requestedAt: 110, - requester: "a", - worker: undefined, - }, + expect(result.declines).toEqual([ + { platform: "ios", worker: "w1" }, + { platform: "ios", worker: "w1" }, + { platform: "android", worker: "w2" }, ]); - }); - - it("counts a gateway's own refused-at-admission rejection with no worker", () => { - const result = read( - [at(120, "lease.rejected", { reason: "killed", requestId: "x", requester: "a" })], - FLEET, - ); - expect(result.requests).toEqual([ - { - outcome: { at: 120, kind: "rejected", reason: "killed" }, - platform: undefined, - requestedAt: undefined, - requester: "a", - worker: undefined, - }, - ]); - }); - - it("does not count a relayed rejection with a refused-at-admission reason as the gateway's own", () => { - const result = read([relayedReject(120, { reason: "killed", requestId: "x" })], FLEET); expect(result.requests).toEqual([]); + expect([...result.workers].sort()).toEqual(["w1", "w2"]); + }); + + it("counts a gateway's own refused-at-admission rejection with no worker, including lease-id-taken", () => { + for (const reason of ["killed", "already-leased", "lease-id-taken"]) { + const result = read( + [at(120, "lease.rejected", { reason, requestId: "x", requester: "a" })], + FLEET, + ); + expect(result.requests).toEqual([ + { + outcome: { at: 120, kind: "rejected", reason }, + platform: undefined, + requestedAt: undefined, + requester: "a", + worker: undefined, + }, + ]); + } }); - it("does not take a relayed lease.requested as the gateway's own request", () => { - const result = read([request(110, "r", "a", { workerId: "w1" })], FLEET); + it("does not count a relayed rejection with a refused-at-admission reason as the gateway's own", () => { + const result = read([gatewayReject(120, "x", "killed", { workerId: "w1" })], FLEET); expect(result.requests).toEqual([]); }); - - it("names the worker from a dispatch that carries no requesterId, and records none for one with no workerId", () => { - const noRequester = read( - [request(110, "r"), at(111, "request.dispatched", { requestId: "r", workerId: "w1" })], - FLEET, - ); - expect(noRequester.requests[0]?.worker).toBe("w1"); - const noWorker = read( - [request(110, "r"), at(111, "request.dispatched", { requestId: "r", requesterId: "a" })], - FLEET, - ); - expect(noWorker.requests[0]?.worker).toBeUndefined(); - expect([...noWorker.workers]).toEqual([]); - }); - - it("keeps a grant answering a request before the requester's next request", () => { - const result = read( - [ - request(110, "r1"), - dispatch(111, "r1"), - relayedGrant(112, { source: "mine" }), - request(113, "r2"), - dispatch(114, "r2"), - ], - FLEET, - ); - expect(result.requests[0]?.outcome).toMatchObject({ source: "mine" }); - expect(result.requests[1]).not.toHaveProperty("outcome"); - }); - - it("does not match a worker-less relayed grant to a dispatch to a worker named undefined", () => { - const result = read( - [ - request(110, "r"), - dispatch(111, "r", "a", "undefined"), - at(112, "lease.granted", { leaseId: "L", requester: "gw:a" }), - ], - FLEET, - ); - expect(result.requests[0]).not.toHaveProperty("outcome"); - }); - - it("does not match a relayed grant with no requester to a requester named undefined", () => { - const result = read( - [ - request(110, "r", "undefined"), - dispatch(111, "r", "undefined"), - relayedGrant(112, { requester: undefined }), - ], - { ...FLEET, requesterPrefix: "" }, - ); - expect(result.requests[0]).not.toHaveProperty("outcome"); - }); }); describe("keys that look like missing ids", () => { @@ -1020,7 +1053,7 @@ describe("keys that look like missing ids", () => { at(120, "lease.granted", { leaseId: "undefined", requestId: "r" }), at(130, "lease.released", {}), ]); - expect(result.requests[0]).not.toHaveProperty("endedAt"); + expect(result.requests[0]).not.toHaveProperty("heldMs"); }); it("does not remember a platform for a provisioning with no device id under the name undefined", () => { diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index cb2ad3bd..07281080 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -3,9 +3,9 @@ import type { UsageFigures } from "../../contract/index.js"; import type { Step } from "./timeline.js"; /** - * Reads a window's events into the facts the figures count (ADR 0016 §2, §6): one fact for each - * request, one for each device event that counts, and the capacity and queue steps. Nothing here - * adds a number up. + * Reads a window's events into the facts the figures count (ADR 0016 §2, ADR 0021 §5): one fact + * for each request, one for each device event that counts, one for each decline, and the capacity + * and queue steps. Nothing here adds a number up. */ export type Platform = "ios" | "android"; @@ -17,16 +17,21 @@ export interface RequestFact { readonly worker: string | undefined; /** `undefined` for a rejection refused before the request was stored, which is no request. */ readonly requestedAt: number | undefined; + /** On a worker: a gateway's dispatch (ADR 0021 §1), which gives no wait, turnaround, request or + * rejection. Absent otherwise. */ + readonly probe?: true; readonly outcome?: Granted | Rejected; - /** When the lease ended, if the window saw it. */ - readonly endedAt?: number; + /** How long the lease was held, when the window saw both its grant and its end. Both are the + * clock of the host that granted it, so on a gateway it is not `outcome.at` to an end. */ + readonly heldMs?: number; } -type RequestFactBase = Pick; +type RequestFactBase = Pick; interface Granted { readonly kind: "granted"; readonly at: number; + /** `unknown` on a gateway for a grant whose worker's own `lease.granted` was not relayed. */ readonly source: string; } @@ -44,6 +49,12 @@ export interface DeviceFact { readonly value: number | string; } +/** One `lease.declined` (ADR 0021 §2): a worker's refusal of a gateway dispatch. Not a request. */ +export interface DeclineFact { + readonly worker: string; + readonly platform: Platform | undefined; +} + export interface CapacityValue { readonly used: number; readonly max: number; @@ -59,6 +70,7 @@ export interface ReadEvents { /** Requests made before the window and still open when it began; only the series counts them. */ readonly carried: readonly RequestFact[]; readonly devices: readonly DeviceFact[]; + readonly declines: readonly DeclineFact[]; readonly capacity: readonly CapacityStep[]; readonly queue: readonly Step[]; readonly workers: ReadonlySet; @@ -68,10 +80,10 @@ export interface ReadOptions { readonly fleet: boolean; /** The `requestId` of every request made before the window. */ readonly requestedBefore: ReadonlySet; + /** The `requestId` of every request answered before the window: granted or rejected. */ + readonly answeredBefore: ReadonlySet; /** The id a worker's own events are attributed to. */ readonly self: string; - /** What a gateway puts in front of a requester it forwards. */ - readonly requesterPrefix: string; } export interface UsageWindow { @@ -83,10 +95,10 @@ type Payload = Readonly>; type Incident = keyof UsageFigures["incidents"]; /** A rejection with no `lease.requested` anywhere in the history read was refused before the - * request was stored; only these two reasons are ever given before admission. One whose request + * request was stored; only these reasons are ever given before admission. One whose request * is in the history follows that request: counted with it, or in no count when it was made before * the window. */ -const REFUSED_AT_ADMISSION = new Set(["already-leased", "killed"]); +const REFUSED_AT_ADMISSION = new Set(["already-leased", "killed", "lease-id-taken"]); /** The device events that count as an incident, by what the figure calls them. */ const INCIDENT_OF: Readonly> = { "device.quarantine-abandoned": "lost", @@ -115,21 +127,12 @@ const platformOf = (value: unknown): Platform | undefined => const objectOf = (value: unknown): Payload | undefined => typeof value === "object" && value !== null ? (value as Payload) : undefined; -/** A relayed grant or rejection, kept for the request that will claim it. */ -type RelayedAnswer = - | { readonly kind: "granted"; readonly source: string; readonly leaseId: string } - | { readonly kind: "rejected"; readonly reason: string }; -type Relayed = { - readonly index: number; - readonly at: number; - readonly worker: string; -} & RelayedAnswer; - interface Request { readonly index: number; readonly at: number; readonly requester: string; readonly platform: Platform | undefined; + readonly probe: boolean; } /** What one event brings to a handler. */ @@ -144,6 +147,13 @@ interface Seen { readonly within: boolean; } +type OwnRejection = Omit & { + readonly requester: string; + readonly platform: Platform | undefined; + /** The worker a `worker-failed` rejection names. */ + readonly worker: string | undefined; +}; + export function readEvents( sorted: readonly EventEnvelope[], window: UsageWindow, @@ -156,39 +166,40 @@ export function readEvents( class Reader { readonly #devices: DeviceFact[] = []; + readonly #declines: DeclineFact[] = []; readonly #capacity: CapacityStep[] = []; readonly #queue: Step[] = []; readonly #workers = new Set(); readonly #devicePlatform = new Map(); readonly #requested = new Map(); - /** The latest request of each requester made before the window, and where each was answered. */ + /** The latest request of each requester made before the window. */ readonly #carriedRequests = new Map(); - readonly #answeredBefore = new Map(); - readonly #ownRejections = new Map< + readonly #ownRejections = new Map(); + /** A worker's own grants by request id: the lease each served. */ + readonly #grants = new Map(); + /** A gateway's `request.granted` by request id (ADR 0021 §4). */ + readonly #handed = new Map< string, - Omit & { - readonly requester: string; - readonly platform: Platform | undefined; - } + { readonly at: number; readonly worker: string; readonly workerLeaseId: string } >(); - readonly #grants = new Map(); + /** The grants workers relayed to a gateway, by worker and the worker's lease id. */ + readonly #relayedGrants = new Map(); readonly #ends = new Map(); - /** The worker each request was dispatched to, by request id. */ - readonly #dispatches = new Map(); - readonly #boundaries = new Map(); - /** Relayed grants and rejections by the namespaced requester they are for. */ - readonly #relayed = new Map(); + /** Where the gateway's own `daemon.started` events fall, in event order (ADR 0021 §5). */ + readonly #restarts: { readonly index: number; readonly at: number }[] = []; readonly #handlers: Record void> = { "capacity.changed": (seen) => this.#capacityChanged(seen), + "daemon.started": (seen) => this.#daemonStarted(seen), "device.provisioned": (seen) => this.#provisioned(seen), "device.ready": (seen) => this.#ready(seen), + "lease.declined": (seen) => this.#declined(seen), "lease.expired": (seen) => this.#ended(seen), "lease.granted": (seen) => this.#granted(seen), "lease.rejected": (seen) => this.#rejected(seen), "lease.released": (seen) => this.#ended(seen), "lease.requested": (seen) => this.#leaseRequested(seen), "queue.changed": (seen) => this.#queueChanged(seen), - "request.dispatched": (seen) => this.#dispatched(seen), + "request.granted": (seen) => this.#requestGranted(seen), }; constructor( @@ -213,9 +224,10 @@ class Reader { finish(): ReadEvents { return { capacity: this.#capacity, + carried: this.#carriedOpen(), + declines: this.#declines, devices: this.#devices, queue: this.#queue, - carried: this.#carriedOpen(), requests: [...this.#requests(), ...this.#refusedAtAdmission()], workers: this.#workers, }; @@ -244,105 +256,80 @@ class Reader { this.#queue.push({ at: seen.event.timestamp, key: "queue", value: depth }); } - #boundary(requester: string | undefined, index: number): void { - if (requester === undefined) return; - this.#boundaries.set(requester, [...this.#boundariesOf(requester), index]); - } - - #boundariesOf(requester: string): readonly number[] { - return this.#boundaries.get(requester) ?? []; - } - #leaseRequested(seen: Seen): void { if (!seen.own) return; const requestId = text(seen.payload, "requestId"); const requester = text(seen.payload, "requester"); - this.#boundary(requester, seen.index); if (requestId === undefined || requester === undefined) return; - if (seen.event.timestamp <= this.window.from) { - this.#carriedRequests.set(requester, { - at: seen.event.timestamp, - index: seen.index, - platform: requestPlatform(seen.payload), - requestId, - requester, - }); - } - if (!seen.within) return; - this.#requested.set(requestId, { + const request: Request = { at: seen.event.timestamp, index: seen.index, platform: requestPlatform(seen.payload), + probe: !this.options.fleet && text(seen.payload, "fleetRequestId") !== undefined, requester, - }); + }; + if (seen.event.timestamp <= this.window.from) { + this.#carriedRequests.set(requester, { ...request, requestId }); + } + if (seen.within) this.#requested.set(requestId, request); + } + + /** A gateway's restart ends every fleet request still without an outcome (ADR 0021 §5). */ + #daemonStarted(seen: Seen): void { + if (!this.options.fleet || !seen.own || seen.event.timestamp > this.window.to) return; + this.#restarts.push({ at: seen.event.timestamp, index: seen.index }); } - #dispatched(seen: Seen): void { + /** A gateway's own `request.granted` is its fleet request's grant. */ + #requestGranted(seen: Seen): void { const requestId = text(seen.payload, "requestId"); - const worker = text(seen.payload, "workerId"); - if (!this.options.fleet || !seen.within) return; - if (requestId === undefined || worker === undefined) return; - this.#dispatches.set(requestId, worker); + const worker = text(seen.payload, "worker"); + const workerLeaseId = text(seen.payload, "workerLeaseId"); + if (!this.options.fleet || !seen.own || !seen.within) return; + if (requestId === undefined || worker === undefined || workerLeaseId === undefined) return; + this.#handed.set(requestId, { at: seen.event.timestamp, worker, workerLeaseId }); } - #relay(seen: Seen, relayed: RelayedAnswer): void { - const requester = text(seen.payload, "requester"); - if (requester === undefined || seen.worker === undefined) return; - const entry = { - ...relayed, - at: seen.event.timestamp, - index: seen.index, - worker: seen.worker, - } as Relayed; - this.#relayed.set(requester, [...(this.#relayed.get(requester) ?? []), entry]); + /** A worker's `lease.declined`, or one a gateway's worker relayed: an event, never a request. */ + #declined(seen: Seen): void { + if (!seen.within || seen.worker === undefined) return; + if (this.options.fleet && seen.own) return; + this.#declines.push({ platform: requestPlatform(seen.payload), worker: seen.worker }); + this.#workers.add(seen.worker); } #rejected(seen: Seen): void { const requestId = text(seen.payload, "requestId"); const reason = text(seen.payload, "reason"); - if (this.#answeredEarlier(seen)) return; - if (!seen.within || requestId === undefined || reason === undefined) return; - const requester = text(seen.payload, "requester"); - if (seen.own) { - if (requester === undefined) return; - this.#ownRejections.set(requestId, { - at: seen.event.timestamp, - platform: requestPlatform(seen.payload), - reason, - requester, - }); - } else { - this.#relay(seen, { kind: "rejected", reason }); - } - } - - /** Notes where a requester was last answered before the window; true for such an answer. */ - #answeredEarlier(seen: Seen): boolean { - if (seen.event.timestamp > this.window.from) return false; const requester = text(seen.payload, "requester"); - if (requester !== undefined) this.#answeredBefore.set(requester, seen.index); - return true; + if (!seen.within || !seen.own) return; + if (requestId === undefined || reason === undefined || requester === undefined) return; + this.#ownRejections.set(requestId, { + at: seen.event.timestamp, + platform: requestPlatform(seen.payload), + reason, + requester, + worker: reason === "worker-failed" ? text(seen.payload, "worker") : undefined, + }); } #granted(seen: Seen): void { - if (this.#answeredEarlier(seen) || !seen.within) return; + if (!seen.within) return; + const leaseId = text(seen.payload, "leaseId"); + if (leaseId === undefined) return; + const source = text(seen.payload, "source") ?? ""; if (this.options.fleet) { - this.#relay(seen, { - kind: "granted", - leaseId: text(seen.payload, "leaseId") ?? "", - source: text(seen.payload, "source") ?? "", - }); + if (seen.worker !== undefined) { + this.#relayedGrants.set(this.#key(seen.worker, leaseId), { + at: seen.event.timestamp, + source, + }); + } return; } const requestId = text(seen.payload, "requestId"); - const leaseId = text(seen.payload, "leaseId"); - if (requestId === undefined || leaseId === undefined) return; - this.#grants.set(requestId, { - at: seen.event.timestamp, - kind: "granted", - leaseId, - source: text(seen.payload, "source") ?? "", - }); + if (requestId === undefined) return; + this.#grants.set(requestId, { at: seen.event.timestamp, kind: "granted", leaseId, source }); } #ended(seen: Seen): void { @@ -408,74 +395,90 @@ class Reader { return facts; } - /** The requests made before the window that nothing had answered when it began. */ + /** + * The requests made before the window that nothing had answered when it began, and that no + * gateway restart had ended. A worker's probe never waits there (ADR 0021 §5). + */ #carriedOpen(): RequestFact[] { const facts: RequestFact[] = []; - for (const [requester, request] of this.#carriedRequests) { - const answered = [requester, `${this.options.requesterPrefix}${requester}`].some( - (key) => (this.#answeredBefore.get(key) ?? -1) > request.index, - ); - if (!answered) facts.push(this.#factFor(request.requestId, request)); + for (const request of this.#carriedRequests.values()) { + if (request.probe || this.options.answeredBefore.has(request.requestId)) continue; + const fact = this.#factFor(request.requestId, request); + if (fact.outcome === undefined || fact.outcome.at > this.window.from) facts.push(fact); } return facts; } #factFor(requestId: string, request: Request): RequestFact { - const rejection = this.#ownRejections.get(requestId); - const own = this.options.fleet ? undefined : this.options.self; const base = { platform: request.platform, requestedAt: request.at, requester: request.requester, + ...(request.probe ? { probe: true as const } : {}), }; - if (rejection !== undefined) { - const outcome: Rejected = { at: rejection.at, kind: "rejected", reason: rejection.reason }; - return { ...base, outcome, worker: own }; - } return this.options.fleet ? this.#fleetFact(base, requestId, request) : this.#workerFact(base, requestId, this.options.self); } #workerFact(base: RequestFactBase, requestId: string, own: string): RequestFact { + const rejection = this.#ownRejections.get(requestId); + if (rejection !== undefined) { + return { ...base, outcome: rejectedOf(rejection), worker: own }; + } const grant = this.#grants.get(requestId); - return grant === undefined - ? { ...base, worker: own } - : this.#withEnd({ ...base, outcome: grant, worker: own }, grant.leaseId, own); + if (grant === undefined) return { ...base, worker: own }; + const outcome: Granted = { at: grant.at, kind: "granted", source: grant.source }; + return this.#withHeld( + { ...base, outcome, worker: own }, + this.#heldFor(own, grant.leaseId, grant), + ); } - /** A fleet request's outcome is the worker's relayed answer for its requester (ADR 0016 §6). */ + /** + * A fleet request's outcome is the gateway's own `request.granted` or `lease.rejected` for its + * request id; with neither, the gateway's next start ends it; else it is open (ADR 0021 §5). + */ #fleetFact(base: RequestFactBase, requestId: string, request: Request): RequestFact { - const dispatched = this.#dispatches.get(requestId); - const relayed = this.#fleetOutcome(request); - const worker = dispatched ?? relayed?.worker; - if (worker !== undefined) this.#workers.add(worker); - if (relayed === undefined) return { ...base, worker }; - if (relayed.kind === "rejected") { - const outcome: Rejected = { at: relayed.at, kind: "rejected", reason: relayed.reason }; - return { ...base, outcome, worker }; + const rejection = this.#ownRejections.get(requestId); + if (rejection !== undefined) { + if (rejection.worker !== undefined) this.#workers.add(rejection.worker); + return { ...base, outcome: rejectedOf(rejection), worker: rejection.worker }; } - const outcome: Granted = { at: relayed.at, kind: "granted", source: relayed.source }; - return this.#withEnd({ ...base, outcome, worker }, relayed.leaseId, relayed.worker); + const handed = this.#handed.get(requestId); + if (handed !== undefined) return this.#grantedFleetFact(base, handed); + const restart = this.#restarts.find((candidate) => candidate.index > request.index); + if (restart === undefined) return { ...base, worker: undefined }; + const outcome: Rejected = { at: restart.at, kind: "rejected", reason: "daemon-restarted" }; + return { ...base, outcome, worker: undefined }; + } + + /** The grant source and the held time are the worker's own facts, by the worker's clock. */ + #grantedFleetFact( + base: RequestFactBase, + handed: { readonly at: number; readonly worker: string; readonly workerLeaseId: string }, + ): RequestFact { + this.#workers.add(handed.worker); + const relayed = this.#relayedGrants.get(this.#key(handed.worker, handed.workerLeaseId)); + const outcome: Granted = { + at: handed.at, + kind: "granted", + source: relayed === undefined ? "unknown" : relayed.source, + }; + const fact = { ...base, outcome, worker: handed.worker }; + return relayed === undefined + ? fact + : this.#withHeld(fact, this.#heldFor(handed.worker, handed.workerLeaseId, relayed)); } - #withEnd(fact: RequestFact, leaseId: string, worker: string): RequestFact { + /** How long a lease was held, from its grant to the end the window saw. */ + #heldFor(worker: string, leaseId: string, grant: { readonly at: number }): number | undefined { const endedAt = this.#ends.get(this.#key(worker, leaseId)); - return endedAt === undefined ? fact : { ...fact, endedAt }; + return endedAt === undefined ? undefined : endedAt - grant.at; } - /** - * The first relayed grant or rejection for the request's namespaced requester at or after the - * gateway's own `lease.requested` for it and before that requester's next one (ADR 0016 §6). A - * grant may come before `request.dispatched`: a warm device is granted before the dispatch - * answer reaches the gateway. - */ - #fleetOutcome(request: Request): Relayed | undefined { - const next = this.#boundariesOf(request.requester).find((index) => index > request.index); - return (this.#relayed.get(`${this.options.requesterPrefix}${request.requester}`) ?? []).find( - (candidate) => - candidate.index >= request.index && (next === undefined || candidate.index < next), - ); + #withHeld(fact: RequestFact, heldMs: number | undefined): RequestFact { + return heldMs === undefined ? fact : { ...fact, heldMs }; } /** A gateway's or worker's own rejection of a request the window did not see made, when its @@ -486,7 +489,7 @@ class Reader { if (this.#requested.has(requestId) || this.options.requestedBefore.has(requestId)) continue; if (!REFUSED_AT_ADMISSION.has(rejection.reason)) continue; facts.push({ - outcome: { at: rejection.at, kind: "rejected", reason: rejection.reason }, + outcome: rejectedOf(rejection), platform: rejection.platform, requestedAt: undefined, requester: rejection.requester, @@ -497,6 +500,12 @@ class Reader { } } +const rejectedOf = (rejection: OwnRejection): Rejected => ({ + at: rejection.at, + kind: "rejected", + reason: rejection.reason, +}); + function requestPlatform(payload: Payload): Platform | undefined { return platformOf(objectOf(payload.requestSpec)?.platform); } diff --git a/src/core/usage/usage-reader.test.ts b/src/core/usage/usage-reader.test.ts index 6bdb3069..407cba21 100644 --- a/src/core/usage/usage-reader.test.ts +++ b/src/core/usage/usage-reader.test.ts @@ -23,7 +23,11 @@ function requested( } /** A history that records what it was asked for and answers what the test sets. */ -function history(events: readonly EventEnvelope[] = [], oldestTs: number | undefined = undefined) { +function history( + events: readonly EventEnvelope[] = [], + oldestTs: number | undefined = undefined, + before: { requested?: readonly string[]; answered?: readonly string[] } = {}, +) { const state = { asked: [] as { sinceTs: number; carry: readonly EventName[] }[], events, @@ -34,7 +38,12 @@ function history(events: readonly EventEnvelope[] = [], oldestTs: number | undef latestId: () => state.newest, read: async (input) => { state.asked.push(input); - return { events: state.events, oldestTs: state.oldestTs, requestedBefore: new Set() }; + return { + answeredBefore: new Set(before.answered), + events: state.events, + oldestTs: state.oldestTs, + requestedBefore: new Set(before.requested), + }; }, }; return { source, state }; @@ -63,7 +72,7 @@ const answer = (result: Awaited>) => { }; describe("UsageReader", () => { - it("rounds the window down to the bucket on both sides, so it never ends in the future, and reads the history from its start, with the step and the request events carried", async () => { + it("rounds the window down to the bucket on both sides, so neither end moves later than asked, and reads the history from its start, with the steps, the requests and a gateway start carried", async () => { const { source, state } = history(); const { usage } = reader(source); @@ -73,18 +82,47 @@ describe("UsageReader", () => { expect(result.bucketMs).toBe(MINUTE); expect(state.asked).toEqual([ { - carry: [ - "capacity.changed", - "lease.granted", - "lease.rejected", - "lease.requested", - "queue.changed", - ], + carry: ["capacity.changed", "daemon.started", "lease.requested", "queue.changed"], sinceTs: T0, }, ]); }); + it("computes with the series bucket picked from the span asked, not from the rounded window", async () => { + const { usage } = reader(history().source); + + const result = answer(await usage.get({ from: T0 + 1_000, to: T0 + 200 * MINUTE + 31_000 })); + + expect(result.window).toEqual({ from: T0, to: T0 + 200 * MINUTE }); + expect(result.bucketMs).toBe(5 * MINUTE); + expect(result.series).toHaveLength(40); + }); + + it("counts no rejection of a request the history says was made before the window, and one of a request it does not say so about", async () => { + const refused = { + event: "lease.rejected", + id: "evt_rej", + module: "test", + payload: { reason: "already-leased", requestId: "old", requester: "a" }, + seq: 1, + timestamp: T0 + 1_000, + } as unknown as EventEnvelope; + const known = reader(history([refused], undefined, { requested: ["old"] }).source).usage; + const unknown = reader(history([refused]).source).usage; + + expect(answer(await known.get({ from: T0, to: T0 + HOUR })).totals.rejected.total).toBe(0); + expect(answer(await unknown.get({ from: T0, to: T0 + HOUR })).totals.rejected.total).toBe(1); + }); + + it("counts a request made before the window as waiting at its start unless the history says it was answered by then", async () => { + const events = [requested(T0 - 30_000, "a", "waits"), requested(T0 - 20_000, "b", "done")]; + const { usage } = reader(history(events, undefined, { answered: ["done"] }).source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result.series[0]?.waiting).toBe(1); + }); + it("leaves a window that is already on bucket edges as it is", async () => { const { source } = history(); const { usage } = reader(source); diff --git a/src/core/usage/usage-reader.ts b/src/core/usage/usage-reader.ts index 7076dc6e..69091ca5 100644 --- a/src/core/usage/usage-reader.ts +++ b/src/core/usage/usage-reader.ts @@ -2,13 +2,13 @@ import type { EventEnvelope, EventName } from "../../bus/index.js"; import type { UsageOutput } from "../../contract/index.js"; import { computeUsage, seriesBucketMs, type UsageWindow } from "./compute-usage.js"; -/** The two timelines whose step in force at a window's start is read with it (ADR 0016 §3). */ -/** The events whose latest envelope before the window the figures need: the two steps, and the - * latest request and answer of each requester, which tell what was still waiting at its start. */ -const STEP_EVENTS: readonly EventName[] = [ +/** The events whose latest envelope at or before the window's start is read with it: the two + * steps in force (ADR 0016 §3), the latest request of each requester, which with the ids of the + * requests answered by then tells what was still waiting at the start, and a gateway's latest + * start, which ends the fleet requests it lost (ADR 0021 §5). */ +const CARRIED_EVENTS: readonly EventName[] = [ "capacity.changed", - "lease.granted", - "lease.rejected", + "daemon.started", "lease.requested", "queue.changed", ]; @@ -20,6 +20,8 @@ export interface UsageHistory { readonly oldestTs: number | undefined; /** The `requestId` of every `lease.requested` at or before `sinceTs`. */ readonly requestedBefore: ReadonlySet; + /** The `requestId` of every request answered at or before `sinceTs`. */ + readonly answeredBefore: ReadonlySet; }>; /** The id of the event published last; the memo is good while it has not moved. */ latestId(): string | undefined; @@ -31,8 +33,9 @@ export interface UsageReaderOptions { readonly tokenLabels: () => Promise>; /** The workers to report: a worker's own entry, a gateway's registry. Read on each answer. */ readonly workers: () => readonly { readonly id: string; readonly label?: string | undefined }[]; - /** Set on a gateway: the figures are the fleet's (ADR 0016 §6), and this is the prefix it - * stamps on the requesters it forwards (`gw::`). */ + /** Set on a gateway: the figures are the fleet's (ADR 0016 §6, ADR 0021 §5), and this is the + * prefix it stamps on the requesters it forwards (`gw::`), which is taken off before a + * token label is looked up. */ readonly fleet?: { readonly requesterPrefix: string }; } @@ -42,8 +45,9 @@ export type UsageResult = { readonly usage: UsageOutput } | { readonly oldestTs: /** * Answers `usage.get` for a worker or a gateway from the same code (ADR 0016): reads the history * for the window, hands it to `computeUsage`, joins token labels. The window is rounded down to the - * series bucket, both ends, so it never reaches a time that has not come, and the last answer is kept by that window and the newest event, so a caller that - * asks again within the bucket while nothing happened reads nothing. + * series bucket, both ends, so neither end moves later than asked, and the last answer is kept by + * that window and the newest event, so a caller that asks again within the bucket while nothing + * happened reads nothing. */ export class UsageReader { #memo: { readonly key: string; readonly usage: UsageOutput } | undefined; @@ -52,7 +56,7 @@ export class UsageReader { async get(asked: UsageWindow): Promise { const bucketMs = seriesBucketMs(asked.to - asked.from); - // Down to the bucket on both sides (ADR 0016): the answer never reaches a time not yet come. + // Down to the bucket on both sides (ADR 0016 §8): neither end moves later than asked. const window = { from: Math.floor(asked.from / bucketMs) * bucketMs, to: Math.floor(asked.to / bucketMs) * bucketMs, @@ -60,18 +64,18 @@ export class UsageReader { const key = `${window.from}:${window.to}:${String(this.options.history.latestId())}`; if (this.#memo?.key === key) return { usage: this.#memo.usage }; - const { events, oldestTs, requestedBefore } = await this.options.history.read({ - carry: STEP_EVENTS, + const { answeredBefore, events, oldestTs, requestedBefore } = await this.options.history.read({ + carry: CARRIED_EVENTS, sinceTs: window.from, }); if (oldestTs !== undefined && oldestTs > asked.to) return { oldestTs }; const usage = computeUsage(events, window, { + answeredBefore, bucketMs, fleet: this.options.fleet !== undefined, labels: await this.#labelsFor(events), oldestTs, requestedBefore, - requesterPrefix: this.options.fleet?.requesterPrefix ?? "", workers: this.options.workers(), }); this.#memo = { key, usage }; diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 045d940e..601a95e4 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -1213,7 +1213,12 @@ describe("Dispatcher: events.replay", () => { /** The parts of the history `usage.get` reads, for a test that is not about it. */ const noUsageHistory = { latestId: () => undefined, - read: async () => ({ events: [], oldestTs: undefined, requestedBefore: new Set() }), + read: async () => ({ + events: [], + oldestTs: undefined, + answeredBefore: new Set(), + requestedBefore: new Set(), + }), }; describe("Dispatcher: usage.get", () => { @@ -1227,7 +1232,12 @@ describe("Dispatcher: usage.get", () => { latestId: () => state.newest, read: async () => { state.reads += 1; - return { events: [], oldestTs, requestedBefore: new Set() }; + return { + events: [], + oldestTs, + answeredBefore: new Set(), + requestedBefore: new Set(), + }; }, replay: async () => [], }; @@ -1247,7 +1257,7 @@ describe("Dispatcher: usage.get", () => { expect(state.reads).toBe(1); expect(second).toEqual(first); - // The answer's window is the window asked for rounded down to the bucket, never into the future. + // The answer's window is the window asked for rounded down to the bucket, so never later than asked. expect(second.window).toEqual({ from: WINDOW.from, to: WINDOW.to }); state.newest = "evt_2"; diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index ea4cdcd1..38ef4e45 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -695,7 +695,12 @@ describe("GatewayDispatcher", () => { const { dispatcher } = harness({ eventHistory: { latestId: () => undefined, - read: async () => ({ events: [], oldestTs: undefined, requestedBefore: new Set() }), + read: async () => ({ + events: [], + oldestTs: undefined, + answeredBefore: new Set(), + requestedBefore: new Set(), + }), replay: async (input) => { asked.push(input); return fromHistory; @@ -815,7 +820,12 @@ describe("GatewayDispatcher", () => { latestId: () => "evt_1", read: async () => { reads += 1; - return { events: [], oldestTs: 99 * 3_600_000, requestedBefore: new Set() }; + return { + events: [], + oldestTs: 99 * 3_600_000, + answeredBefore: new Set(), + requestedBefore: new Set(), + }; }, replay: async () => [], }, diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index 7fbde3f7..2d074f3e 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -232,6 +232,7 @@ describe("connectSimlock: handshake", () => { const figures = { boot: none, bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, failures: { byEvent: {} }, granted: 0, held: none, From e7dcd61a09045fd65d9a70e5651732010396a465 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 9 Oct 2026 22:49:54 +0200 Subject: [PATCH 19/23] test: the gateway usage handler reads a fleet grant from request.granted --- src/gateway/dispatcher.test.ts | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 38ef4e45..f74450ac 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -778,20 +778,6 @@ describe("GatewayDispatcher", () => { }, "gateway", ); - eventBus.emit( - "request.dispatched", - { - model: "m", - platform: "ios", - queuedMs: 0, - reason: "warm-hit", - requestId: "req_1", - requesterId: "agent-1", - stage: "warm-hit", - workerId: "wrk_1", - }, - "gateway", - ); eventBus.republish({ event: "lease.granted", id: "evt_worker_1", @@ -807,9 +793,15 @@ describe("GatewayDispatcher", () => { timestamp: 1_000, }); + eventBus.emit( + "request.granted", + { leaseId: "gwl_1", requestId: "req_1", worker: "wrk_1", workerLeaseId: "l1" }, + "gateway", + ); + const usage = await dispatcher.dispatch("usage.get", { from: 0, to: 600_000 }, session()); - expect(usage.totals).toMatchObject({ granted: 1, requests: 1 }); + expect(usage.totals).toMatchObject({ bySource: { warm: 1 }, granted: 1, requests: 1 }); expect(usage.workers).toMatchObject([{ granted: 1, id: "wrk_1", label: "mac-mini-1" }]); }); From 52533dd505aadadd8de16a926bf8997626c77a57 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 9 Oct 2026 22:51:06 +0200 Subject: [PATCH 20/23] refactor(usage): a decline is counted wherever a worker is named The own-event guard was unreachable: a gateway's own event names no worker. 0 failing -> 0 failing --- src/core/usage/read-events.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index 07281080..4bedb9a4 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -293,7 +293,6 @@ class Reader { /** A worker's `lease.declined`, or one a gateway's worker relayed: an event, never a request. */ #declined(seen: Seen): void { if (!seen.within || seen.worker === undefined) return; - if (this.options.fleet && seen.own) return; this.#declines.push({ platform: requestPlatform(seen.payload), worker: seen.worker }); this.#workers.add(seen.worker); } From b9f0f70b507f4832f2c5f50a27588fb93c4ed3ba Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 9 Oct 2026 22:51:30 +0200 Subject: [PATCH 21/23] refactor(usage): count a requester's request in flat terms Fallow flagged countRequest for complexity. 0 failing -> 0 failing --- src/core/usage/compute-usage.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts index 5377b56c..7dc8f711 100644 --- a/src/core/usage/compute-usage.ts +++ b/src/core/usage/compute-usage.ts @@ -304,14 +304,15 @@ type Requester = UsageOutput["requesters"][number]; /** `entry` with `fact` counted in it: a request made, a grant and the time it was held, a rejection. * A probe counts only as a grant and its held time. */ function countRequest(entry: Requester, fact: RequestFact): Requester { - const { outcome } = fact; const probe = fact.probe === true; + const kind = fact.outcome?.kind; return { ...entry, - granted: entry.granted + (outcome?.kind === "granted" ? 1 : 0), - heldTotalMs: entry.heldTotalMs + (outcome?.kind === "granted" ? (fact.heldMs ?? 0) : 0), - rejected: entry.rejected + (outcome?.kind === "rejected" && !probe ? 1 : 0), - requests: entry.requests + (fact.requestedAt === undefined || probe ? 0 : 1), + granted: entry.granted + Number(kind === "granted"), + // Only a granted request has a held time. + heldTotalMs: entry.heldTotalMs + (fact.heldMs ?? 0), + rejected: entry.rejected + Number(kind === "rejected" && !probe), + requests: entry.requests + Number(fact.requestedAt !== undefined && !probe), }; } From 066d3e986ef33e39ef60b618225fd8aa7aab7347 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Fri, 9 Oct 2026 22:59:56 +0200 Subject: [PATCH 22/23] test: kill the mutants the fleet join left alive Settle a carried fleet request at a restart whatever its time; drop the redundant outcome filter, the grant kind and the ring re-sort; test a restart at the window end, a rejected probe, an end with no relayed grant, worker order and label. 0 failing -> 0 failing --- src/bus/event-file.ts | 4 +++- src/core/usage/compute-usage.test.ts | 25 ++++++++++++++++++++++ src/core/usage/compute-usage.ts | 20 +++++++----------- src/core/usage/read-events.test.ts | 31 ++++++++++++++++++++++++---- src/core/usage/read-events.ts | 15 ++++++++------ 5 files changed, 72 insertions(+), 23 deletions(-) diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index acee6d41..d960ffde 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -265,7 +265,9 @@ export class EventHistory { } const newer = ring.filter((envelope) => envelope.timestamp > sinceTs); return { - events: [...carried.values(), ...newer].sort(byTimeThenSeq), + // The ring replays by time then seq, and every carried envelope is at or before `sinceTs`, so + // the carried ones (in that order) come before the newer ones. + events: [...carried.values(), ...newer], oldestTs, answeredBefore: before.answered, requestedBefore: before.requested, diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index e320a864..16320dc0 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -505,6 +505,7 @@ describe("computeUsage", () => { ["w2", undefined, 1], ]); expect(usage.workers[1]?.provisioning.p50).toBe(7_000); + expect(usage.workers[1]).not.toHaveProperty("label"); expect(usage.workers[0]?.provisioning.count).toBe(0); expect(usage.totals.granted).toBe(2); }); @@ -840,6 +841,30 @@ describe("computeUsage", () => { ]); }); + it("on a worker counts no rejection for a probe, which a worker declines and never rejects", () => { + const usage = computeUsage( + [probe(T0 + 1_000, "p1"), rejected(T0 + 1_500, "p1", "no-wait", "gw:g1:a")], + WINDOW, + WORKER, + ); + + expect(usage.totals).toMatchObject({ probes: 1, rejected: { byReason: {}, total: 0 } }); + expect(usage.requesters).toEqual([]); + }); + + it("lists workers by id whichever order the events name them in", () => { + const usage = computeUsage( + [ + capacity(T0 + 1_000, figures(0, 0, 2), undefined, { workerId: "w2" }), + capacity(T0 + 2_000, figures(0, 0, 2), undefined, { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.workers.map((worker) => worker.id)).toEqual(["w1", "w2"]); + }); + it("on a worker leaves a probe between its lease.requested and its grant out of series[].waiting", () => { const usage = computeUsage( [ diff --git a/src/core/usage/compute-usage.ts b/src/core/usage/compute-usage.ts index 7dc8f711..3dc7394f 100644 --- a/src/core/usage/compute-usage.ts +++ b/src/core/usage/compute-usage.ts @@ -129,7 +129,7 @@ function figuresFor( const requests = read.requests.filter((fact) => inScope(fact, scope)); const devices = read.devices.filter((fact) => inScope(fact, scope)); const asked = requests.filter((fact) => fact.requestedAt !== undefined && fact.probe !== true); - const probes = requests.filter((fact) => fact.requestedAt !== undefined && fact.probe === true); + const probes = requests.filter((fact) => fact.probe === true); const grants = requests.flatMap((fact) => fact.outcome?.kind === "granted" ? [{ ...fact, outcome: fact.outcome }] : [], ); @@ -174,17 +174,13 @@ function spanOf(start: number | undefined, end: number | undefined): number[] { type GrantSource = "booted" | "provisioned" | "unknown" | "warm"; function sourcesOf(grantedSources: readonly string[]): Record { - const sources = { booted: 0, provisioned: 0, unknown: 0, warm: 0 }; - for (const source of grantedSources) { - if ( - source === "warm" || - source === "booted" || - source === "provisioned" || - source === "unknown" - ) - sources[source] += 1; - } - return sources; + const counts = countBy(grantedSources); + return { + booted: counts.booted ?? 0, + provisioned: counts.provisioned ?? 0, + unknown: counts.unknown ?? 0, + warm: counts.warm ?? 0, + }; } /** The figures a device event brings: durations, incidents, and failures by name. */ diff --git a/src/core/usage/read-events.test.ts b/src/core/usage/read-events.test.ts index b68857f5..a6d81d8c 100644 --- a/src/core/usage/read-events.test.ts +++ b/src/core/usage/read-events.test.ts @@ -653,9 +653,13 @@ describe("requests carried into the window", () => { ).toEqual([]); }); - it("does not carry a gateway request that a restart before the window ended, and carries one a restart inside it ends", () => { - const ended = read([request(80, "old"), restarted(90)], FLEET); - expect(ended.carried).toEqual([]); + it("settles a carried gateway request at the gateway's next start, whether that is before the window or inside it, and leaves one a start preceded open", () => { + const before = read([request(80, "old"), restarted(90)], FLEET); + expect(before.carried).toEqual([ + expect.objectContaining({ + outcome: { at: 90, kind: "rejected", reason: "daemon-restarted" }, + }), + ]); const later = read([request(80, "old"), restarted(150)], FLEET); expect(later.carried).toEqual([ expect.objectContaining({ @@ -664,7 +668,7 @@ describe("requests carried into the window", () => { }), ]); const after = read([restarted(70), request(80, "old")], FLEET); - expect(after.carried).toHaveLength(1); + expect(after.carried[0]).not.toHaveProperty("outcome"); }); }); @@ -871,6 +875,15 @@ describe("a gateway", () => { }); }); + it("is a daemon.started at exactly the window end", () => { + const result = read([request(110, "r"), restarted(200)], FLEET); + expect(result.requests[0]?.outcome).toEqual({ + at: 200, + kind: "rejected", + reason: "daemon-restarted", + }); + }); + it("stays open past a daemon.started before the request, a relayed one, one after the window, and one on a worker", () => { const open = (events: EventEnvelope[], options: ReadOptions = FLEET) => read(events, options).requests[0]?.outcome; @@ -964,6 +977,16 @@ describe("a gateway", () => { FLEET, ); expect(otherWorker.requests[0]?.outcome).toMatchObject({ source: "unknown" }); + // An end of the lease with no relayed grant to start from gives no held time either. + const endOnly = read( + [ + request(110, "r"), + handed(115, "r"), + at(130, "lease.released", { leaseId: "L", workerId: "w1" }), + ], + FLEET, + ); + expect(endOnly.requests[0]).not.toHaveProperty("heldMs"); }); it("default a relayed grant's missing source to empty, and ignore a relayed grant with no lease id or worker", () => { diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index 4bedb9a4..ff4616a1 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -176,7 +176,10 @@ class Reader { readonly #carriedRequests = new Map(); readonly #ownRejections = new Map(); /** A worker's own grants by request id: the lease each served. */ - readonly #grants = new Map(); + readonly #grants = new Map< + string, + { readonly at: number; readonly leaseId: string; readonly source: string } + >(); /** A gateway's `request.granted` by request id (ADR 0021 §4). */ readonly #handed = new Map< string, @@ -328,7 +331,7 @@ class Reader { } const requestId = text(seen.payload, "requestId"); if (requestId === undefined) return; - this.#grants.set(requestId, { at: seen.event.timestamp, kind: "granted", leaseId, source }); + this.#grants.set(requestId, { at: seen.event.timestamp, leaseId, source }); } #ended(seen: Seen): void { @@ -395,15 +398,15 @@ class Reader { } /** - * The requests made before the window that nothing had answered when it began, and that no - * gateway restart had ended. A worker's probe never waits there (ADR 0021 §5). + * The requests made before the window that the history does not report answered by its start. + * One a gateway restart ended may carry an outcome from before the window, which settles it for + * every time in it. A worker's probe never waits there (ADR 0021 §5). */ #carriedOpen(): RequestFact[] { const facts: RequestFact[] = []; for (const request of this.#carriedRequests.values()) { if (request.probe || this.options.answeredBefore.has(request.requestId)) continue; - const fact = this.#factFor(request.requestId, request); - if (fact.outcome === undefined || fact.outcome.at > this.window.from) facts.push(fact); + facts.push(this.#factFor(request.requestId, request)); } return facts; } From 866849d954915775a2be4a76969da5ad8d674129 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sat, 10 Oct 2026 07:52:51 +0200 Subject: [PATCH 23/23] fix(usage): join a reused lease ID to its own lease, run the e2e with --since 1h, fix stale docs 13 review findings: 13 open -> 0 open --- docs/CLI.md | 10 ++-- docs/CLIENT.md | 2 +- docs/internal/EVENTS.md | 2 +- e2e/usage-stats.test.ts | 51 +++++++----------- src/bus/event-file.test.ts | 24 +++++++++ src/bus/event-file.ts | 7 +-- src/core/usage/compute-usage.test.ts | 55 ++++++++++++++++++-- src/core/usage/read-events.ts | 77 ++++++++++++++++++++++------ src/core/usage/usage-reader.test.ts | 8 +++ 9 files changed, 173 insertions(+), 63 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index 8e0cbb44..0edee57d 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -2000,8 +2000,9 @@ What the figures count: `provisioned` was created for the request. - **Rejected** are the requests that ended without a device, by reason (`timeout`, `no-wait`, `cancelled`, ...), plus the requests refused before - they were stored (a requester that already holds a lease, or one that asked - for a lease ID that is taken). Those are in the window their rejection falls + they were stored (a requester that already holds a lease, one that asked + for a lease ID that is taken, or one that came while admission was closed + for `nuke` or maintenance). Those are in the window their rejection falls in and are not requests, so granted plus rejected can be more than requests. - **Wait** is the time from the request to its grant or rejection. **Held** is the time from the grant to the release or expiry of the lease. **Turnaround** @@ -2039,9 +2040,8 @@ with the reason `daemon-restarted`. How the device came to be ready and how long the lease was held come from the worker's record of the lease; a grant whose worker's record the gateway never received counts as `unknown` and has no held time. A worker's row has the requests granted on it or that failed on it -(`worker-failed`), its declined count and its device figures; a request that -ended any other way, or is still waiting, is in the totals and the platform rows -only. The queue figure is the fleet queue's. +(`worker-failed`) and its device figures; a request that ended any other way, +or is still waiting, is in the totals and the platform rows only. The queue figure is the fleet queue's. A worker's own `simlock stats` covers that worker only: its row is itself. A request that a gateway sent it is not one of the worker's requests: it is counted diff --git a/docs/CLIENT.md b/docs/CLIENT.md index ee9e33b8..aff33093 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -589,7 +589,7 @@ usage.totals.wait.p95; // milliseconds, or null when nothing waited usage.totals.failures.byEvent; // a count per failure event, e.g. "device.purge-failed" usage.workers[0]?.label; // one entry for each worker; a worker lists itself usage.series; // one point per bucket, for a chart; `waiting` counts every - // request open at the bucket's end + // request open at the bucket's end, leaving out a gateway's probes ``` The daemon computes the figures from its event history, so they cover only what diff --git a/docs/internal/EVENTS.md b/docs/internal/EVENTS.md index 0f932594..9788ccbb 100644 --- a/docs/internal/EVENTS.md +++ b/docs/internal/EVENTS.md @@ -52,7 +52,7 @@ in short: `subject.past-tense-fact`, emitted post-commit, facts not commands. | `lease.renewed` | lease id, new deadline | a `lease.renew` succeeded — whether it came from `simlock lease renew`, `POST /v1/leases/{id}/renew`, or the renew timer a running `simlock lease` / MCP session keeps over its own lease. There is one renew path and this is it | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.released` | lease id, device id, reason (explicit/killed/device-lost), owner id | an explicit `lease.release` (which is what a `simlock lease` holder does on its way out), (killed) an operator `release --all` or `nuke`, or (device-lost) a leased device could not be recovered after it stopped running outside simlock, or a daemon start found its device not running, and the device of that lease was wiped and returned to the pool, left waiting in `reclaiming` on a platform the daemon could not list, or marked missing. Closing a connection is not a release and never emits this | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.expired` | lease id, device id, owner id | the lease's deadline passed with no `lease.renew` behind it — the grant-time TTL, or the TTL of the last renew, simply ran out. This is the one way a lease ends without somebody asking, and the only bound on a holder that was killed outright | LeaseLifecycle | implemented (payload per ADR 0004 pending) | -| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted/worker-failed), `code` and `worker` on `worker-failed` (ADR 0021 §4, additive; `worker`, never `workerId`, which marks a relayed event, ADR 0014 §6) | a request ended without a grant. A worker emits it only for its local requests; a probe ends in `lease.declined` there (ADR 0021 §2). A gateway emits it for every ending that is not a grant (ADR 0021 §4): its own reasons, `unresolvable-spec` for the last cannot-serve refusal whether or not the request had queued, and `worker-failed` for any other failure on the worker it went to -- a terminal refusal, a failure after progress, `WORKER_UNREACHABLE`, `INTERNAL`, a dispatch timeout, a worker's own `REQUESTER_ALREADY_LEASED` or `LEASE_ID_TAKEN` -- with `code` the error code the caller got and `worker` the worker. `dispose()` emits nothing, so a request still open when a gateway stops has no ending event (ADR 0021 §5 plans for usage to close those at the next `daemon.started`; not built yet). A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never stored and has no `lease.requested` (except a gateway's `lease-id-taken` after a grant, below); it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec`, whether or not it had entered the gateway queue (the worker only declined it, ADR 0021 §4); `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | +| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted/worker-failed), `code` and `worker` on `worker-failed` (ADR 0021 §4, additive; `worker`, never `workerId`, which marks a relayed event, ADR 0014 §6) | a request ended without a grant. A worker emits it only for its local requests; a probe ends in `lease.declined` there (ADR 0021 §2). A gateway emits it for every ending that is not a grant (ADR 0021 §4): its own reasons, `unresolvable-spec` for the last cannot-serve refusal whether or not the request had queued, and `worker-failed` for any other failure on the worker it went to -- a terminal refusal, a failure after progress, `WORKER_UNREACHABLE`, `INTERNAL`, a dispatch timeout, a worker's own `REQUESTER_ALREADY_LEASED` or `LEASE_ID_TAKEN` -- with `code` the error code the caller got and `worker` the worker. `dispose()` emits nothing, so a request still open when a gateway stops has no ending event (ADR 0021 §5: usage closes those at the next `daemon.started`, as rejected with `daemon-restarted`). A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never stored and has no `lease.requested` (except a gateway's `lease-id-taken` after a grant, below); it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec`, whether or not it had entered the gateway queue (the worker only declined it, ADR 0021 §4); `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | | `lease.declined` | `{ requestId, fleetRequestId, requester, requestSpec, reason }` -- `reason` takes `lease.rejected`'s values; `requestSpec` is there for the same reason as on `lease.rejected`: a decline at admission has no `lease.requested` (ADR 0021 §2) | a worker refused or failed a probe, a `lease.request` carrying `fleetRequestId` (ADR 0021 §1), whatever the reason (`no-wait`, `unresolvable-spec`, `already-leased`, `lease-id-taken`, `boot-timeout`, `killed`, `daemon-restarted` at the next start) and however far the work got. The gateway owns the outcome of a probe, so the worker never rejects one and never queues one: where it would queue (a second failed provision) it declines with `no-wait` and answers `NO_CAPACITY`. The event depends only on the request being a probe, never on whether the gateway will retry. A local request is rejected exactly as before | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) | implemented | On a **gateway**, `lease.requested`, `lease.queued` and `lease.rejected` are its own fleet queue's facts (ADR 0005 §11/§14), diff --git a/e2e/usage-stats.test.ts b/e2e/usage-stats.test.ts index 4084953f..a593afc2 100644 --- a/e2e/usage-stats.test.ts +++ b/e2e/usage-stats.test.ts @@ -41,33 +41,19 @@ interface Usage { readonly requesters: readonly { readonly id: string; readonly requests: number }[]; } -const HOUR_MS = 3_600_000; -const MINUTE_MS = 60_000; - -/** - * The last hour up to `to`, asked with `--from` and `--to`. The window is rounded down to the - * series bucket (a minute here), so what happened in the minute now running is counted once the - * minute is over; a test that names the end itself does not wait the minute out. - */ -async function stats( - env: { cli: (args: string[]) => Promise<{ code: number | null; json?: unknown }> }, - to: number, -): Promise { - const result = await env.cli([ - "stats", - "--from", - new Date(to - HOUR_MS).toISOString(), - "--to", - new Date(to).toISOString(), - "--json", - ]); +/** The last hour, asked with `--since 1h`, as JSON. */ +async function stats(env: { + cli: (args: string[]) => Promise<{ code: number | null; json?: unknown }>; +}): Promise { + const result = await env.cli(["stats", "--since", "1h", "--json"]); expect(result.code).toBe(0); return result.json as Usage; } /** - * The figures for a window that closes over the newest event: its end is the first minute - * boundary after that event, or `at` when given (so two calls compare the same window). + * The figures for the last hour once the minute that holds the newest event has closed: the + * window is rounded down to the series bucket (a minute here), so what happened in the minute now + * running is counted when that minute is over. Waits for it, up to a minute. */ async function settledStats( env: { @@ -75,17 +61,15 @@ async function settledStats( events: () => Promise; }, until: (usage: Usage) => boolean = () => true, - at?: number, ): Promise { let usage: Usage | undefined; await waitFor( async () => { const newest = Math.max(...(await env.events()).map((event) => event.timestamp)); - const to = at ?? (Math.floor(newest / MINUTE_MS) + 1) * MINUTE_MS; - usage = await stats(env, to); - return (at !== undefined || usage.window.to >= newest) && until(usage); + usage = await stats(env); + return usage.window.to >= newest && until(usage); }, - { interval: 500, label: "the window closed over the newest event", timeout: 30_000 }, + { interval: 1_000, label: "the window closed over the newest event", timeout: 90_000 }, ); return usage as Usage; } @@ -105,7 +89,7 @@ function payload(event: RecordedEvent): Record { } describe("simlock stats", () => { - it("after a scripted run of leases against the fake driver, simlock stats --json for the last hour, named with --from and --to, reports the counts a test derives independently from the same hour of simlock events", async () => { + it("after a scripted run of leases against the fake driver, simlock stats --since 1h --json reports the counts a test derives independently from the same hour of simlock events --since 1h", async () => { const env = await withDaemon({ configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }, }); @@ -192,7 +176,7 @@ describe("simlock stats", () => { expect(table.code).toBe(0); expect(table.stdout).toContain("Totals"); expect(table.stdout).toContain("Requests:"); - }); + }, 240_000); it("after a --no-wait refusal, requests and rejected.byReason.no-wait each rise by one and granted is unchanged", async () => { const env = await withDaemon({ @@ -217,9 +201,9 @@ describe("simlock stats", () => { (before.totals.rejected.byReason["no-wait"] ?? 0) + 1, ); expect(after.totals.granted).toBe(before.totals.granted); - }); + }, 240_000); - it("simlock stats for the last hour, named with --from and --to, returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { + it("simlock stats --since 1h returns the same figures before and after simlock daemon stop and simlock daemon start", async () => { const env = await withDaemon(); await env.driverScript.set({ ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, @@ -232,7 +216,8 @@ describe("simlock stats", () => { expect(before.totals.granted).toBe(1); await env.restartDaemon(); - const after = await settledStats(env, () => true, before.window.to); + const after = await stats(env); + expect(after.window.to).toBeGreaterThanOrEqual(before.window.to); expect(after.totals).toMatchObject({ boot: before.totals.boot, @@ -247,7 +232,7 @@ describe("simlock stats", () => { wait: before.totals.wait, }); expect(after.requesters).toEqual(before.requesters); - }); + }, 240_000); it("on a gateway with two workers, simlock stats has fleet totals and one row per worker, and each worker's own simlock stats covers itself only", async () => { const port = await freeLoopbackPort(); diff --git a/src/bus/event-file.test.ts b/src/bus/event-file.test.ts index 99b3830d..ea6ff105 100644 --- a/src/bus/event-file.test.ts +++ b/src/bus/event-file.test.ts @@ -240,6 +240,30 @@ describe("EventHistory", () => { ]); }); + it("reads the ring with the carried envelopes in time order, though the earlier of two kinds was first set", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events = history({ bus, path: "/data/events.jsonl", filesystem: new MemoryFilesystem() }); + bus.emit("queue.changed", { depth: 1 }, "wait-queue"); + clock.advance(1_000); + const figures = { maxRunning: 2, reserved: 0, running: 1, warm: 0 }; + const capacity = bus.emit( + "capacity.changed", + { android: figures, global: figures, ios: figures }, + "capacity", + ); + clock.advance(1_000); + const queue = bus.emit("queue.changed", { depth: 3 }, "wait-queue"); + + const read = await events.read({ + carry: ["queue.changed", "capacity.changed"], + sinceTs: 5_000, + }); + + expect(read.events.map((envelope) => envelope.timestamp)).toEqual([2_000, 3_000]); + expect(read.events).toEqual([capacity, queue]); + }); + it("reads the ring with an event at exactly sinceTs as the step in force and not as a newer event", async () => { const clock = new FakeClock(1_000); const bus = new EventBus(clock); diff --git a/src/bus/event-file.ts b/src/bus/event-file.ts index d960ffde..70f62cbe 100644 --- a/src/bus/event-file.ts +++ b/src/bus/event-file.ts @@ -265,9 +265,10 @@ export class EventHistory { } const newer = ring.filter((envelope) => envelope.timestamp > sinceTs); return { - // The ring replays by time then seq, and every carried envelope is at or before `sinceTs`, so - // the carried ones (in that order) come before the newer ones. - events: [...carried.values(), ...newer], + // Every carried envelope is at or before `sinceTs`, so the carried ones, put in time order + // (a key keeps its first slot in the map), come before the newer ones, which the ring replays + // in order. + events: [...[...carried.values()].sort(byTimeThenSeq), ...newer], oldestTs, answeredBefore: before.answered, requestedBefore: before.requested, diff --git a/src/core/usage/compute-usage.test.ts b/src/core/usage/compute-usage.test.ts index 16320dc0..d0c53b63 100644 --- a/src/core/usage/compute-usage.test.ts +++ b/src/core/usage/compute-usage.test.ts @@ -514,15 +514,17 @@ describe("computeUsage", () => { const usage = computeUsage( [ requested(T0 + 1_000, "gw-r1"), - // The worker's own copy of the same request, relayed. + // The worker's own copy of the same request, relayed from a worker the grant is not on. at(T0 + 1_300, "lease.requested", { fleetRequestId: "gw-r1", requestId: "w-r1", requestSpec: { platform: "ios" }, requester: "gw:g1:agent-a", waitPolicy: "noWait", - workerId: "w1", + workerId: "w9", }), + // A grant on that other worker under the same lease id: not the one the request got. + granted(T0 + 3_900, "w9-r", "l1", "warm", "gw:g1:agent-b", { workerId: "w9" }), granted(T0 + 4_000, "w-r1", "l1", "booted", "gw:g1:agent-a", { workerId: "w1" }), handed(T0 + 4_200, "gw-r1", "w1", "l1"), released(T0 + 9_000, "l1", { workerId: "w1" }), @@ -533,7 +535,7 @@ describe("computeUsage", () => { expect(usage.totals.requests).toBe(1); expect(usage.totals.granted).toBe(1); - expect(usage.totals.bySource.booted).toBe(1); + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, unknown: 0, warm: 0 }); expect(usage.workers).toHaveLength(1); expect(usage.workers[0]).toMatchObject({ granted: 1, id: "w1", requests: 1 }); }); @@ -705,11 +707,15 @@ describe("computeUsage", () => { const usage = computeUsage( [ requested(T0 + 1_000, "g1", "a"), - // The same lease id on three workers: only (w2, l1) is the one g1 names. + // The same lease id on w1 and w2, and other leases on w2: only (w2, l1) is the one g1 names. + granted(T0 + 1_050, "w2-r0", "l0", "warm", "gw:g1:z", { workerId: "w2" }), granted(T0 + 1_100, "w1-r", "l1", "warm", "gw:g1:a", { workerId: "w1" }), granted(T0 + 1_200, "w2-r", "l1", "provisioned", "gw:g1:a", { workerId: "w2" }), + granted(T0 + 1_300, "w2-r2", "l2", "booted", "gw:g1:y", { workerId: "w2" }), handed(T0 + 1_500, "g1", "w2", "l1"), + released(T0 + 4_000, "l0", { workerId: "w2" }), released(T0 + 6_200, "l1", { workerId: "w2" }), + released(T0 + 7_300, "l2", { workerId: "w2" }), released(T0 + 9_000, "l1", { workerId: "w1" }), requested(T0 + 2_000, "g2", "b"), handed(T0 + 2_400, "g2", "w3", "l9"), @@ -729,6 +735,47 @@ describe("computeUsage", () => { ]); }); + it("gives a lease id chosen again after its release the end of its own lease on a worker, not the later one's", () => { + const usage = computeUsage( + [ + requested(T0 + 100_000, "r1"), + granted(T0 + 120_000, "r1", "ci-1"), + released(T0 + 130_000, "ci-1"), + requested(T0 + 140_000, "r2"), + granted(T0 + 150_000, "r2", "ci-1"), + released(T0 + 190_000, "ci-1"), + ], + WINDOW, + WORKER, + ); + + expect(usage.totals.held).toEqual({ count: 2, max: 40_000, p50: 10_000, p95: 40_000 }); + }); + + it("with fleet: true gives two fleet requests granted on one worker under one lease id each their own source and held time", () => { + const usage = computeUsage( + [ + requested(T0 + 100_000, "g1", "a"), + granted(T0 + 120_000, "w-r1", "ci-1", "warm", "gw:g1:a", { workerId: "w1" }), + handed(T0 + 120_100, "g1", "w1", "ci-1"), + released(T0 + 130_000, "ci-1", { workerId: "w1" }), + requested(T0 + 140_000, "g2", "b"), + granted(T0 + 150_000, "w-r2", "ci-1", "booted", "gw:g1:b", { workerId: "w1" }), + handed(T0 + 150_100, "g2", "w1", "ci-1"), + released(T0 + 190_000, "ci-1", { workerId: "w1" }), + ], + WINDOW, + FLEET, + ); + + expect(usage.totals.bySource).toEqual({ booted: 1, provisioned: 0, unknown: 0, warm: 1 }); + expect(usage.totals.held).toEqual({ count: 2, max: 40_000, p50: 10_000, p95: 40_000 }); + expect(usage.requesters.map((requester) => [requester.id, requester.heldTotalMs])).toEqual([ + ["a", 10_000], + ["b", 40_000], + ]); + }); + it("with fleet: true counts each relayed lease.declined under its worker's and platform's declined, two declines of one request as two", () => { const usage = computeUsage( [ diff --git a/src/core/usage/read-events.ts b/src/core/usage/read-events.ts index ff4616a1..5fb8d3bb 100644 --- a/src/core/usage/read-events.ts +++ b/src/core/usage/read-events.ts @@ -67,7 +67,11 @@ export type CapacityStep = Step< export interface ReadEvents { readonly requests: readonly RequestFact[]; - /** Requests made before the window and still open when it began; only the series counts them. */ + /** + * Requests with no outcome by the window's start, which only the series counts: ones made + * before the window and still open when it began, and a gateway's requests that a restart + * before the window's end ended (outcome `daemon-restarted`). + */ readonly carried: readonly RequestFact[]; readonly devices: readonly DeviceFact[]; readonly declines: readonly DeclineFact[]; @@ -127,6 +131,12 @@ const platformOf = (value: unknown): Platform | undefined => const objectOf = (value: unknown): Payload | undefined => typeof value === "object" && value !== null ? (value as Payload) : undefined; +interface RelayedGrant { + readonly at: number; + readonly index: number; + readonly source: string; +} + interface Request { readonly index: number; readonly at: number; @@ -178,16 +188,25 @@ class Reader { /** A worker's own grants by request id: the lease each served. */ readonly #grants = new Map< string, - { readonly at: number; readonly leaseId: string; readonly source: string } + { + readonly at: number; + readonly index: number; + readonly leaseId: string; + readonly source: string; + } >(); /** A gateway's `request.granted` by request id (ADR 0021 §4). */ readonly #handed = new Map< string, { readonly at: number; readonly worker: string; readonly workerLeaseId: string } >(); - /** The grants workers relayed to a gateway, by worker and the worker's lease id. */ - readonly #relayedGrants = new Map(); - readonly #ends = new Map(); + /** + * The grants workers relayed to a gateway, by worker and the worker's lease id, in event order. + * A lease id is unique only while its lease is active (ADR 0020), so one key can hold several. + */ + readonly #relayedGrants = new Map(); + /** The ends of leases, by worker and lease id, in event order: a key can hold several. */ + readonly #ends = new Map(); /** Where the gateway's own `daemon.started` events fall, in event order (ADR 0021 §5). */ readonly #restarts: { readonly index: number; readonly at: number }[] = []; readonly #handlers: Record void> = { @@ -322,22 +341,25 @@ class Reader { const source = text(seen.payload, "source") ?? ""; if (this.options.fleet) { if (seen.worker !== undefined) { - this.#relayedGrants.set(this.#key(seen.worker, leaseId), { - at: seen.event.timestamp, - source, - }); + const key = this.#key(seen.worker, leaseId); + const earlier = this.#relayedGrants.get(key) ?? []; + earlier.push({ at: seen.event.timestamp, index: seen.index, source }); + this.#relayedGrants.set(key, earlier); } return; } const requestId = text(seen.payload, "requestId"); if (requestId === undefined) return; - this.#grants.set(requestId, { at: seen.event.timestamp, leaseId, source }); + this.#grants.set(requestId, { at: seen.event.timestamp, index: seen.index, leaseId, source }); } #ended(seen: Seen): void { const leaseId = text(seen.payload, "leaseId"); if (!this.#counts(seen) || leaseId === undefined) return; - this.#ends.set(this.#key(seen.worker, leaseId), seen.event.timestamp); + const key = this.#key(seen.worker, leaseId); + const earlier = this.#ends.get(key) ?? []; + earlier.push({ at: seen.event.timestamp, index: seen.index }); + this.#ends.set(key, earlier); } #provisioned(seen: Seen): void { @@ -461,7 +483,7 @@ class Reader { handed: { readonly at: number; readonly worker: string; readonly workerLeaseId: string }, ): RequestFact { this.#workers.add(handed.worker); - const relayed = this.#relayedGrants.get(this.#key(handed.worker, handed.workerLeaseId)); + const relayed = this.#relayedGrantFor(handed); const outcome: Granted = { at: handed.at, kind: "granted", @@ -473,10 +495,33 @@ class Reader { : this.#withHeld(fact, this.#heldFor(handed.worker, handed.workerLeaseId, relayed)); } - /** How long a lease was held, from its grant to the end the window saw. */ - #heldFor(worker: string, leaseId: string, grant: { readonly at: number }): number | undefined { - const endedAt = this.#ends.get(this.#key(worker, leaseId)); - return endedAt === undefined ? undefined : endedAt - grant.at; + /** Of the grants a worker relayed under one lease id, the one nearest in time to the handover. */ + #relayedGrantFor(handed: { + readonly at: number; + readonly worker: string; + readonly workerLeaseId: string; + }): RelayedGrant | undefined { + const candidates = this.#relayedGrants.get(this.#key(handed.worker, handed.workerLeaseId)); + let nearest: RelayedGrant | undefined; + for (const candidate of candidates ?? []) { + if ( + nearest === undefined || + Math.abs(candidate.at - handed.at) < Math.abs(nearest.at - handed.at) + ) { + nearest = candidate; + } + } + return nearest; + } + + /** How long a lease was held, from its grant to the first end after it the window saw. */ + #heldFor( + worker: string, + leaseId: string, + grant: { readonly at: number; readonly index: number }, + ): number | undefined { + const end = this.#ends.get(this.#key(worker, leaseId))?.find((e) => e.index > grant.index); + return end === undefined ? undefined : end.at - grant.at; } #withHeld(fact: RequestFact, heldMs: number | undefined): RequestFact { diff --git a/src/core/usage/usage-reader.test.ts b/src/core/usage/usage-reader.test.ts index 407cba21..59903b83 100644 --- a/src/core/usage/usage-reader.test.ts +++ b/src/core/usage/usage-reader.test.ts @@ -160,6 +160,14 @@ describe("UsageReader", () => { expect(result.totals.requests).toBe(1); }); + it("says the figures cover the window when the history reaches before it, though no event falls in its first minutes", async () => { + const { usage } = reader(history([requested(T0 + 30 * MINUTE, "a")], T0 - HOUR).source); + + const result = answer(await usage.get({ from: T0, to: T0 + HOUR })); + + expect(result).toMatchObject({ coversFrom: T0, partial: false }); + }); + it("keeps its answer while the window and the newest event are the same, and reads again when either changes", async () => { const { source, state } = history(); const { usage } = reader(source);