From 572cdcba6ccc47a41f05aae5b07a6ebb84676429 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:00:29 +0200 Subject: [PATCH 1/9] test: let requesters choose the lease ID (#410) --- e2e/contention.test.ts | 61 ++++++++ e2e/gateway-fleet.test.ts | 43 ++++++ e2e/http-api.test.ts | 131 +++++++++++++++++ src/cli/index.test.ts | 36 ++++- src/core/registry.test.ts | 87 ++++++++++++ src/gateway/fleet-coordinator.test.ts | 194 +++++++++++++++++++++++++- src/gateway/lease-index.test.ts | 84 +++++++++++ src/gateway/test-support.ts | 4 + src/leasing/create-leasing.test.ts | 194 ++++++++++++++++++++++++++ src/mcp/server.test.ts | 25 ++++ 10 files changed, 856 insertions(+), 3 deletions(-) diff --git a/e2e/contention.test.ts b/e2e/contention.test.ts index a96c8ff0..b9849b04 100644 --- a/e2e/contention.test.ts +++ b/e2e/contention.test.ts @@ -69,6 +69,67 @@ describe("contention & queueing", () => { waiterE.kill("SIGTERM"); await waiterE.waitForExit(15_000); }); + + it("the CLI --lease-id flag names the lease, and exits 13 with LEASE_ID_TAKEN for an ID an active lease or a waiting request holds", async () => { + const env = await withDaemon({ + configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } }, + }); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + + const held = await env.cli([ + ...LEASE_ARGS, + "--agent-id", + "agent-a", + "--lease-id", + "ad-7f3a", + "--detach", + ]); + expect(held.code).toBe(0); + expect((held.json as { lease: { id: string } }).lease.id).toBe("ad-7f3a"); + + // The ID of an active lease is taken. + const clash = await env.cli([ + ...LEASE_ARGS, + "--agent-id", + "agent-b", + "--lease-id", + "ad-7f3a", + "--detach", + ]); + expect(clash.code).toBe(13); + expect(clash.error).toMatchObject({ code: "LEASE_ID_TAKEN" }); + expect(clash.error?.message).toContain("ad-7f3a"); + + // A request waiting for the one device holds its ID too. + const waiter = env.cliBackground([ + ...LEASE_ARGS, + "--agent-id", + "agent-c", + "--lease-id", + "waits-1", + ]); + await waitFor(() => waiter.progressEvents().some((event) => isQueuedAt(event, 1)), { + label: "agent-c queued at position 1", + }); + const waitingClash = await env.cli([ + ...LEASE_ARGS, + "--agent-id", + "agent-d", + "--lease-id", + "waits-1", + "--detach", + ]); + expect(waitingClash.code).toBe(13); + expect(waitingClash.error).toMatchObject({ code: "LEASE_ID_TAKEN" }); + + expect((await env.cli(["release", "ad-7f3a"])).code).toBe(0); + const granted = JSON.parse(await waiter.firstStdoutLine(15_000)) as { lease: { id: string } }; + expect(granted.lease.id).toBe("waits-1"); + waiter.kill("SIGTERM"); + await waiter.waitForExit(15_000); + }); }); function isQueuedAt(event: unknown, position: number): boolean { diff --git a/e2e/gateway-fleet.test.ts b/e2e/gateway-fleet.test.ts index a74440fa..060c6630 100644 --- a/e2e/gateway-fleet.test.ts +++ b/e2e/gateway-fleet.test.ts @@ -734,4 +734,47 @@ describe("gateway fleet", () => { expect(await listWorkers(gateway)).toEqual([]); }); + + it("through a gateway, a caller-chosen lease ID comes back with no worker prefix, renews and releases by that ID, and a second request for it is LEASE_ID_TAKEN", async () => { + const port = await freeLoopbackPort(); + const gateway = await withDaemon({ + configOverrides: { http: { host: "127.0.0.1", port }, mode: "gateway" }, + driver: "none", + }); + const { secret } = (await gateway.cli(["token", "create", "--role", "worker"])).json as { + secret: string; + }; + await withDaemon({ + configOverrides: { + gateway: { label: "worker", token: secret, url: `ws://127.0.0.1:${port}` }, + }, + driverScript: { ios: { availableOsVersions: ["26.0"], knownModels: ["iPhone 16 Pro"] } }, + }); + await waitForWorkers( + gateway, + (views) => + views.length === 1 && views[0]?.connection === "connected" && views[0].catalog.length > 0, + "the worker connected with its catalog", + ); + const lease = (agentId: string, extra: readonly string[]) => + gateway.cli(["lease", "--platform", "ios", "--agent-id", agentId, "--detach", ...extra], { + timeout: 30_000, + }); + + const chosen = await lease("agent-a", ["--lease-id", "ad-7f3a"]); + expect(chosen.code, chosen.stderr).toBe(0); + expect((chosen.json as { lease: { id: string } }).lease.id).toBe("ad-7f3a"); + + const clash = await lease("agent-b", ["--lease-id", "ad-7f3a"]); + expect(clash.code).toBe(13); + expect(clash.error).toMatchObject({ code: "LEASE_ID_TAKEN" }); + + const renewed = await gateway.cli(["lease", "renew", "ad-7f3a"], { timeout: 30_000 }); + expect(renewed.code, renewed.stderr).toBe(0); + expect((await gateway.cli(["release", "ad-7f3a"], { timeout: 30_000 })).code).toBe(0); + + const generated = await lease("agent-a", []); + expect(generated.code, generated.stderr).toBe(0); + expect((generated.json as { lease: { id: string } }).lease.id).toMatch(/^[^.]+\.lse_/); + }); }); diff --git a/e2e/http-api.test.ts b/e2e/http-api.test.ts index 64402925..dd56b885 100644 --- a/e2e/http-api.test.ts +++ b/e2e/http-api.test.ts @@ -413,4 +413,135 @@ describe("HTTP API", () => { expect(polled.state).toBe("granted"); expect(polled.lease).toMatchObject({ device: "iPhone 16", os: "18.4", platform: "ios" }); }); + + it("POST /v1/lease-requests with leaseId grants a lease with exactly that ID, and renew and release name it", async () => { + const port = await reservePort(); + const env = await withDaemon({ + configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, + driverScript: { ios: { availableOsVersions: ["18.4"], knownModels: ["iPhone 16"] } }, + }); + const baseUrl = `http://127.0.0.1:${port}`; + const { secret } = (await env.cli(["token", "create", "--role", "agent"])).json as { + secret: string; + }; + const headers = { authorization: `Bearer ${secret}`, "content-type": "application/json" }; + await waitFor( + async () => { + try { + return (await fetch(`${baseUrl}/v1/healthz`)).ok; + } catch { + return false; + } + }, + { label: "HTTP gateway accepting connections" }, + ); + + const created = await fetch(`${baseUrl}/v1/lease-requests`, { + body: JSON.stringify({ device: "iPhone 16", leaseId: "ad-7f3a_01", platform: "ios" }), + headers, + method: "POST", + }); + expect(created.status).toBe(201); + let polled = ((await created.json()) as { request: RequestResource }).request; + while (polled.state !== "granted" && polled.state !== "failed") { + const response = await fetch(`${baseUrl}/v1/lease-requests/${polled.id}?wait=10`, { + headers, + }); + polled = ((await response.json()) as { request: RequestResource }).request; + } + expect(polled.state).toBe("granted"); + expect(polled.lease?.id).toBe("ad-7f3a_01"); + + const renewed = await fetch(`${baseUrl}/v1/leases/ad-7f3a_01/renew`, { + headers, + method: "POST", + }); + expect(renewed.status).toBe(200); + expect(((await renewed.json()) as { leaseId: string }).leaseId).toBe("ad-7f3a_01"); + + const released = await fetch(`${baseUrl}/v1/leases/ad-7f3a_01`, { headers, method: "DELETE" }); + expect(released.status).toBe(202); + }); + + it("POST /v1/lease-requests answers 400 BAD_REQUEST for each leaseId the pattern rejects, and 201 for a 64-character one", async () => { + const port = await reservePort(); + const env = await withDaemon({ + configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, + driverScript: { ios: { availableOsVersions: ["18.4"], knownModels: ["iPhone 16"] } }, + }); + const baseUrl = `http://127.0.0.1:${port}`; + const { secret } = (await env.cli(["token", "create", "--role", "agent"])).json as { + secret: string; + }; + const headers = { authorization: `Bearer ${secret}`, "content-type": "application/json" }; + await waitFor( + async () => { + try { + return (await fetch(`${baseUrl}/v1/healthz`)).ok; + } catch { + return false; + } + }, + { label: "HTTP gateway accepting connections" }, + ); + + for (const leaseId of ["", "a".repeat(65), "w1.myid", "my id", "-s", "--help", "_x", "ząb"]) { + const response = await fetch(`${baseUrl}/v1/lease-requests`, { + body: JSON.stringify({ device: "iPhone 16", leaseId, platform: "ios" }), + headers, + method: "POST", + }); + expect(response.status, `leaseId ${JSON.stringify(leaseId)}`).toBe(400); + expect( + ((await response.json()) as { error: { code: string } }).error.code, + `leaseId ${JSON.stringify(leaseId)}`, + ).toBe("BAD_REQUEST"); + } + + const longest = await fetch(`${baseUrl}/v1/lease-requests`, { + body: JSON.stringify({ device: "iPhone 16", leaseId: "a".repeat(64), platform: "ios" }), + headers, + method: "POST", + }); + expect(longest.status).toBe(201); + }); + + it("POST /v1/lease-requests answers 409 LEASE_ID_TAKEN, naming the ID, for an ID an active lease holds", async () => { + const port = await reservePort(); + const env = await withDaemon({ + configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, + driverScript: { ios: { availableOsVersions: ["18.4"], knownModels: ["iPhone 16"] } }, + }); + const baseUrl = `http://127.0.0.1:${port}`; + const tokens = await Promise.all( + [0, 1].map(async () => { + const result = await env.cli(["token", "create", "--role", "agent"]); + return `Bearer ${(result.json as { secret: string }).secret}`; + }), + ); + await waitFor( + async () => { + try { + return (await fetch(`${baseUrl}/v1/healthz`)).ok; + } catch { + return false; + } + }, + { label: "HTTP gateway accepting connections" }, + ); + const post = (authorization: string) => + fetch(`${baseUrl}/v1/lease-requests`, { + body: JSON.stringify({ device: "iPhone 16", leaseId: "ad-7f3a", platform: "ios" }), + headers: { authorization, "content-type": "application/json" }, + method: "POST", + }); + + const first = await post(tokens[0] ?? ""); + expect(first.status).toBe(201); + const second = await post(tokens[1] ?? ""); + expect(second.status).toBe(409); + expect(await second.json()).toMatchObject({ + error: { code: "LEASE_ID_TAKEN", leaseId: "ad-7f3a" }, + }); + }); }); diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index c62b714b..18ced57a 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -29,7 +29,7 @@ import { DaemonEndpointHost } from "../daemon/connection-host.js"; import { DaemonServer } from "../daemon/server.js"; import { AdminSecretManager } from "../daemon/admin-secret.js"; import { createCredentialRoleResolver } from "../daemon/session.js"; -import { SimlockError, type AnySimlockError } from "../contract/index.js"; +import { fromWireError, SimlockError, type AnySimlockError } from "../contract/index.js"; import type { CatalogGetOutput, DeviceRecoveredPush, @@ -3973,6 +3973,40 @@ describe("CLI: holder renew and release (ADR 0004 §2)", () => { expect(requested).toEqual([30 * 60_000, undefined]); }); + it("the CLI --lease-id flag sends leaseId and exits 13 on LEASE_ID_TAKEN", async () => { + const output = outputCapture(); + const requested: unknown[] = []; + let answer: "grant" | "taken" = "grant"; + const environment = output.environmentWith({ + clock: new FakeClock(0), + connectAdmin: async () => + fakeClient({ + requestLease: (input) => { + requested.push((input as Record).leaseId); + return answer === "grant" + ? Promise.resolve(detachedGrant) + : Promise.reject( + fromWireError("LEASE_ID_TAKEN", "lease ID ad-7f3a is already in use", { + leaseId: "ad-7f3a", + }), + ); + }, + }), + }); + const lease = ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"]; + + await runCli([...lease, "--lease-id", "ad-7f3a"], environment); + await runCli(lease, environment); + answer = "taken"; + const exitCode = await runCli([...lease, "--lease-id", "ad-7f3a"], environment); + + // The flag's value goes out as typed and an omitted flag sends none: the contract decides + // what an ID may look like. + expect(requested).toEqual(["ad-7f3a", undefined, "ad-7f3a"]); + expect(exitCode).toBe(13); + expect(output.stderr).toContain('"code":"LEASE_ID_TAKEN"'); + }); + it.each(["SIGINT", "SIGTERM"] as const)( "releases explicitly on %s (ADR 0004 §2's catchable signals)", async (signal) => { diff --git a/src/core/registry.test.ts b/src/core/registry.test.ts index 06f095cd..b2caa976 100644 --- a/src/core/registry.test.ts +++ b/src/core/registry.test.ts @@ -1584,6 +1584,93 @@ describe("Registry", () => { expect(reloaded.snapshot.leases[0]?.ownerId).toBe("agent-1"); }); + it("a state file without idChosenByRequester loads every lease as false", async () => { + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + const lease = (id: string, deviceId: string) => ({ + deviceId, + grantedAt: 1_000, + id, + lastRenewedAt: 1_000, + ownerId: "agent-1", + requesterId: "agent-1", + ttlDeadline: 2_000, + ttlMs: 60_000, + }); + const device = (id: string) => ({ + createdAt: 1_000, + driverData: {}, + driverDeviceId: `driver_${id}`, + id, + spec, + state: "leased", + }); + await filesystem.writeFileAtomic( + statePath, + JSON.stringify({ + devices: [device("dev_1"), device("dev_2")], + leases: [lease("lse_1", "dev_1"), lease("ad-7f3a", "dev_2")], + }), + ); + + const registry = await Registry.load({ + clock, + eventBus: new EventBus(clock), + filesystem, + idGenerator: { generate: () => "unexpected" }, + statePath, + }); + + expect(registry.snapshot.leases.map((loaded) => [loaded.id, loaded])).toEqual([ + ["lse_1", expect.objectContaining({ idChosenByRequester: false })], + ["ad-7f3a", expect.objectContaining({ idChosenByRequester: false })], + ]); + }); + + it("writes idChosenByRequester true for a lease created with a leaseId, false without one, and keeps both across a reload", async () => { + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + let next = 0; + const options = { + clock, + eventBus: new EventBus(clock), + filesystem, + idGenerator: { generate: () => String((next += 1)) }, + statePath, + }; + const registry = await Registry.load(options); + const ready = async (driverDeviceId: string) => { + const device = await registry.registerDevice({ + driverData: {}, + driverDeviceId, + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + return device.id; + }; + const base = { ownerId: "agent-1", requesterId: "agent-1", ttlDeadline: 2_000, ttlMs: 60_000 }; + + const chosen = await registry.createLease({ + ...base, + deviceId: await ready("one"), + leaseId: "ad-7f3a", + } as Parameters[0]); + const generated = await registry.createLease({ ...base, deviceId: await ready("two") }); + + expect(chosen).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); + expect(generated).toMatchObject({ + id: expect.stringMatching(/^lse_/), + idChosenByRequester: false, + }); + const reloaded = await Registry.load(options); + expect(reloaded.snapshot.leases).toEqual([chosen, generated]); + }); + it("migrates a lease record written before ADR 0004's ttlMs/lastRenewedAt, dropping mode", async () => { const filesystem = new MemoryFilesystem(); const clock = new FakeClock(5_000); diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index bb15d673..f81f3bb0 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -1,14 +1,15 @@ import { describe, expect, it } from "vitest"; import { EventBus, type EventMap } from "../bus/index.js"; -import { SimlockError } from "../contract/index.js"; +import type { LeaseRecord } from "../admin/index.js"; +import { fromWireError, SimlockError } from "../contract/index.js"; import { QueueTimeoutError } from "../leasing/index.js"; import { DispatchError } from "../daemon/dispatch.js"; import { FakeClock, type Logger } from "../ports/index.js"; import { promiseState } from "../test-support/promise-state.js"; import type { WorkerDirectory, WorkerDispatchTarget } from "./fleet-ports.js"; import { FleetLeaseCoordinator } from "./fleet-coordinator.js"; -import { FleetLeaseIndex } from "./lease-index.js"; +import { FleetLeaseIndex, type WorkerReportedLease } from "./lease-index.js"; import { RequesterAlreadyLeasedError } from "./queue.js"; import { createRoutingPolicy, type RoutingPolicy } from "./routing.js"; import { @@ -3575,3 +3576,192 @@ describe("FleetLeaseCoordinator: a worker that is still starting", () => { expect(starting.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); }); }); + +describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { + /** A worker's grant of `leaseId`, flagged as named by its requester. */ + function chosenGrant(leaseId: string) { + return grantFixture({ + lease: { ...grantFixture().lease, id: leaseId, idChosenByRequester: true } as LeaseRecord, + }); + } + + function leaseRequests(client: ScriptedWorkerClient): string[] { + return client.calls.filter((call) => call.startsWith("lease.request")); + } + + function forwardedLeaseId(client: ScriptedWorkerClient): unknown { + return (client.lastRequestLeaseInput as Record | undefined)?.leaseId; + } + + function oneWorker(overrides: Parameters[0] = {}) { + const fleet = harness(overrides); + const client = new ScriptedWorkerClient(); + fleet.directory.add("wrk_a", client); + connectWorker(fleet.workers, "wrk_a"); + return { ...fleet, client }; + } + + /** `leaseId` rides beside the other request options. */ + function chosen(leaseId: string, overrides: Parameters[0] = {}) { + return { ...requestOptions(overrides), leaseId } as ReturnType; + } + + it("through a gateway, a caller-chosen ID comes back with no worker prefix", async () => { + const { client, coordinator, leaseIndex } = oneWorker(); + client.requestLeaseQueue.push({ grant: chosenGrant("ad-7f3a"), kind: "grant" }); + + const grant = await coordinator.request(REQUEST, chosen("ad-7f3a")); + + expect(forwardedLeaseId(client)).toBe("ad-7f3a"); + expect(grant.lease.id).toBe("ad-7f3a"); + expect(leaseIndex.resolve("ad-7f3a")).toMatchObject({ + gatewayLeaseId: "ad-7f3a", + workerId: "wrk_a", + workerLeaseId: "ad-7f3a", + }); + }); + + it("through a gateway, a request without leaseId gets .lse_ as today", async () => { + const { client, coordinator } = oneWorker(); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grant = await coordinator.request(REQUEST, requestOptions()); + + expect(forwardedLeaseId(client)).toBeUndefined(); + expect(grant.lease.id).toBe("wrk_a.lse_1"); + }); + + it("through a gateway, renew and release by a caller-chosen ID reach the right worker", async () => { + const { client, coordinator, leaseIndex } = oneWorker(); + client.requestLeaseQueue.push({ grant: chosenGrant("ad-7f3a"), kind: "grant" }); + await coordinator.request(REQUEST, chosen("ad-7f3a")); + + await coordinator.renew("ad-7f3a", undefined); + await coordinator.release("ad-7f3a"); + + expect(client.calls).toEqual( + expect.arrayContaining(["lease.renew:ad-7f3a", "lease.release:ad-7f3a"]), + ); + expect(leaseIndex.resolve("ad-7f3a")).toBeUndefined(); + }); + + it("through a gateway, a second request for an ID held by a gateway lease fails with LEASE_ID_TAKEN, emits lease.rejected with reason lease-id-taken, and is not forwarded", async () => { + const { client, coordinator, eventBus } = oneWorker(); + client.requestLeaseQueue.push({ grant: chosenGrant("myid"), kind: "grant" }); + await coordinator.request(REQUEST, chosen("myid")); + const rejected: unknown[] = []; + eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + + const second = coordinator.request( + REQUEST, + chosen("myid", { ownerId: "agent-2", requesterId: "agent-2" }), + ); + const secondState = promiseState(second); + const refusal = second.catch((error: unknown) => error); + await tick(); + + expect(secondState.state).toBe("rejected"); + expect(await refusal).toMatchObject({ code: "LEASE_ID_TAKEN", details: { leaseId: "myid" } }); + expect(rejected).toEqual([ + expect.objectContaining({ reason: "lease-id-taken", requester: "agent-2" }), + ]); + expect(leaseRequests(client)).toHaveLength(1); + }); + + it("through a gateway, a second request for an ID held by a waiting gateway request fails with LEASE_ID_TAKEN and is not forwarded", async () => { + const { client, coordinator, directory, workers } = oneWorker(); + // The only worker is full, so the first request waits in the gateway queue. + connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); + directory.add("wrk_a", client); + void coordinator.request(REQUEST, chosen("myid")).catch(() => undefined); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + const second = coordinator.request( + REQUEST, + chosen("myid", { ownerId: "agent-2", requesterId: "agent-2" }), + ); + const secondState = promiseState(second); + const refusal = second.catch((error: unknown) => error); + await tick(); + + expect(secondState.state).toBe("rejected"); + expect(await refusal).toMatchObject({ code: "LEASE_ID_TAKEN" }); + expect(leaseRequests(client)).toEqual([]); + expect(coordinator.queueDepth).toBe(1); + }); + + it("through a gateway, a worker's LEASE_ID_TAKEN is passed to the caller and no other worker is tried", async () => { + const { client, coordinator, directory, workers } = oneWorker(); + const other = new ScriptedWorkerClient(); + directory.add("wrk_b", other); + connectWorker(workers, "wrk_b"); + // Both workers are free. Each one refuses once and would grant on a second ask. + const taken = fromWireError("LEASE_ID_TAKEN", "lease ID myid is already in use", { + leaseId: "myid", + }); + for (const scripted of [client, other]) { + scripted.requestLeaseQueue.push( + { error: taken, kind: "error" }, + { grant: chosenGrant("myid"), kind: "grant" }, + ); + } + + await expect(coordinator.request(REQUEST, chosen("myid"))).rejects.toMatchObject({ + code: "LEASE_ID_TAKEN", + details: { leaseId: "myid" }, + }); + + expect(leaseRequests(client).length + leaseRequests(other).length).toBe(1); + }); + + it("a worker grant whose lease ID differs from the forwarded leaseId is released on the worker, never enters the gateway index, and the request is queued again", async () => { + const logger = new RecordingLogger(); + const { client, coordinator, leaseIndex } = oneWorker({ logger }); + client.requestLeaseQueue.push({ grant: chosenGrant("not-what-was-sent"), kind: "grant" }); + const outcome = promiseState(coordinator.request(REQUEST, chosen("myid"))); + await tick(); + + expect(client.calls).toContain("lease.release:not-what-was-sent"); + expect(leaseIndex.all()).toEqual([]); + expect(outcome.state).toBe("pending"); + expect(coordinator.queueDepth).toBe(1); + expect(JSON.stringify(logger.warnings)).toContain("wrk_a"); + }); + + it("the gateway answers UNKNOWN_LEASE for a renew of an ID missing from its index", async () => { + const { coordinator } = oneWorker(); + + await expect(coordinator.renew("myid", undefined)).rejects.toMatchObject({ + code: "UNKNOWN_LEASE", + details: { leaseId: "myid" }, + }); + }); + + it("a grant for a bare ID the index maps to another worker is released on the new worker and answers LEASE_ID_TAKEN, and the first entry stays routed", async () => { + const { client, coordinator, leaseIndex } = oneWorker(); + client.requestLeaseQueue.push({ + beforeGrant: () => { + // While wrk_a's grant is on its way, wrk_b reports a lease with the same ID. + leaseIndex.rebuildFromWorker("wrk_b", [ + { + grantedAt: 1, + id: "myid", + idChosenByRequester: true, + ownerId: "agent-9", + requesterId: `${GATEWAY_PREFIX}agent-9`, + } as WorkerReportedLease, + ]); + }, + grant: chosenGrant("myid"), + kind: "grant", + }); + + await expect(coordinator.request(REQUEST, chosen("myid"))).rejects.toMatchObject({ + code: "LEASE_ID_TAKEN", + }); + + expect(client.calls).toContain("lease.release:myid"); + expect(leaseIndex.resolve("myid")).toMatchObject({ workerId: "wrk_b" }); + }); +}); diff --git a/src/gateway/lease-index.test.ts b/src/gateway/lease-index.test.ts index e155ddbb..a4823f65 100644 --- a/src/gateway/lease-index.test.ts +++ b/src/gateway/lease-index.test.ts @@ -302,4 +302,88 @@ describe("FleetLeaseIndex", () => { }); }); }); + + describe("caller-chosen lease IDs", () => { + /** A lease a worker reports as named by its requester. */ + function chosen(id: string, overrides: Partial = {}): WorkerReportedLease { + return { ...reported(id, overrides), idChosenByRequester: true } as WorkerReportedLease; + } + + it("after a gateway restart, a caller-chosen lease is rebuilt under its bare ID", () => { + const index = new FleetLeaseIndex(PREFIX); + + index.rebuildFromWorker("wrk_1", [chosen("ad-7f3a"), reported("lse_9")]); + + expect(index.resolve("ad-7f3a")).toEqual({ + gatewayLeaseId: "ad-7f3a", + grantedAt: 1, + ownerId: "alice-principal", + requesterId: "alice", + workerId: "wrk_1", + workerLeaseId: "ad-7f3a", + }); + expect(index.resolve("wrk_1.lse_9")).toMatchObject({ workerLeaseId: "lse_9" }); + expect(index.resolve("wrk_1.ad-7f3a")).toBeUndefined(); + }); + + it("a rebuilt lease flagged idChosenByRequester whose ID does not match the leaseId pattern is prefixed and logged", () => { + const logger = new RecordingLogger(); + const index = new FleetLeaseIndex(PREFIX, logger); + + index.rebuildFromWorker("wrk_1", [chosen("not.a-valid id")]); + + expect(index.resolve("wrk_1.not.a-valid id")).toMatchObject({ workerId: "wrk_1" }); + expect(index.resolve("not.a-valid id")).toBeUndefined(); + expect(logger.warnings).toEqual([ + expect.objectContaining({ fields: expect.objectContaining({ workerId: "wrk_1" }) }), + ]); + }); + + it("when two workers report the same caller-chosen ID on rebuild, the first stays routed and a warning names both workers", () => { + const logger = new RecordingLogger(); + const index = new FleetLeaseIndex(PREFIX, logger); + + index.rebuildFromWorker("wrk_1", [chosen("myid")]); + index.rebuildFromWorker("wrk_2", [chosen("myid", { ownerId: "bob-principal" })]); + + expect(index.resolve("myid")).toMatchObject({ + ownerId: "alice-principal", + workerId: "wrk_1", + }); + expect(index.findByWorkerLease("wrk_2", "myid")).toBeUndefined(); + expect(index.all()).toHaveLength(1); + expect(logger.warnings).toHaveLength(1); + expect(JSON.stringify(logger.warnings[0]?.fields)).toContain("wrk_1"); + expect(JSON.stringify(logger.warnings[0]?.fields)).toContain("wrk_2"); + }); + + it("add never overwrites a bare entry that belongs to another worker, and says it did not add", () => { + const index = new FleetLeaseIndex(PREFIX); + const first = entry({ gatewayLeaseId: "myid", workerId: "wrk_1", workerLeaseId: "myid" }); + expect(index.add(first)).toBe(true); + + const added = index.add( + entry({ gatewayLeaseId: "myid", workerId: "wrk_2", workerLeaseId: "myid" }), + ); + + expect(added).toBe(false); + expect(index.resolve("myid")).toEqual(first); + expect(index.findByWorkerLease("wrk_2", "myid")).toBeUndefined(); + }); + + it("a second worker reporting the same bare ID does not reset the first entry's missing count", () => { + const index = new FleetLeaseIndex(PREFIX); + index.rebuildFromWorker("wrk_1", [chosen("myid")]); + // wrk_1 stops reporting it: missing once. + index.rebuildFromWorker("wrk_1", []); + expect(index.resolve("myid")).toBeDefined(); + + // wrk_2 reports the same ID: that is not a report from wrk_1. + index.rebuildFromWorker("wrk_2", [chosen("myid")]); + // wrk_1 omits it a second time in a row: gone. + index.rebuildFromWorker("wrk_1", []); + + expect(index.resolve("myid")).toBeUndefined(); + }); + }); }); diff --git a/src/gateway/test-support.ts b/src/gateway/test-support.ts index c1460c93..f5f053d6 100644 --- a/src/gateway/test-support.ts +++ b/src/gateway/test-support.ts @@ -172,6 +172,9 @@ export type RequestLeaseOutcome = readonly kind: "grant"; readonly grant: LeaseGrant; readonly progress?: readonly LeaseProgressLike[]; + /** Runs after the worker has decided to grant and before the answer reaches the gateway: + * what lands on the gateway while the grant is on its way. */ + readonly beforeGrant?: () => void; } | { readonly kind: "error"; @@ -387,6 +390,7 @@ export class ScriptedWorkerClient { for (const progress of outcome.progress ?? []) options.onProgress?.(progress); if (outcome.kind === "hang") return new Promise(() => {}); if (outcome.kind === "error") throw outcome.error; + outcome.beforeGrant?.(); return outcome.grant; } diff --git a/src/leasing/create-leasing.test.ts b/src/leasing/create-leasing.test.ts index fe707fc9..762c7901 100644 --- a/src/leasing/create-leasing.test.ts +++ b/src/leasing/create-leasing.test.ts @@ -3545,3 +3545,197 @@ describe("createLeasing's port for the warm pool", () => { expect([defaultMode("ios"), defaultMode("android")]).toEqual(["slim", "full"]); }); }); + +describe("createLeasing a lease ID chosen by the requester", () => { + const roomy = { + android: { maxDevices: 2, maxRunning: 2 }, + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 4, + }; + + /** What a caller that sends `leaseId` passes: the field is spread in so this file still builds + * against options that do not name it. */ + function asking(requesterId: string, leaseId?: string, more: Record = {}) { + return { + ownerId: requesterId, + requesterId, + ...(leaseId === undefined ? {} : { leaseId }), + ...more, + }; + } + + /** Leaves `requesterId`'s request for `leaseId` waiting behind the one device. */ + async function withWaiting(requesterId: string, leaseId: string) { + const harness = await createHarness(); + await harness.engine.request(request, asking("holder")); + void harness.engine.request(request, asking(requesterId, leaseId)).catch(() => undefined); + await expect + .poll(() => + harness.registry.leaseRequests().find((record) => record.requesterId === requesterId), + ) + .toMatchObject({ state: "open" }); + return harness; + } + + it("a lease request with leaseId gets a lease with exactly that ID", async () => { + const harness = await createHarness(); + + const grant = await harness.engine.request(request, asking("agent-1", "ad-7f3a")); + + expect(grant.lease.id).toBe("ad-7f3a"); + expect(grant.lease).toMatchObject({ idChosenByRequester: true }); + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual(["ad-7f3a"]); + }); + + it("a lease request without leaseId gets an lse_ ID as today", async () => { + const harness = await createHarness(); + + const grant = await harness.engine.request(request, asking("agent-1")); + + expect(grant.lease.id).toMatch(/^lse_/); + expect(grant.lease).toMatchObject({ idChosenByRequester: false }); + }); + + it("renew and release by a caller-chosen ID work", async () => { + const harness = await createHarness(); + await harness.engine.request(request, asking("agent-1", "ad-7f3a")); + + await expect(harness.engine.renew("ad-7f3a", 5_000)).resolves.toMatchObject({ + id: "ad-7f3a", + ttlMs: 5_000, + }); + await harness.engine.release("ad-7f3a", "explicit"); + + expect(harness.registry.snapshot.leases).toEqual([]); + expect( + harness.bus.replay().find((event) => event.event === "lease.released")?.payload, + ).toMatchObject({ + leaseId: "ad-7f3a", + }); + }); + + it("MyID and myid are two different IDs", async () => { + const harness = await createHarness({ limits: roomy }); + + const first = await harness.engine.request(request, asking("agent-1", "myid")); + const second = await harness.engine.request(request, asking("agent-2", "MyID")); + + expect([first.lease.id, second.lease.id]).toEqual(["myid", "MyID"]); + }); + + it("a second request for an ID held by an active lease fails with LEASE_ID_TAKEN", async () => { + const harness = await createHarness({ limits: roomy }); + await harness.engine.request(request, asking("agent-1", "myid")); + + await expect(harness.engine.request(request, asking("agent-2", "myid"))).rejects.toMatchObject({ + leaseId: "myid", + name: "LeaseIdTakenError", + }); + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual(["myid"]); + }); + + it("a second request for an ID held by a waiting request fails with LEASE_ID_TAKEN", async () => { + const harness = await withWaiting("agent-2", "myid"); + + await expect(harness.engine.request(request, asking("agent-3", "myid"))).rejects.toMatchObject({ + leaseId: "myid", + name: "LeaseIdTakenError", + }); + }); + + it("a requester that already holds a lease gets REQUESTER_ALREADY_LEASED, not LEASE_ID_TAKEN, even when it repeats its own lease ID", async () => { + const harness = await createHarness({ limits: roomy }); + const first = await harness.engine.request(request, asking("agent-1", "myid")); + + await expect(harness.engine.request(request, asking("agent-1", "myid"))).rejects.toMatchObject({ + existingLeaseId: first.lease.id, + name: "RequesterAlreadyLeasedError", + }); + }); + + it("a LEASE_ID_TAKEN refusal emits lease.rejected with reason lease-id-taken", async () => { + const harness = await createHarness({ limits: roomy }); + await harness.engine.request(request, asking("agent-1", "myid")); + + await harness.engine.request(request, asking("agent-2", "myid")).catch(() => undefined); + + const rejected = harness.bus.replay().filter((event) => event.event === "lease.rejected"); + expect(rejected.map((event) => event.payload)).toEqual([ + { + reason: "lease-id-taken", + requestId: expect.stringMatching(/^req_/), + requester: "agent-2", + requestSpec: request, + }, + ]); + }); + + it("lease.requested and lease.rejected requestSpec, and the status waiting spec, carry no leaseId", async () => { + const harness = await withWaiting("agent-2", "myid"); + await harness.engine.request(request, asking("agent-3", "myid")).catch(() => undefined); + + const requested = harness.bus.replay().filter((event) => event.event === "lease.requested"); + const rejected = harness.bus.replay().filter((event) => event.event === "lease.rejected"); + expect(requested.map((event) => event.payload)).toContainEqual( + expect.objectContaining({ requester: "agent-2", requestSpec: request }), + ); + expect( + rejected.map((event) => (event.payload as { requestSpec: unknown }).requestSpec), + ).toEqual([request]); + expect(harness.engine.waitingRequests().map((entry) => entry.spec)).toEqual([request]); + expect( + harness.registry.leaseRequests().find((record) => record.requesterId === "agent-2")?.request, + ).toEqual(request); + }); + + it("a retry with the same idempotency key and the same leaseId replays the first answer", async () => { + const harness = await createHarness(); + const asker = asking("agent-1", "myid", { idempotencyKey: "key-1" }); + const first = await harness.engine.request(request, asker); + + const retry = await harness.engine.request(request, asker); + + expect(retry).toEqual(first); + expect(retry.lease.id).toBe("myid"); + expect(harness.registry.snapshot.leases).toHaveLength(1); + }); + + it.each([ + ["a different leaseId", "other"], + ["no leaseId", undefined], + ])( + "a retry with the same idempotency key and %s fails with IDEMPOTENCY_CONFLICT", + async (_name, retryId) => { + const harness = await createHarness(); + await harness.engine.request(request, asking("agent-1", "myid", { idempotencyKey: "key-1" })); + + await expect( + harness.engine.request(request, asking("agent-1", retryId, { idempotencyKey: "key-1" })), + ).rejects.toMatchObject({ name: "IdempotencyConflictError" }); + }, + ); + + it("a retry with the same idempotency key that adds a leaseId the first call did not have fails with IDEMPOTENCY_CONFLICT", async () => { + const harness = await createHarness(); + const first = await harness.engine.request( + request, + asking("agent-1", undefined, { idempotencyKey: "key-1" }), + ); + + await expect( + harness.engine.request(request, asking("agent-1", "myid", { idempotencyKey: "key-1" })), + ).rejects.toMatchObject({ name: "IdempotencyConflictError" }); + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual([first.lease.id]); + }); + + it("after a daemon restart, an ID held only by a waiting request is accepted again", async () => { + const before = await withWaiting("agent-2", "myid"); + // Started again with room to spare, so the only thing that could refuse "myid" is the ID. + const after = await createHarness({ filesystem: before.filesystem, limits: roomy }); + await after.engine.convergeRunningCapacity(); + + await expect(after.engine.request(request, asking("agent-3", "myid"))).resolves.toMatchObject({ + lease: { id: "myid" }, + }); + }); +}); diff --git a/src/mcp/server.test.ts b/src/mcp/server.test.ts index 824b951d..e0e0d9f0 100644 --- a/src/mcp/server.test.ts +++ b/src/mcp/server.test.ts @@ -120,6 +120,31 @@ describe("MCP server (smoke)", () => { } }); + it("the MCP lease tool passes leaseId through", async () => { + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "ad-7f3a" })); + const { mcpClient, close } = await connectedServer(client); + try { + const tools = await mcpClient.request({ method: "tools/list" }, ListToolsResultSchema); + const leaseTool = tools.tools.find((tool) => tool.name === "lease_simulator"); + expect(Object.keys(leaseTool?.inputSchema.properties ?? {})).toContain("leaseId"); + + const lease = await call(mcpClient, "lease_simulator", { + leaseId: "ad-7f3a", + model: "iPhone 17 Pro", + platform: "ios", + }); + + expect(lease.isError).not.toBe(true); + expect(client.calls[0]).toMatchObject({ + input: { leaseId: "ad-7f3a", model: "iPhone 17 Pro", platform: "ios" }, + method: "requestLease", + }); + } finally { + await close(); + } + }); + it("surfaces a daemon FORBIDDEN as-is when releasing a lease this session does not own -- no client-side pre-check", async () => { const client = new FakeSimlockClient(); client.releaseLeaseImpl = () => From 456605d8637b82d116ea70c56e45aff18dc9031d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:06:15 +0200 Subject: [PATCH 2/9] feat: a requester chooses its lease ID on a host and through a gateway (#410) 24 failing -> 0 failing in the unit lane; e2e not yet run --- e2e/console/long-lists.spec.ts | 1 + src/bus/index.ts | 4 +- src/cli/index.test.ts | 16 +++- src/cli/index.ts | 6 +- src/contract/errors.ts | 15 +++- src/contract/operations.test.ts | 2 + src/contract/operations.ts | 22 ++++- src/contract/protocol.ts | 8 +- src/contract/schemas.test.ts | 1 + src/contract/schemas.ts | 5 ++ src/core/cleanup/idle-destroy.test.ts | 1 + src/core/domain.ts | 12 +++ src/core/lease-request-store.ts | 2 + src/core/nuke-service.test.ts | 1 + src/core/reclaim-coordinator.test.ts | 1 + src/core/registry.test.ts | 3 + src/core/registry.ts | 26 +++++- src/core/startup-converger.test.ts | 3 + src/core/warm-pool/converger.test.ts | 1 + src/core/warm-pool/policy.test.ts | 1 + src/daemon-protocol/index.test.ts | 4 +- src/daemon/dispatcher.ts | 12 ++- src/daemon/error-code.ts | 4 + src/daemon/server.test.ts | 5 +- src/gateway/dispatcher.ts | 2 + src/gateway/fleet-coordinator.test.ts | 14 +++ src/gateway/fleet-coordinator.ts | 91 ++++++++++++++++++-- src/gateway/lease-index.ts | 54 ++++++++++-- src/gateway/test-support.ts | 1 + src/http/app.ts | 4 + src/http/test-fakes.ts | 1 + src/http/tracker.ts | 3 + src/leasing/acquisition-planner.test.ts | 2 + src/leasing/create-leasing.test.ts | 18 ++++ src/leasing/index.ts | 1 + src/leasing/lease-acquisition-coordinator.ts | 36 +++++++- src/leasing/lease-expiry-scheduler.test.ts | 1 + src/leasing/lease-lifecycle.ts | 4 + src/leasing/lease-request-book.ts | 23 +++-- src/leasing/wait-queue.ts | 14 +++ src/mcp/contracts.test.ts | 1 + src/mcp/server.test.ts | 1 + src/mcp/session.test.ts | 10 +++ src/mcp/test-support.ts | 1 + src/simlock-client/client.test.ts | 1 + ui/src/views/leases.test.tsx | 1 + ui/src/views/stats.test.tsx | 1 + ui/src/views/worker-detail.test.tsx | 1 + ui/src/views/workers.test.tsx | 3 + 49 files changed, 409 insertions(+), 36 deletions(-) diff --git a/e2e/console/long-lists.spec.ts b/e2e/console/long-lists.spec.ts index 9edfd0f4..d923df27 100644 --- a/e2e/console/long-lists.spec.ts +++ b/e2e/console/long-lists.spec.ts @@ -139,6 +139,7 @@ function leases(count: number): unknown[] { grantedAt: now - 3_600_000 + index * 1_000, id: `lse_${pad(index + 1)}`, lastRenewedAt: now - 60_000, + idChosenByRequester: false, ownerId: "tok_load", requesterId: "tok_load", ttlDeadline: now + 600_000, diff --git a/src/bus/index.ts b/src/bus/index.ts index 9fb8d3a9..6b5bd239 100644 --- a/src/bus/index.ts +++ b/src/bus/index.ts @@ -59,7 +59,7 @@ export interface EventMap { readonly ownerId: string; }; "lease.rejected": { - /** The request's id. A request refused at admission (`killed`, `already-leased`) was never + /** The request's id. A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never * stored and has no `lease.requested`, but it carries the id it would have been stored * under. */ readonly requestId: string; @@ -74,6 +74,8 @@ export interface EventMap { /** A gateway's request that no worker taking requests can serve (ADR 0009 §4). */ | "no-worker" | "already-leased" + /** The request named a lease ID an active lease or a waiting request holds (ADR 0020). */ + | "lease-id-taken" | "boot-timeout" | "killed" | "cancelled" diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 18ced57a..ef5969b4 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -82,6 +82,7 @@ const detachedGrant: LeaseGrant = { ownerId: "test-requester", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 61_000, }, @@ -391,6 +392,7 @@ describe("CLI: exit codes", () => { grantedAt: 0, id: "lse_mine", lastRenewedAt: 0, + idChosenByRequester: false, ownerId: "some-other-principal", requesterId: "test-requester", ttlDeadline: 60_000, @@ -2584,6 +2586,7 @@ describe("CLI: worker commands (ADR 0005 §8/§23)", () => { grantedAt: 1, id: "lease_1", lastRenewedAt: 1, + idChosenByRequester: false, ownerId: "agent-1", requesterId: "agent-1", ttlDeadline: 2, @@ -2886,6 +2889,7 @@ describe("CLI: status renders the fleet a gateway reports (ADR 0005 §20)", () = grantedAt: 1, id: "lease_1", lastRenewedAt: 1, + idChosenByRequester: false, ownerId: "agent-1", requesterId: "agent-1", ttlDeadline: 2, @@ -3543,6 +3547,7 @@ describe("CLI: lease pushes and exit codes (own logic, not the dispatcher's)", ( ownerId: "test-requester", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 60_000, }, @@ -3598,6 +3603,7 @@ describe("CLI: lease pushes and exit codes (own logic, not the dispatcher's)", ( ownerId: "test-requester", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 60_000, }, @@ -3648,6 +3654,7 @@ describe("CLI: lease pushes and exit codes (own logic, not the dispatcher's)", ( ownerId: "test-requester", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 60_000, }, @@ -3710,6 +3717,7 @@ describe("CLI: lease pushes and exit codes (own logic, not the dispatcher's)", ( ownerId: "test-requester", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 60_000, }, @@ -3755,6 +3763,7 @@ describe("CLI: holder renew and release (ADR 0004 §2)", () => { ownerId: "test-requester", requesterId: "test-requester", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: clock.now() + 30_000, }); @@ -3858,6 +3867,7 @@ describe("CLI: holder renew and release (ADR 0004 §2)", () => { ownerId: "test-requester", requesterId: "test-requester", lastRenewedAt: clock.now(), + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: clock.now() + 60_000, }), @@ -4245,6 +4255,7 @@ describe("CLI: holder renew and release (ADR 0004 §2)", () => { ownerId: "test-requester", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 60_000, }, @@ -4315,6 +4326,7 @@ describe("CLI: holder renew and release (ADR 0004 §2)", () => { ownerId: "test-requester", requesterId: "test-requester", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: clock.now() + 60_000, }); @@ -5039,6 +5051,7 @@ function fakeClient(overrides: Partial = {}): SimlockAdminCl ownerId: "test-requester", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 60_000, }, @@ -5549,6 +5562,7 @@ describe("simlock lease: a request names a model, a class, or nothing", () => { grantedAt: 0, id: "lse_1", lastRenewedAt: 0, + idChosenByRequester: false, ownerId: "test-requester", requesterId: "test-requester", ttlDeadline: 60_000, @@ -5607,7 +5621,7 @@ describe("simlock lease: a request names a model, a class, or nothing", () => { expect(output.stdout).toBe( "Usage: simlock lease --platform [--device | --class ]\n" + " [--os ] [--mode ] [--image-tag ] [--agent-id ]\n" + - " [--timeout ]\n" + + " [--timeout ] [--lease-id ]\n" + " [--no-wait] [--detach] [--ttl ] [--allow-download]\n" + " [--export-env] [--bind-pid ]\n", ); diff --git a/src/cli/index.ts b/src/cli/index.ts index 68b4cf0d..6eb0e68c 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -943,6 +943,7 @@ async function runLease( "export-env": { type: "boolean" }, help: { type: "boolean", short: "h" }, "image-tag": { type: "string" }, + "lease-id": { type: "string" }, mode: { type: "string" }, "no-wait": { type: "boolean" }, os: { type: "string" }, @@ -954,7 +955,7 @@ async function runLease( environment.stdout.write( "Usage: simlock lease --platform [--device | --class ]\n" + " [--os ] [--mode ] [--image-tag ] [--agent-id ]\n" + - " [--timeout ]\n" + + " [--timeout ] [--lease-id ]\n" + " [--no-wait] [--detach] [--ttl ] [--allow-download]\n" + " [--export-env] [--bind-pid ]\n", ); @@ -1113,6 +1114,9 @@ async function runLease( // Sent as typed too: the contract bounds it, and the platform's driver decides whether // the platform has image tags at all. ...(typeof values["image-tag"] === "string" ? { imageTag: values["image-tag"] } : {}), + // Sent as typed as well: the contract bounds what an ID may look like (`BAD_REQUEST`), and + // the daemon refuses one already in use (`LEASE_ID_TAKEN`, exit 13). + ...(typeof values["lease-id"] === "string" ? { leaseId: values["lease-id"] } : {}), ...(timeoutMs === undefined ? {} : { timeoutMs }), ...(ttlMs === undefined ? {} : { ttlMs }), }, diff --git a/src/contract/errors.ts b/src/contract/errors.ts index 7400c212..5ec36812 100644 --- a/src/contract/errors.ts +++ b/src/contract/errors.ts @@ -43,8 +43,12 @@ export interface ErrorDetailsMap { QUEUE_TIMEOUT: { readonly requestId: string }; REQUESTER_ALREADY_LEASED: { readonly requesterId: string; readonly existingLeaseId?: string }; /** `lease.request` repeated an `idempotencyKey` its requester already used for a different - * device. The stored request is untouched; a new request needs a new key. */ + * device or a different `leaseId` (a missing one on either side counts as different). The + * stored request is untouched; a new request needs a new key. */ IDEMPOTENCY_CONFLICT: Record; + /** `lease.request` named a `leaseId` an active lease or a waiting request already holds + * (ADR 0020). Nothing was stored. */ + LEASE_ID_TAKEN: { readonly leaseId: string }; NO_DRIVER: { readonly platform: Platform }; RUNTIME_MISSING: { readonly platform: Platform; @@ -189,6 +193,7 @@ const CODES_WITH_DECLARED_DETAILS_BY_CODE: Record PROTOCOL_VERSION_UNSUPPORTED: true, QUEUE_TIMEOUT: true, REQUESTER_ALREADY_LEASED: true, + LEASE_ID_TAKEN: true, NO_DRIVER: true, RUNTIME_MISSING: true, UNKNOWN_MODEL: true, @@ -278,6 +283,14 @@ export const ERROR_TABLE: { readonly [Code in SimlockErrorCode]: ErrorTableEntry cliExitCode: 2, httpStatus: 409, }, + // 409 and exit 13 like `REQUESTER_ALREADY_LEASED`, its neighbour: the request clashes with a + // lease or request that is already there. + LEASE_ID_TAKEN: { + code: "LEASE_ID_TAKEN", + kind: "domain", + cliExitCode: 13, + httpStatus: 409, + }, NO_DRIVER: { code: "NO_DRIVER", kind: "domain", cliExitCode: 12, httpStatus: 422 }, RUNTIME_MISSING: { code: "RUNTIME_MISSING", kind: "domain", cliExitCode: 12, httpStatus: 422 }, UNKNOWN_MODEL: { code: "UNKNOWN_MODEL", kind: "domain", cliExitCode: 12, httpStatus: 422 }, diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index f62ac1fe..5b511cff 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -314,6 +314,7 @@ describe("operation input/output round trips", () => { ownerId: "req_1", grantedAt: 1, lastRenewedAt: 1, + idChosenByRequester: false, ttlMs: 1, ttlDeadline: 2, }, @@ -616,6 +617,7 @@ describe("operation input/output round trips", () => { ttlMs: 2, ttlDeadline: 3, lastRenewedAt: 1, + idChosenByRequester: false, workerId: "wrk_1", }; const parsed = OPERATIONS["status.get"].output.parse({ diff --git a/src/contract/operations.ts b/src/contract/operations.ts index cf0c7eee..2f8087ec 100644 --- a/src/contract/operations.ts +++ b/src/contract/operations.ts @@ -168,6 +168,17 @@ export const statusGet = defineOperation({ * `imageTag` names the type of installed image the device is created from, as the catalog lists * it; a request that names one never downloads, and a platform without image types refuses it. */ +/** + * The lease ID a requester may choose (ADR 0020 §1): ASCII, 1 to 64 characters, starting with a + * letter or digit, then letters, digits, `-` and `_`, case-sensitive. It has no `.`, so it can + * never look like a gateway's `.`. The one definition: a gateway reads it too, + * to decide which reported lease it names bare. + */ +export const LEASE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; + +/** `leaseId` on the wire: a string of that shape, for every transport that takes one. */ +export const leaseIdSchema = z.string().regex(LEASE_ID_PATTERN); + const leaseRequestBaseSchema = z .object({ /** @@ -213,11 +224,20 @@ const leaseRequestBaseSchema = z * Makes this request repeatable: the daemon stores it under `(requesterId, idempotencyKey)`, * and the same request sent again returns the stored result instead of a second lease -- * the way a client that lost its answer, to a disconnect or a daemon restart, gets it back. - * The same key naming a different device is `IDEMPOTENCY_CONFLICT`; a repeat from another + * The same key naming a different device, or a different `leaseId` (a missing one on either + * side counts as different), is `IDEMPOTENCY_CONFLICT`; a repeat from another * principal is `FORBIDDEN`. Optional: a request without one is still stored, it just cannot * be repeated. Bounded because the daemon stores it. */ idempotencyKey: z.string().min(1).max(200).optional(), + /** + * ADR 0020: the ID the granted lease gets, instead of one simlock generates. The requester + * guarantees it is unique for all time; simlock refuses one an active lease or a waiting + * request holds (`LEASE_ID_TAKEN`) and keeps no record of IDs already used. A request + * without one works as before. Like `idempotencyKey` it is an option of the request, not + * part of the device it names. Anything outside the pattern is `BAD_REQUEST`. + */ + leaseId: leaseIdSchema.optional(), }) .strict(); diff --git a/src/contract/protocol.ts b/src/contract/protocol.ts index 745bb287..7b44649d 100644 --- a/src/contract/protocol.ts +++ b/src/contract/protocol.ts @@ -14,7 +14,7 @@ export interface ProtocolRange { const protocolRangeSchema = z.object({ min: z.number().int(), max: z.number().int() }); /** - * The range this contract's daemon speaks. Both ends are 13, and every move that got it there + * The range this contract's daemon speaks. Both ends are 21, and every move that got it there * was breaking with no shim kept behind them, which is exactly when ADR 0003 §6's honesty * rule says a range must *not* widen: ADR 0004 removed `lease.heartbeat` and `mode` from the * wire (taking it to 4), and ADR 0005 adds `device.exec` and its `output` push family, a @@ -29,7 +29,7 @@ const protocolRangeSchema = z.object({ min: z.number().int(), max: z.number().in * workers, so a worker without it must be `incompatible` rather than fail in the middle of a * relay. ADR 0014 gives every event envelope an `id` (taking it to 10): a worker's pushed events * without one would fail the gateway's schema, so a worker on 9 is `incompatible` instead. ADR 0009 §7 makes `atRamBudget` a required field of each platform's `status.get` - * capacity (taking it to 11), which a gateway routing on it depends on. ADR 0015 §3 makes `modelClasses` a required field of each platform's catalog (taking it to 12): a gateway reading a worker's catalog would fail its schema without it, so a worker on 11 is `incompatible`. ADR 0015 §4 makes `classDefaults` a required field of each platform's catalog too (taking it to 13), for the same reason: a worker on 12 is `incompatible`. ADR 0009 §6 makes `servesDefaultMode` a required field of each device in `status.get` (taking it to 16), which a gateway telling a warm hit for a request with no mode depends on: a worker on 15 is `incompatible`. ADR 0017 §4 and §8 make `warmPool.reserveRunning` a required field of `config.get`'s output (taking it to 17): a client parsing an older daemon's `config.get` would fail its schema without it, so a daemon on 16 is `incompatible`. While a daemon is `starting`, its `status.get` answers `daemon` and `host` alone, so `devices`, `leases`, `capacity` and `queueDepth` become optional (taking it to 18): a peer on 17 cannot parse that answer, and a gateway reading it from a worker on 17 would fail its schema, so a worker on 17 is `incompatible`. `warmPool.targets` and `warmPool.maxConcurrentBoots` are required fields of `config.get`'s output too (taking it to 19), for the same reason: a daemon on 18 is `incompatible`. It only ever widens once a second version is actually kept alive side by side with the + * capacity (taking it to 11), which a gateway routing on it depends on. ADR 0015 §3 makes `modelClasses` a required field of each platform's catalog (taking it to 12): a gateway reading a worker's catalog would fail its schema without it, so a worker on 11 is `incompatible`. ADR 0015 §4 makes `classDefaults` a required field of each platform's catalog too (taking it to 13), for the same reason: a worker on 12 is `incompatible`. ADR 0009 §6 makes `servesDefaultMode` a required field of each device in `status.get` (taking it to 16), which a gateway telling a warm hit for a request with no mode depends on: a worker on 15 is `incompatible`. ADR 0017 §4 and §8 make `warmPool.reserveRunning` a required field of `config.get`'s output (taking it to 17): a client parsing an older daemon's `config.get` would fail its schema without it, so a daemon on 16 is `incompatible`. While a daemon is `starting`, its `status.get` answers `daemon` and `host` alone, so `devices`, `leases`, `capacity` and `queueDepth` become optional (taking it to 18): a peer on 17 cannot parse that answer, and a gateway reading it from a worker on 17 would fail its schema, so a worker on 17 is `incompatible`. `warmPool.targets` and `warmPool.maxConcurrentBoots` are required fields of `config.get`'s output too (taking it to 19), for the same reason: a daemon on 18 is `incompatible`. `lease.request` takes `leaseId` and every lease record carries `idChosenByRequester` (ADR 0020, taking it to 21): a gateway never forwards a `leaseId` to a worker that would reject it, and reads the flag from every worker's lease list, so a worker on 20 is `incompatible`. It only ever widens once a second version is actually kept alive side by side with the * first, which nothing here does. * * A client from before any of those changes simply does not overlap this daemon, and `hello` @@ -42,9 +42,9 @@ const protocolRangeSchema = z.object({ min: z.number().int(), max: z.number().in * The consequence ADR 0005 §31 names: a pre-0005 worker's uplink negotiates nothing, so its * gateway marks it `incompatible` -- with both ranges on the view -- and never dispatches to it. * A worker on 5 is marked `incompatible` the same way (ADR 0008 §10), and so is one on 7 - * (ADR 0007 §12), and so is one on 8 (ADR 0010 §9), one on 10 (ADR 0009 §7), one on 11 (ADR 0015 §3), one on 12 (ADR 0015 §4), one on 13 (ADR 0015 §1), one on 14 (ADR 0015 §2), one on 15 (ADR 0009 §6), and one on 16 (`warmPool.reserveRunning`), one on 17 (a starting `status.get`), one on 18 (`warmPool.targets`), and one on 19 (`status.get`'s `warmPool`). + * (ADR 0007 §12), and so is one on 8 (ADR 0010 §9), one on 10 (ADR 0009 §7), one on 11 (ADR 0015 §3), one on 12 (ADR 0015 §4), one on 13 (ADR 0015 §1), one on 14 (ADR 0015 §2), one on 15 (ADR 0009 §6), and one on 16 (`warmPool.reserveRunning`), one on 17 (a starting `status.get`), one on 18 (`warmPool.targets`), and one on 19 (`status.get`'s `warmPool`), and one on 20 (`leaseId`). */ -export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 20, max: 20 }; +export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 21, max: 21 }; /** * The one protocol version that ever existed before ranges did. Used only to build the diff --git a/src/contract/schemas.test.ts b/src/contract/schemas.test.ts index d5b912d7..5e035253 100644 --- a/src/contract/schemas.test.ts +++ b/src/contract/schemas.test.ts @@ -78,6 +78,7 @@ describe("leaseGrantSchema's device projection", () => { ownerId: "req", grantedAt: 0, lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 1000, ttlDeadline: 1000, }, diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 430a4baf..182892b9 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -235,6 +235,11 @@ export const leaseRecordSchema = z.object({ ttlMs: z.number(), ttlDeadline: z.number(), lastRenewedAt: z.number(), + /** + * ADR 0020: whether the requester chose `id` (`lease.request`'s `leaseId`) rather than simlock + * generating it. A gateway rebuilding its routing table reads it to name such a lease bare. + */ + idChosenByRequester: z.boolean(), /** * ADR 0005 §18: "the lease object gains `worker: { id, label }` (additive) so a client and * the console can tell where the device lives. A worker's network address is never exposed." diff --git a/src/core/cleanup/idle-destroy.test.ts b/src/core/cleanup/idle-destroy.test.ts index b562ad7c..0f6d1d90 100644 --- a/src/core/cleanup/idle-destroy.test.ts +++ b/src/core/cleanup/idle-destroy.test.ts @@ -152,6 +152,7 @@ describe("idleDestroyRule", () => { requesterId: "agent-1", ownerId: "agent-1", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 1, }, diff --git a/src/core/domain.ts b/src/core/domain.ts index 53f96744..ca69996f 100644 --- a/src/core/domain.ts +++ b/src/core/domain.ts @@ -281,6 +281,12 @@ export interface LeaseRecord { * shared one backstop width. A record written before ADR 0004 loads with `grantedAt` here. */ readonly lastRenewedAt: number; + /** + * Whether the requester chose `id` (`lease.request`'s `leaseId`, ADR 0020) rather than simlock + * generating it. Written at grant. A record written before this field existed loads with + * `false`: every ID then was generated. + */ + readonly idChosenByRequester: boolean; } export interface LeaseTiming { @@ -332,6 +338,12 @@ export interface LeaseRequestRecord { readonly requesterId: string; readonly ownerId: string; readonly idempotencyKey?: string; + /** + * The lease ID the requester chose (ADR 0020), kept beside the request, not in it, so the + * request stays the device it names. While the record is `open` it holds that ID: nothing else + * is granted it. + */ + readonly leaseId?: string; readonly request: DeviceRequest; readonly createdAt: number; readonly state: LeaseRequestState; diff --git a/src/core/lease-request-store.ts b/src/core/lease-request-store.ts index 9e455b67..a1fe1791 100644 --- a/src/core/lease-request-store.ts +++ b/src/core/lease-request-store.ts @@ -19,6 +19,7 @@ export interface NewLeaseRequest { readonly requesterId: string; readonly ownerId: string; readonly idempotencyKey?: string; + readonly leaseId?: string; readonly request: DeviceRequest; } @@ -101,6 +102,7 @@ export function newLeaseRequestRecord( createdAt: now, id, ...(input.idempotencyKey === undefined ? {} : { idempotencyKey: input.idempotencyKey }), + ...(input.leaseId === undefined ? {} : { leaseId: input.leaseId }), ownerId: input.ownerId, request: { ...input.request }, requesterId: input.requesterId, diff --git a/src/core/nuke-service.test.ts b/src/core/nuke-service.test.ts index 4a08d80a..21a34edf 100644 --- a/src/core/nuke-service.test.ts +++ b/src/core/nuke-service.test.ts @@ -143,6 +143,7 @@ describe("NukeService", () => { requesterId: "agent", ownerId: "agent", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 1, }, diff --git a/src/core/reclaim-coordinator.test.ts b/src/core/reclaim-coordinator.test.ts index 371f3077..fe264db2 100644 --- a/src/core/reclaim-coordinator.test.ts +++ b/src/core/reclaim-coordinator.test.ts @@ -137,6 +137,7 @@ function released(device: DeviceRecord): ReleasedLease { requesterId: "agent", ownerId: "agent", lastRenewedAt: 1, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 100, }, diff --git a/src/core/registry.test.ts b/src/core/registry.test.ts index b2caa976..a7640361 100644 --- a/src/core/registry.test.ts +++ b/src/core/registry.test.ts @@ -1563,6 +1563,7 @@ describe("Registry", () => { ownerId: "agent-1", grantedAt: 1_000, lastRenewedAt: 1_000, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 2_000, }, @@ -1727,6 +1728,7 @@ describe("Registry", () => { ttlDeadline: 2_000, // A lease that has never been renewed reports the moment it was granted. lastRenewedAt: 1_000, + idChosenByRequester: false, }, ]); @@ -1741,6 +1743,7 @@ describe("Registry", () => { grantedAt: 1_000, id: "lse_1", lastRenewedAt: 5_000, + idChosenByRequester: false, ownerId: "agent-1", requesterId: "agent-1", ttlDeadline: 9_000, diff --git a/src/core/registry.ts b/src/core/registry.ts index 77233356..517b81b2 100644 --- a/src/core/registry.ts +++ b/src/core/registry.ts @@ -102,6 +102,11 @@ export interface CreateLeaseInput { /** The width this lease is granted with; stored on the record, see `LeaseRecord.ttlMs`. */ readonly ttlMs: number; readonly ttlDeadline: number; + /** + * The ID the requester chose for this lease (ADR 0020). Omitted, the registry generates one. + * The caller has already refused an ID that is in use: one place enforces that rule. + */ + readonly leaseId?: string; /** * The request this lease is granted for, and the rest of the grant its repeat answers. When * given, the commit that adds the lease also marks that request `granted` with the whole @@ -586,6 +591,7 @@ export class Registry implements LeaseRequestStore { requesterId, ttlMs, ttlDeadline, + leaseId, request, }: CreateLeaseInput): Promise { const index = this.#devices.findIndex((device) => device.id === deviceId); @@ -606,7 +612,8 @@ export class Registry implements LeaseRequestStore { const lease: LeaseRecord = { deviceId, grantedAt, - id: `lse_${this.options.idGenerator.generate()}`, + id: leaseId ?? `lse_${this.options.idGenerator.generate()}`, + idChosenByRequester: leaseId !== undefined, // ADR 0004: set at grant, then again on every renew -- a lease that has never been // renewed reports the moment it was granted rather than nothing at all. lastRenewedAt: grantedAt, @@ -920,12 +927,14 @@ const leaseRecordKeys = [ "ttlMs", "ttlDeadline", "lastRenewedAt", + "idChosenByRequester", ] as const; const leaseRequestRecordKeys = [ "id", "requesterId", "ownerId", "idempotencyKey", + "leaseId", "request", "createdAt", "state", @@ -1198,6 +1207,7 @@ function parseLease(value: unknown, defaultTtlMs: number): LeaseRecord { deviceId, grantedAt, id, + idChosenByRequester: value.idChosenByRequester === true, lastRenewedAt: finiteTimestampOr(lastRenewedAt, grantedAt), ownerId: ownerId ?? requesterId, requesterId, @@ -1212,13 +1222,14 @@ function parseLease(value: unknown, defaultTtlMs: number): LeaseRecord { */ function parseLeaseRequest(value: unknown): LeaseRequestRecord | undefined { if (!hasLeaseRequestFields(value)) return undefined; - const { createdAt, id, idempotencyKey, ownerId, request, requesterId, state } = value; + const { createdAt, id, idempotencyKey, leaseId, ownerId, request, requesterId, state } = value; const result = parseLeaseRequestResult(state, value); if (result === undefined) return undefined; return { createdAt, id, ...(idempotencyKey === undefined ? {} : { idempotencyKey }), + ...(leaseId === undefined ? {} : { leaseId }), ownerId, request: withoutRetiredRequestKeys(request), requesterId, @@ -1232,6 +1243,7 @@ function hasLeaseRequestFields(value: unknown): value is Record readonly createdAt: number; readonly id: string; readonly idempotencyKey?: string; + readonly leaseId?: string; readonly ownerId: string; readonly request: DeviceRequest; readonly requesterId: string; @@ -1243,6 +1255,7 @@ function hasLeaseRequestFields(value: unknown): value is Record typeof value.requesterId === "string" && typeof value.ownerId === "string" && (value.idempotencyKey === undefined || typeof value.idempotencyKey === "string") && + (value.leaseId === undefined || typeof value.leaseId === "string") && isDeviceRequest(value.request) && typeof value.createdAt === "number" && isLeaseRequestState(value.state) @@ -1271,7 +1284,8 @@ function parseLeaseRequestResult( /** * A stored grant's device follows the device record's own load rule (ADR 0007 §11): no `mode` * loads as `full`, the retired keys (and the spec's `full`) are dropped, and an unknown `mode` makes the record - * unusable, so it is skipped like any other inconsistent lease request. + * unusable, so it is skipped like any other inconsistent lease request. Its lease loads + * `idChosenByRequester` as a lease record does (`parseLease`). */ function parseStoredGrant(grant: unknown): LeaseGrant | undefined { if (!isObject(grant) || !isObject(grant.device)) return undefined; @@ -1283,7 +1297,11 @@ function parseStoredGrant(grant: unknown): LeaseGrant | undefined { } if (device.mode === undefined) device.mode = "full"; if (device.mode !== "slim" && device.mode !== "full") return undefined; - return { ...grant, device } as unknown as LeaseGrant; + // A grant written before ADR 0020 names a lease whose ID simlock generated, as every ID then was. + const lease = isObject(grant.lease) + ? { ...grant.lease, idChosenByRequester: grant.lease.idChosenByRequester === true } + : grant.lease; + return { ...grant, device, lease } as unknown as LeaseGrant; } function isDeviceRequest(value: unknown): value is DeviceRequest { diff --git a/src/core/startup-converger.test.ts b/src/core/startup-converger.test.ts index e009b933..e846d418 100644 --- a/src/core/startup-converger.test.ts +++ b/src/core/startup-converger.test.ts @@ -134,6 +134,7 @@ describe("StartupConverger", () => { requesterId: "a", ownerId: "a", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 10, }, @@ -144,6 +145,7 @@ describe("StartupConverger", () => { requesterId: "b", ownerId: "b", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 10, }, @@ -217,6 +219,7 @@ describe("StartupConverger", () => { requesterId: "a", ownerId: "a", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 1000, }, diff --git a/src/core/warm-pool/converger.test.ts b/src/core/warm-pool/converger.test.ts index 126b78eb..179bc09f 100644 --- a/src/core/warm-pool/converger.test.ts +++ b/src/core/warm-pool/converger.test.ts @@ -46,6 +46,7 @@ function leaseOn(deviceId: string): LeaseRecord { grantedAt: 1, id: `lease-${deviceId}`, lastRenewedAt: 1, + idChosenByRequester: false, ownerId: "holder", requesterId: "holder", ttlDeadline: now + minute, diff --git a/src/core/warm-pool/policy.test.ts b/src/core/warm-pool/policy.test.ts index 7c1e243c..3fe69cf2 100644 --- a/src/core/warm-pool/policy.test.ts +++ b/src/core/warm-pool/policy.test.ts @@ -61,6 +61,7 @@ function leaseOn(target: DeviceRecord): LeaseRecord { grantedAt: 1, id: `lease-${target.id}`, lastRenewedAt: 1, + idChosenByRequester: false, ownerId: "holder", requesterId: "holder", ttlDeadline: now + minute, diff --git a/src/daemon-protocol/index.test.ts b/src/daemon-protocol/index.test.ts index 11141549..61b20678 100644 --- a/src/daemon-protocol/index.test.ts +++ b/src/daemon-protocol/index.test.ts @@ -18,8 +18,8 @@ describe("daemon protocol", () => { // lets a lease request choose it, taking it to 8, each the same way. ADR 0010 adds // `component.install` and its `component-progress` push, which a gateway relays to its // workers, taking it to 9. ADR 0014 gives every event envelope an `id`, taking it to 10. ADR 0009 §7 makes `atRamBudget` a - // required field of each platform's `status.get` capacity, taking it to 11, then ADR 0015 §3 made `modelClasses` a required field of each platform catalog, taking it to 12, then ADR 0015 §4 made `classDefaults` a required field of each platform catalog too, taking it to 13, then ADR 0015 §1 made a lease request's `model` optional beside a new `class`, taking it to 14, then ADR 0015 §2 let `osVersion` be a range, taking it to 15, then ADR 0009 §6 made `servesDefaultMode` a required field of each device in `status.get`, taking it to 16, then `warmPool.reserveRunning` became a required field of `config.get`, taking it to 17, then a starting `status.get` made `devices`, `leases`, `capacity` and `queueDepth` optional, taking it to 18, then `warmPool.targets` and `warmPool.maxConcurrentBoots` became required fields of `config.get`, taking it to 19, then `status.get` gained `warmPool`, taking it to 20. - expect(DAEMON_PROTOCOL_VERSION).toBe(20); + // required field of each platform's `status.get` capacity, taking it to 11, then ADR 0015 §3 made `modelClasses` a required field of each platform catalog, taking it to 12, then ADR 0015 §4 made `classDefaults` a required field of each platform catalog too, taking it to 13, then ADR 0015 §1 made a lease request's `model` optional beside a new `class`, taking it to 14, then ADR 0015 §2 let `osVersion` be a range, taking it to 15, then ADR 0009 §6 made `servesDefaultMode` a required field of each device in `status.get`, taking it to 16, then `warmPool.reserveRunning` became a required field of `config.get`, taking it to 17, then a starting `status.get` made `devices`, `leases`, `capacity` and `queueDepth` optional, taking it to 18, then `warmPool.targets` and `warmPool.maxConcurrentBoots` became required fields of `config.get`, taking it to 19, then `status.get` gained `warmPool`, taking it to 20, then `lease.request` gained `leaseId` and every lease record `idChosenByRequester`, taking it to 21. + expect(DAEMON_PROTOCOL_VERSION).toBe(21); expect(serializeFrame({ id: 1, type: "hello" })).toBe('{"id":1,"type":"hello"}\n'); }); diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index 20ea2e8e..c59c0304 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -26,7 +26,12 @@ import { type CatalogReader, type PassthroughResolver, } from "../core/index.js"; -import { type DeviceModeReader, type LeaseCommands, type QueueControl } from "../leasing/index.js"; +import { + type DeviceModeReader, + LeaseIdTakenError, + type LeaseCommands, + type QueueControl, +} from "../leasing/index.js"; import type { Clock, Logger, @@ -396,10 +401,15 @@ export class Dispatcher { ? {} : { onAdmitted: session.onRequestAdmitted }), ...(input.idempotencyKey === undefined ? {} : { idempotencyKey: input.idempotencyKey }), + ...(input.leaseId === undefined ? {} : { leaseId: input.leaseId }), ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }), ...(input.ttlMs === undefined ? {} : { ttlMs: input.ttlMs }), }); } catch (error: unknown) { + // The refusal carries the ID in `details` on every transport (`ErrorDetailsMap`). + if (error instanceof LeaseIdTakenError) { + throw new DispatchError("LEASE_ID_TAKEN", error.message, { leaseId: error.leaseId }); + } // The lease path only ever sees the clamped-to-false permission, so it cannot itself // tell the caller that config, not missing consent, is what stood between this request // and success. Recover that distinction here, the one place that saw both sides. Moved diff --git a/src/daemon/error-code.ts b/src/daemon/error-code.ts index 9b5bed95..7b53c762 100644 --- a/src/daemon/error-code.ts +++ b/src/daemon/error-code.ts @@ -37,6 +37,7 @@ import { } from "../core/index.js"; import { IdempotencyConflictError, + LeaseIdTakenError, LeaseRequestForbiddenError, NoCapacityError, QueueTimeoutError, @@ -94,6 +95,9 @@ export function classifyError(error: unknown): SimlockErrorCode | undefined { if (error instanceof IdempotencyConflictError) { return "IDEMPOTENCY_CONFLICT"; } + if (error instanceof LeaseIdTakenError) { + return "LEASE_ID_TAKEN"; + } if (error instanceof LeaseRequestForbiddenError) { return "FORBIDDEN"; } diff --git a/src/daemon/server.test.ts b/src/daemon/server.test.ts index c2186c72..979c901f 100644 --- a/src/daemon/server.test.ts +++ b/src/daemon/server.test.ts @@ -182,8 +182,9 @@ describe("DaemonServer", () => { it.each([ [17, "cannot parse a starting status.get answer"], [18, "cannot parse config.get's warmPool.targets"], + [20, "cannot send lease.request's leaseId or read a lease's idChosenByRequester"], ])( - "answers PROTOCOL_VERSION_UNSUPPORTED, naming protocol 20, to a client on protocol %i, which %s", + "answers PROTOCOL_VERSION_UNSUPPORTED, naming protocol 21, to a client on protocol %i, which %s", async (version) => { const harness = await createHarness(); const previous = await createClient(harness.socketPath); @@ -193,7 +194,7 @@ describe("DaemonServer", () => { ).resolves.toMatchObject({ error: { code: "PROTOCOL_VERSION_UNSUPPORTED", - details: { client: { min: version, max: version }, daemon: { min: 20, max: 20 } }, + details: { client: { min: version, max: version }, daemon: { min: 21, max: 21 } }, }, ok: false, }); diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index c8152d85..3f7dadac 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -456,6 +456,8 @@ export class GatewayDispatcher { ? {} : { onAdmitted: session.onRequestAdmitted }), ...(input.idempotencyKey === undefined ? {} : { idempotencyKey: input.idempotencyKey }), + // ADR 0020: forwarded to the worker as sent, and the gateway's own lease ID when it grants. + ...(input.leaseId === undefined ? {} : { leaseId: input.leaseId }), ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }), // Always explicit past this point (§15): a request that named none is filled in here, // before dispatch, rather than left for whichever worker happens to grant it to default diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index f81f3bb0..bfef956a 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -3142,6 +3142,19 @@ describe("FleetLeaseCoordinator: lease.rejected names its request", () => { await ask(fleet, { requesterId: "agent-1" }); return { fleet, requester: "agent-1" }; }, + "lease-id-taken": async () => { + const fleet = harness(); + const client = new ScriptedWorkerClient(); + fleet.directory.add("wrk_a", client); + connectWorker(fleet.workers, "wrk_a"); + client.requestLeaseQueue.push({ + grant: grantFixture({ lease: { ...grantFixture().lease, id: "myid" } }), + kind: "grant", + }); + await fleet.coordinator.request(REQUEST, requestOptions({ leaseId: "myid" })); + await ask(fleet, { leaseId: "myid", ownerId: "agent-2", requesterId: "agent-2" }); + return { fleet, requester: "agent-2" }; + }, cancelled: async () => { const fleet = await busyFleet(); const requestId = await ask(fleet, { ownerId: "agent-2", requesterId: "agent-2" }); @@ -3440,6 +3453,7 @@ describe("FleetLeaseCoordinator: a worker that is still starting", () => { grantedAt: 1, id: "lse_1", lastRenewedAt: 1, + idChosenByRequester: false, ownerId: "agent-9", requesterId: `${GATEWAY_PREFIX}agent-9`, ttlDeadline: 900_001, diff --git a/src/gateway/fleet-coordinator.ts b/src/gateway/fleet-coordinator.ts index 98ac46f1..b709c329 100644 --- a/src/gateway/fleet-coordinator.ts +++ b/src/gateway/fleet-coordinator.ts @@ -248,6 +248,7 @@ export class FleetLeaseCoordinator { if (replay !== undefined) return { replay }; const requestId = newLeaseRequestId(this.options.idGenerator); this.#refuseIfAlreadyLeased(deviceRequest, options.requesterId, requestId); + this.#refuseIfLeaseIdTaken(deviceRequest, options, requestId); const { id, started: created } = await this.requests.admit( deviceRequest, options, @@ -286,6 +287,34 @@ export class FleetLeaseCoordinator { } } + /** + * ADR 0020: the one clash check on a gateway, after the one-lease check and in the same + * admission section. A caller-chosen id is held by one of this gateway's leases and by one of + * its own requests still open; leases a local client holds on a worker are not checked (a + * worker refuses its own, and the requester's uniqueness guarantee makes a clash unlikely). + */ + #refuseIfLeaseIdTaken( + deviceRequest: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): void { + const { leaseId } = options; + if (leaseId === undefined) return; + if ( + this.options.leaseIndex.resolve(leaseId) === undefined && + !this.requests.holdsLeaseId(leaseId) + ) { + return; + } + this.#emit("lease.rejected", { + requestId, + requester: options.requesterId, + requestSpec: deviceRequest, + reason: "lease-id-taken", + }); + throw leaseIdTaken(leaseId); + } + async cancelPending(requesterId: string): Promise<"cancelled" | "not-found" | "not-cancellable"> { return this.#decisions.run(async () => { const waiter = this.#queue.findPendingWaiter(requesterId); @@ -925,6 +954,9 @@ export class FleetLeaseCoordinator { // own* queue is where a "wait" request actually waits. noWait: true, ...(waiter.options.ttlMs === undefined ? {} : { ttlMs: waiter.options.ttlMs }), + // ADR 0020: sent as the requester sent it. The worker grants exactly this id, and + // its refusal (`LEASE_ID_TAKEN`) is the caller's answer: no other worker is tried. + ...(waiter.options.leaseId === undefined ? {} : { leaseId: waiter.options.leaseId }), }, { onProgress: (progress) => { @@ -985,7 +1017,14 @@ export class FleetLeaseCoordinator { } announceDispatched(); - this.#settleGrant(waiter, workerId, grant); + if (this.#settleGrant(waiter, workerId, grant) === "retry") { + // The worker answered with an id the gateway did not ask for: the attempt failed, and the + // request goes back to the queue as after an unreachable worker. This worker is left out + // until its view changes, so a worker that keeps answering so is not asked in a loop. + this.#rememberRefusal(waiter, workerId); + this.#staleView(waiter, workerId); + return; + } // C1 (round 3 review): see the catch branch above -- a grant settling this waiter is just as // much a reason for whoever else is queued to get another look, not only a terminal failure. this.#dispatch(); @@ -1070,8 +1109,23 @@ export class FleetLeaseCoordinator { * worker, but this path already knows the answer. A mismatched echo is logged -- it means * either a worker bug or something worth knowing about, never silently swallowed. */ - #settleGrant(waiter: FleetWaiter, workerId: string, grant: LeaseGrant): void { - const gatewayLeaseId = `${workerId}.${grant.lease.id}`; + #settleGrant(waiter: FleetWaiter, workerId: string, grant: LeaseGrant): "settled" | "retry" { + // ADR 0020: a forwarded `leaseId` is the gateway lease id, bare, and never the worker's echo + // of it -- a worker that granted something else is not believed, and its lease is given back. + const forwardedLeaseId = waiter.options.leaseId; + if (forwardedLeaseId !== undefined && grant.lease.id !== forwardedLeaseId) { + this.#logger.warn( + "Worker granted a lease ID different from the leaseId the gateway forwarded", + { + forwardedLeaseId, + grantedLeaseId: grant.lease.id, + workerId, + }, + ); + this.#releaseOnWorker(workerId, grant.lease.id); + return "retry"; + } + const gatewayLeaseId = forwardedLeaseId ?? `${workerId}.${grant.lease.id}`; if (grant.lease.ownerId !== waiter.options.ownerId) { this.#logger.warn("Worker echoed an ownerId different from the one the gateway forwarded", { echoedOwnerId: grant.lease.ownerId, @@ -1088,7 +1142,13 @@ export class FleetLeaseCoordinator { workerId, workerLeaseId: grant.lease.id, }; - this.options.leaseIndex.add(entry); + if (!this.options.leaseIndex.add(entry)) { + // The index already routes this bare id to another worker, which got it there first: the + // new worker's lease is given back and the caller is told the id is taken. + this.#releaseOnWorker(workerId, grant.lease.id); + this.#reject(waiter, leaseIdTaken(gatewayLeaseId), "lease-id-taken"); + return "settled"; + } const fleetGrant: FleetLeaseGrant = { device: grant.device, environment: grant.environment, @@ -1096,6 +1156,21 @@ export class FleetLeaseCoordinator { timing: grant.timing, }; this.#queue.resolve(waiter, fleetGrant); + return "settled"; + } + + /** Gives a lease back to the worker that granted it, when the gateway will not route it. A + * failure is logged: the worker's own TTL ends the lease. */ + #releaseOnWorker(workerId: string, workerLeaseId: string): void { + void this.#forwardToWorker(workerId, (client) => + client.releaseLease({ leaseId: workerLeaseId }), + ).catch((error: unknown) => { + this.#logger.warn("Failed to release a lease the gateway will not route", { + message: error instanceof Error ? error.message : String(error), + workerId, + workerLeaseId, + }); + }); } #enqueue(waiter: FleetWaiter): void { @@ -1108,7 +1183,7 @@ export class FleetLeaseCoordinator { #reject( waiter: FleetWaiter, error: Error, - reason: Rejection["reason"] | "no-wait" | "cancelled" | "timeout", + reason: Rejection["reason"] | "no-wait" | "cancelled" | "timeout" | "lease-id-taken", ): void { if (this.#queue.reject(waiter, error)) { this.#emitRejected(waiter, reason); @@ -1268,6 +1343,12 @@ function forwardedModel( return { model: (view === undefined ? undefined : matchRequest(view, request)) ?? request.model }; } +/** The refusal for a lease ID this gateway already holds (ADR 0020), with the code and details a + * worker gives the same refusal. */ +function leaseIdTaken(leaseId: string): DispatchError { + return new DispatchError("LEASE_ID_TAKEN", `lease ID ${leaseId} is already in use`, { leaseId }); +} + /** The `model` a forward overrides the request's own with: none for a class request. */ interface ModelName { readonly model?: string; diff --git a/src/gateway/lease-index.ts b/src/gateway/lease-index.ts index e75f9dcb..be7f13f7 100644 --- a/src/gateway/lease-index.ts +++ b/src/gateway/lease-index.ts @@ -1,3 +1,4 @@ +import { LEASE_ID_PATTERN } from "../contract/index.js"; import type { Logger } from "../ports/index.js"; import { NoopLogger } from "../ports/index.js"; @@ -7,8 +8,9 @@ import { NoopLogger } from "../ports/index.js"; * worker reports (`rebuildFromWorker`) whenever the gateway can see it, which is what makes a * gateway restart lose nothing a worker restart would not also lose (Decision 5). * - * A gateway lease id is minted once, at grant, as `${workerId}.${workerLeaseId}` (§16) and never - * re-derived by splitting the string back apart -- every lookup in this class goes through the + * A gateway lease id is minted once, at grant, as `${workerId}.${workerLeaseId}` (§16) -- or, for + * a lease whose requester chose its id (ADR 0020), as that id bare, routed by this index alone -- + * and never re-derived by splitting the string back apart -- every lookup in this class goes through the * map this index keeps, keyed by the id it minted or by the `(workerId, workerLeaseId)` pair a * relayed worker event carries. The "split on the first `.`" the ADR describes is what makes the * id *routable in principle* (a worker id is a UUID, so it can never itself contain the @@ -35,6 +37,8 @@ export interface WorkerReportedLease { readonly requesterId: string; readonly ownerId: string; readonly grantedAt: number; + /** ADR 0020: whether the requester chose `id`. Absent, the id is a generated one. */ + readonly idChosenByRequester?: boolean; } /** A lease record projected for a fleet client: rewritten to the gateway's id and fleet-level @@ -132,12 +136,19 @@ export class FleetLeaseIndex { return this.#byRequester.get(requesterId); } - /** Records a lease this gateway just granted. Idempotent by `gatewayLeaseId`: granting twice - * under the same id (it never happens outside a test) replaces rather than duplicates. */ - add(entry: FleetLeaseEntry): void { + /** + * Records a lease this gateway just granted. Idempotent by `gatewayLeaseId` for one worker: + * granting twice under the same id (it never happens outside a test) replaces rather than + * duplicates. A bare id (ADR 0020) belongs to the worker that first held it: another worker's + * entry under it is not added, and `false` says so. + */ + add(entry: FleetLeaseEntry): boolean { + const existing = this.#byGatewayId.get(entry.gatewayLeaseId); + if (existing !== undefined && existing.workerId !== entry.workerId) return false; this.#byGatewayId.set(entry.gatewayLeaseId, entry); this.#byRequester.set(entry.requesterId, entry.gatewayLeaseId); this.#byWorkerLease.set(workerKey(entry.workerId, entry.workerLeaseId), entry.gatewayLeaseId); + return true; } /** Forgets one lease by its gateway id -- the gateway's own `lease.release` completing. */ @@ -216,10 +227,21 @@ export class FleetLeaseIndex { const reported = new Set(); for (const lease of leases) { if (!this.isGatewayRequester(lease.requesterId)) continue; - const gatewayLeaseId = `${workerId}.${lease.id}`; + const gatewayLeaseId = this.#gatewayLeaseId(workerId, lease); + const existing = this.#byGatewayId.get(gatewayLeaseId); + if (existing !== undefined && existing.workerId !== workerId) { + // ADR 0020: two workers hold the same caller-chosen id. The first reported keeps it; this + // lease is not routed and expires at its TTL. Not counted as a report from this worker's + // entry, because it is not one, so it never resets the first entry's missing count. + this.logger.warn( + "Two workers report a lease with the same caller-chosen id; routing the first and ignoring the second", + { gatewayLeaseId, firstWorkerId: existing.workerId, secondWorkerId: workerId }, + ); + continue; + } reported.add(gatewayLeaseId); this.#missingSince.delete(gatewayLeaseId); - if (this.#byGatewayId.has(gatewayLeaseId)) continue; + if (existing !== undefined) continue; // C1 (round 2 review): this id was forgotten moments ago by the worker's own relayed // `lease.expired`/`lease.released` -- reappearing in a snapshot now means that snapshot // was gathered before the relayed fact landed and is only completing late (see @@ -244,6 +266,24 @@ export class FleetLeaseIndex { return reported; } + /** + * The id a reported lease is routed under (ADR 0020): bare when the requester chose it and it + * is a valid chosen id, `.` otherwise. A lease flagged as chosen whose id could + * not have been chosen is prefixed and logged: the worker's claim is not trusted past the + * pattern a requester's id must match (safety rule 10). + */ + #gatewayLeaseId(workerId: string, lease: WorkerReportedLease): string { + if (lease.idChosenByRequester !== true) return `${workerId}.${lease.id}`; + if (LEASE_ID_PATTERN.test(lease.id)) return lease.id; + if (!this.#byWorkerLease.has(workerKey(workerId, lease.id))) { + this.logger.warn( + "A worker reported a lease flagged as caller-chosen whose id is not a valid one; routing it under a worker-prefixed id", + { leaseId: lease.id, workerId }, + ); + } + return `${workerId}.${lease.id}`; + } + /** `rebuildFromWorker`'s removal half: forgets an existing entry for `workerId` that this * snapshot did not report, but only once it has failed to appear in *two* consecutive * snapshots (see `#missingSince`'s own doc comment for why one miss is not enough). */ diff --git a/src/gateway/test-support.ts b/src/gateway/test-support.ts index f5f053d6..a2f45a26 100644 --- a/src/gateway/test-support.ts +++ b/src/gateway/test-support.ts @@ -88,6 +88,7 @@ export function leaseFixture(id: string, deviceId: string) { grantedAt: 1, id, lastRenewedAt: 1, + idChosenByRequester: false, ownerId: "agent-1", requesterId: "agent-1", ttlDeadline: 900_001, diff --git a/src/http/app.ts b/src/http/app.ts index 8cf2d97f..77c6a022 100644 --- a/src/http/app.ts +++ b/src/http/app.ts @@ -8,6 +8,7 @@ import { describeSchemaIssues, refuseModelWithClass, deviceClassSchema, + leaseIdSchema, imageTagSchema, } from "../contract/index.js"; import { parseDurationMs } from "../contract/duration.js"; @@ -96,6 +97,8 @@ const leaseRequestBodySchema = z class: deviceClassSchema.optional(), device: z.string().min(1).optional(), imageTag: imageTagSchema.optional(), + // ADR 0020: the lease ID the requester chooses. Shape and bounds are the contract's own. + leaseId: leaseIdSchema.optional(), mode: z.enum(["slim", "full"]).optional(), noWait: z.boolean().optional(), // ADR §27a (H7, round 2 review): declared and forwarded, not silently dropped -- the shared @@ -181,6 +184,7 @@ function leaseRequestOptionFields(body: z.infer) ...(body.mode === undefined ? {} : { mode: body.mode }), ...(body.imageTag === undefined ? {} : { imageTag: body.imageTag }), ...(body.owner === undefined ? {} : { owner: body.owner }), + ...(body.leaseId === undefined ? {} : { leaseId: body.leaseId }), }; } diff --git a/src/http/test-fakes.ts b/src/http/test-fakes.ts index 8810f801..7ca66b06 100644 --- a/src/http/test-fakes.ts +++ b/src/http/test-fakes.ts @@ -129,6 +129,7 @@ export function makeLease(overrides: Partial = {}): LeaseRecord { ownerId: "tok_agent", requesterId: "tok_agent", lastRenewedAt: 1_000, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 1_000 + 900_000, ...overrides, diff --git a/src/http/tracker.ts b/src/http/tracker.ts index df64302b..4f251a4d 100644 --- a/src/http/tracker.ts +++ b/src/http/tracker.ts @@ -26,6 +26,8 @@ export interface LeaseRequestInput { * -- an identity named and answered as though it had not been is the kind of silence that * reads like authorization). */ readonly owner?: string; + /** ADR 0020: the lease ID the requester chooses, sent on as `lease.request`'s `leaseId`. */ + readonly leaseId?: string; } /** Matches the issue's lease object exactly; `dataPlane` is reserved and always `null` in v1. */ @@ -120,6 +122,7 @@ function requestOptions(body: LeaseRequestInput) { // ADR §27a (H7, round 2 review): forwarded as-is -- the shared dispatcher's own // `lease.request` handler is what rejects a non-admin token naming this. ...(body.owner === undefined ? {} : { owner: body.owner }), + ...(body.leaseId === undefined ? {} : { leaseId: body.leaseId }), }; } diff --git a/src/leasing/acquisition-planner.test.ts b/src/leasing/acquisition-planner.test.ts index af1a82fe..3f37d45a 100644 --- a/src/leasing/acquisition-planner.test.ts +++ b/src/leasing/acquisition-planner.test.ts @@ -134,6 +134,7 @@ function leaseOn(target: DeviceRecord): LeaseRecord { grantedAt: 1, id: `lease-${target.id}`, lastRenewedAt: 1, + idChosenByRequester: false, ownerId: "holder", requesterId: "holder", ttlDeadline: 100, @@ -209,6 +210,7 @@ describe("AcquisitionPlanner", () => { requesterId: "holder", ownerId: "holder", lastRenewedAt: 1, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 100, }, diff --git a/src/leasing/create-leasing.test.ts b/src/leasing/create-leasing.test.ts index 762c7901..4f52bd22 100644 --- a/src/leasing/create-leasing.test.ts +++ b/src/leasing/create-leasing.test.ts @@ -2641,6 +2641,24 @@ describe("createLeasing: lease.rejected names its request", () => { await settledOrPending(harness.engine.request(request, holder)); return { harness, requester: "holder" }; }, + "lease-id-taken": async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 2, maxRunning: 2 }, + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 4, + }, + }); + await harness.engine.request(request, { ...holder, leaseId: "myid" }); + await settledOrPending( + harness.engine.request(request, { + leaseId: "myid", + ownerId: "clash", + requesterId: "clash", + }), + ); + return { harness, requester: "clash" }; + }, "boot-timeout": async () => { const driver = new FakeDriver({ availableOsVersions: ["26.5"], diff --git a/src/leasing/index.ts b/src/leasing/index.ts index cceb8f9b..7f871db7 100644 --- a/src/leasing/index.ts +++ b/src/leasing/index.ts @@ -11,6 +11,7 @@ export { type WaitingRequest, } from "./lease-request-book.js"; export { + LeaseIdTakenError, type LeaseRequestOptions, QueueTimeoutError, type QueuePlace, diff --git a/src/leasing/lease-acquisition-coordinator.ts b/src/leasing/lease-acquisition-coordinator.ts index 1e2814bc..3e41da05 100644 --- a/src/leasing/lease-acquisition-coordinator.ts +++ b/src/leasing/lease-acquisition-coordinator.ts @@ -55,6 +55,7 @@ import { type AcquisitionPlan, type AcquisitionPlanner } from "./acquisition-pla import { type LeaseRequestBook } from "./lease-request-book.js"; import { type LeaseLifecycle } from "./lease-lifecycle.js"; import { + LeaseIdTakenError, type LeaseRequestOptions, RequestCancelledError, RequesterAlreadyLeasedError, @@ -142,7 +143,7 @@ export interface LeaseAcquisitionCoordinatorOptions { readonly registry: LeaseAcquisitionRegistry; readonly logger?: Logger; /** Stores each request before it is queued and answers repeats of it (`LeaseRequestBook`). */ - readonly requests: Pick, "admit" | "replay">; + readonly requests: Pick, "admit" | "holdsLeaseId" | "replay">; } interface AcquisitionWaiter extends Waiter { @@ -265,6 +266,7 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { ); throw new RequesterAlreadyLeasedError(options.requesterId, activeLease?.id); } + this.#refuseIfLeaseIdTaken(request, options, requestId); const { id, started: accepted } = await this.options.requests.admit( request, options, @@ -293,6 +295,37 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { return waiter.promise; } + /** + * ADR 0020: the one clash check on a host, inside the same admission section as the + * one-lease-per-requester check and after it. An ID is held by an active lease and by a + * request still waiting, so a restart, which fails every waiting request, frees it. + */ + #refuseIfLeaseIdTaken( + request: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): void { + const { leaseId } = options; + if (leaseId === undefined) return; + if ( + !this.options.registry.snapshot.leases.some((lease) => lease.id === leaseId) && + !this.options.requests.holdsLeaseId(leaseId) + ) { + return; + } + this.options.eventBus.emit( + "lease.rejected", + { + requestId, + requester: options.requesterId, + requestSpec: request, + reason: "lease-id-taken", + }, + "lease-acquisition-coordinator", + ); + throw new LeaseIdTakenError(leaseId); + } + /** Closes acquisition admission, settles all demand, and drains driver work. */ async beginMaintenance(): Promise { await this.options.decisions.run(async () => { @@ -774,6 +807,7 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { source: GRANT_SOURCE[kind], timing: waiter.timing, ...(waiter.options.ttlMs === undefined ? {} : { ttlMs: waiter.options.ttlMs }), + ...(waiter.options.leaseId === undefined ? {} : { leaseId: waiter.options.leaseId }), }); this.options.queue.resolve(waiter, granted); } diff --git a/src/leasing/lease-expiry-scheduler.test.ts b/src/leasing/lease-expiry-scheduler.test.ts index 5928cd49..a80d73e1 100644 --- a/src/leasing/lease-expiry-scheduler.test.ts +++ b/src/leasing/lease-expiry-scheduler.test.ts @@ -8,6 +8,7 @@ const lease = (id: string, ttlDeadline: number) => ({ grantedAt: 0, id, lastRenewedAt: 0, + idChosenByRequester: false, requesterId: "agent", ownerId: "agent", ttlMs: ttlDeadline, diff --git a/src/leasing/lease-lifecycle.ts b/src/leasing/lease-lifecycle.ts index 00eb6a0c..fd020003 100644 --- a/src/leasing/lease-lifecycle.ts +++ b/src/leasing/lease-lifecycle.ts @@ -19,6 +19,7 @@ export interface LeaseLifecycleRegistry { readonly ownerId: string; readonly ttlMs: number; readonly ttlDeadline: number; + readonly leaseId?: string; readonly request?: { readonly id: string; readonly environment: LeaseGrant["environment"]; @@ -75,6 +76,9 @@ export class LeaseLifecycle { * none. Whatever it resolves to is stored on the record, because that is what a later * body-less renew re-applies. */ readonly ttlMs?: number; + /** ADR 0020: the lease ID the requester chose, when it did. The caller has already refused one + * that is in use. */ + readonly leaseId?: string; }): Promise { const { ttlMs, requestId, environment, timing, source, ...createInput } = input; const effectiveTtlMs = ttlMs ?? this.options.ttl.defaultMs; diff --git a/src/leasing/lease-request-book.ts b/src/leasing/lease-request-book.ts index a284a573..62fa81d2 100644 --- a/src/leasing/lease-request-book.ts +++ b/src/leasing/lease-request-book.ts @@ -99,14 +99,14 @@ function waitingRequest( }; } -/** The same `(requesterId, idempotencyKey)` arrived again naming a different device. */ +/** The same `(requesterId, idempotencyKey)` arrived again naming a different device or lease ID. */ export class IdempotencyConflictError extends Error { constructor( readonly requesterId: string, readonly idempotencyKey: string, ) { super( - `Idempotency key ${idempotencyKey} was already used by requester ${requesterId} for a different device request`, + `Idempotency key ${idempotencyKey} was already used by requester ${requesterId} for a different device request or lease ID`, ); this.name = "IdempotencyConflictError"; } @@ -171,8 +171,9 @@ interface OpenRequest { * - a request is stored before anything queues it (`admit`); * - a repeat under the same `(requesterId, idempotencyKey)` returns the stored result, or attaches * to the wait that is still open, and never starts a second one (`replay`); - * - the same key naming a different device is `IdempotencyConflictError`, and a repeat from - * another principal is `LeaseRequestForbiddenError`; + * - the same key naming a different device or a different `leaseId` (one side having none counts + * as different) is `IdempotencyConflictError`, and a repeat from another principal is + * `LeaseRequestForbiddenError`; * - a request's result is written once and never re-evaluated: when its own promise settles, or, * for a daemon's grant, already in the commit that added the lease (`Registry.createLease`), * in which case the settle that follows finds it granted and writes nothing. @@ -224,7 +225,7 @@ export class LeaseRequestBook record.leaseId === leaseId && !isSettled(record)); + } + /** The id of the stored request that was granted `leaseId`, while that record is retained. */ requestIdForLease(leaseId: string): string | undefined { return this.options.store diff --git a/src/leasing/wait-queue.ts b/src/leasing/wait-queue.ts index f2f9fcef..fb106287 100644 --- a/src/leasing/wait-queue.ts +++ b/src/leasing/wait-queue.ts @@ -21,6 +21,12 @@ export interface LeaseRequestOptions { * `requesterId`; both are the caller's own claims, so a replay is authorized on `ownerId`. */ readonly idempotencyKey?: string; + /** + * ADR 0020: the lease ID the requester chose. The granted lease has exactly this ID; the + * requester guarantees it is unique for all time, and while the request waits it holds the ID. + * An option of the request, not part of the device it names. + */ + readonly leaseId?: string; /** Called with the stored request's id once it is admitted (`replayed` false), or once a * repeat finds it (`replayed` true). */ readonly onAdmitted?: (requestId: string, replayed: boolean) => void; @@ -55,6 +61,14 @@ export class RequesterAlreadyLeasedError extends Error { } } +/** A request named a lease ID an active lease or a waiting request already holds (ADR 0020). */ +export class LeaseIdTakenError extends Error { + constructor(readonly leaseId: string) { + super(`lease ID ${leaseId} is already in use`); + this.name = "LeaseIdTakenError"; + } +} + /** Thrown when a caller passes a waiter created by a different queue instance. */ export class ForeignWaiterError extends Error { constructor() { diff --git a/src/mcp/contracts.test.ts b/src/mcp/contracts.test.ts index 61527fff..31c1b925 100644 --- a/src/mcp/contracts.test.ts +++ b/src/mcp/contracts.test.ts @@ -78,6 +78,7 @@ describe("MCP contracts", () => { grantedAt: 0, id: "lease-1", lastRenewedAt: 0, + idChosenByRequester: false, ownerId: "mcp:1", requesterId: "mcp:1", ttlMs: 60_000, diff --git a/src/mcp/server.test.ts b/src/mcp/server.test.ts index e0e0d9f0..8bdde385 100644 --- a/src/mcp/server.test.ts +++ b/src/mcp/server.test.ts @@ -51,6 +51,7 @@ describe("MCP server (smoke)", () => { ownerId: grant.lease.ownerId, requesterId: grant.lease.requesterId, lastRenewedAt: grant.lease.grantedAt, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: grant.lease.ttlDeadline, }, diff --git a/src/mcp/session.test.ts b/src/mcp/session.test.ts index d132fc3e..efff3e79 100644 --- a/src/mcp/session.test.ts +++ b/src/mcp/session.test.ts @@ -105,6 +105,7 @@ describe("McpSession", () => { ownerId: "mcp-test", requesterId: "mcp-test", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 5_000, }, @@ -128,6 +129,7 @@ describe("McpSession", () => { ownerId: "mcp-test", requesterId: "mcp-test", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 5_000, }); @@ -150,6 +152,7 @@ describe("McpSession", () => { ownerId: "mcp-test", requesterId: "mcp-test", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 99_999, }, @@ -177,6 +180,7 @@ describe("McpSession", () => { ownerId: "mcp-test", requesterId: "mcp-test", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 5_000, }, @@ -312,6 +316,7 @@ describe("McpSession", () => { ownerId: "mcp-test", requesterId: "mcp-test", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: clock.now() + 3_000, }); @@ -359,6 +364,7 @@ describe("McpSession", () => { ...grant.lease, id: input.leaseId, lastRenewedAt: clock.now(), + idChosenByRequester: false, ttlDeadline: clock.now() + 300_000, ttlMs: 30_000, }); @@ -451,6 +457,7 @@ describe("McpSession", () => { grantedAt: 0, id: input.leaseId, lastRenewedAt: clock.now(), + idChosenByRequester: false, ownerId: "mcp-test", requesterId: "mcp-test", ttlMs: 12_345, @@ -765,6 +772,7 @@ describe("McpSession", () => { ownerId: "mcp-test", requesterId: "mcp-test", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: clock.now() + 12_000, }); @@ -897,6 +905,7 @@ describe("McpSession", () => { grantedAt: 0, id: input.leaseId, lastRenewedAt: clock.now(), + idChosenByRequester: false, ownerId: "agent-1", requesterId: "agent-1", ttlMs: 1_000, @@ -1007,6 +1016,7 @@ describe("McpSession", () => { grantedAt: 0, id: input.leaseId, lastRenewedAt: clock.now(), + idChosenByRequester: false, ownerId: "mcp-test", requesterId: "mcp-test", ttlMs: 60_000, diff --git a/src/mcp/test-support.ts b/src/mcp/test-support.ts index 5cc332c2..ed9cb67f 100644 --- a/src/mcp/test-support.ts +++ b/src/mcp/test-support.ts @@ -222,6 +222,7 @@ export function sampleGrant(overrides: { readonly leaseId?: string } = {}): Leas ownerId: "mcp-test", requesterId: "mcp-test", lastRenewedAt: 0, + idChosenByRequester: false, ttlMs: 60_000, ttlDeadline: 12_345, }, diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index 8e294f45..d8c937f4 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -27,6 +27,7 @@ function sampleGrant( grantedAt: 0, id: leaseId, lastRenewedAt: 0, + idChosenByRequester: false, ownerId: "agent-1", requesterId: "agent-1", ttlMs: 1_000, diff --git a/ui/src/views/leases.test.tsx b/ui/src/views/leases.test.tsx index ed74e875..2955e76a 100644 --- a/ui/src/views/leases.test.tsx +++ b/ui/src/views/leases.test.tsx @@ -23,6 +23,7 @@ function lease(overrides: Partial = {}): LeaseRecord { ownerId: "tok_1", requesterId: "tok_1", ttlDeadline: NOW + 125_000, + idChosenByRequester: false, ttlMs: 215_000, ...overrides, }; diff --git a/ui/src/views/stats.test.tsx b/ui/src/views/stats.test.tsx index 6377bfce..9f059bcc 100644 --- a/ui/src/views/stats.test.tsx +++ b/ui/src/views/stats.test.tsx @@ -57,6 +57,7 @@ function lease(id: string, requesterId: string, ttlDeadline: number): LeaseRecor ownerId: requesterId, requesterId, ttlDeadline, + idChosenByRequester: false, ttlMs: 15 * MINUTE, }; } diff --git a/ui/src/views/worker-detail.test.tsx b/ui/src/views/worker-detail.test.tsx index 2b0ab7f8..e47e66dc 100644 --- a/ui/src/views/worker-detail.test.tsx +++ b/ui/src/views/worker-detail.test.tsx @@ -64,6 +64,7 @@ const RUNNING: WorkerView = { ownerId: "agent", requesterId: "agent", ttlDeadline: NOW + 60_000, + idChosenByRequester: false, ttlMs: 60_000, }, ], diff --git a/ui/src/views/workers.test.tsx b/ui/src/views/workers.test.tsx index 31055268..13ea9706 100644 --- a/ui/src/views/workers.test.tsx +++ b/ui/src/views/workers.test.tsx @@ -81,6 +81,7 @@ describe("the workers views", () => { ownerId: "agent", requesterId: "agent", ttlDeadline: NOW + 60_000, + idChosenByRequester: false, ttlMs: 185_000, }; // The worker answered 10 seconds ago, when the provisioning device was 4 seconds in. @@ -159,6 +160,7 @@ describe("the busiest workers", () => { ownerId: "agent", requesterId: "agent", ttlDeadline: NOW + 60_000, + idChosenByRequester: false, ttlMs: 60_000, }); const fleet = [ @@ -194,6 +196,7 @@ describe("the busiest workers", () => { ownerId: "agent", requesterId: "agent", ttlDeadline: NOW + 60_000, + idChosenByRequester: false, ttlMs: 60_000, }; // A gateway's view of a starting worker: its health and host, and nothing else it reports. From 65da92f0b1e0ec1a9f4f14897749fbd5e2d1fb32 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:07:51 +0200 Subject: [PATCH 3/9] test: lease a free model in the gateway e2e for a caller-chosen ID (#410) --- e2e/gateway-fleet.test.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/e2e/gateway-fleet.test.ts b/e2e/gateway-fleet.test.ts index 060c6630..eb6d158b 100644 --- a/e2e/gateway-fleet.test.ts +++ b/e2e/gateway-fleet.test.ts @@ -757,9 +757,22 @@ describe("gateway fleet", () => { "the worker connected with its catalog", ); const lease = (agentId: string, extra: readonly string[]) => - gateway.cli(["lease", "--platform", "ios", "--agent-id", agentId, "--detach", ...extra], { - timeout: 30_000, - }); + gateway.cli( + [ + "lease", + "--platform", + "ios", + "--device", + "iPhone 16 Pro", + "--agent-id", + agentId, + "--detach", + ...extra, + ], + { + timeout: 30_000, + }, + ); const chosen = await lease("agent-a", ["--lease-id", "ad-7f3a"]); expect(chosen.code, chosen.stderr).toBe(0); From 2a017f7bc0d24e055e88afd6581a821ce88e3b4c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:09:21 +0200 Subject: [PATCH 4/9] docs: a requester chooses the lease ID (#410) --- docs/CLI.md | 22 ++++++++++++++---- docs/CLIENT.md | 24 +++++++++++++++++-- docs/EVENTS.md | 4 ++-- docs/HTTP-API.md | 43 ++++++++++++++++++++++++----------- docs/internal/ARCHITECTURE.md | 29 ++++++++++++++++------- docs/internal/COMPONENTS.md | 6 ++--- docs/internal/EVENTS.md | 4 ++-- src/gateway/lease-index.ts | 23 ++++++++++--------- 8 files changed, 110 insertions(+), 45 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index b78a0fe3..7db78e7f 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -84,6 +84,7 @@ command starts it again) to bring the platform up. | 12 | `COMPONENT_NOT_OWNED` | `component remove` of a component Simlock did not install, or one that changed on disk since | | 12 | `COMPONENT_IN_USE` | `component remove` of a component a device uses, Simlock's or your own | | 13 | `REQUESTER_ALREADY_LEASED` | requester already holds a lease or has a pending request — one lease per agent in v1; release the named lease first | +| 13 | `LEASE_ID_TAKEN` | `lease --lease-id` named an ID an active lease or a waiting request already holds | | 14 | — | `lease` without `--detach` only: the daemon ended the lease without the holder asking (TTL expiry, operator `release`, or an unrecoverable device) | | 15 | — | `component install --worker`/`--all-workers` on a gateway only: at least one worker did not end `installed` or `already-installed` (it refused, failed, was skipped, or its result is unknown) | @@ -176,7 +177,7 @@ a timer and releasing it when it exits. ``` simlock lease --platform [--device | --class ] [--os ] [--mode ] [--image-tag ] [--agent-id ] - [--timeout ] + [--timeout ] [--lease-id ] [--no-wait] [--detach] [--ttl ] [--allow-download] [--export-env] [--bind-pid ] ``` @@ -209,6 +210,15 @@ granted. [Agent identity](#agent-identity). Defaults to `SIMLOCK_AGENT_ID`, then the agent tool's session id, then a pid-derived value. - `--timeout` — max time to wait in the queue (exit 10 on expiry). +- `--lease-id ` — the ID the granted lease gets, in place of one Simlock + generates, for a caller that already has its own ID for the lease. 1 to 64 + ASCII letters, digits, `-` and `_`, starting with a letter or digit, and + case-sensitive; anything else is a `BAD_REQUEST` (exit 2). The ID is yours + to keep unique for all time: use it for one lease and do not send it again + once that lease has ended. An ID an active lease or a waiting request + already holds is `LEASE_ID_TAKEN` (exit 13); a requester that already holds + a lease gets `REQUESTER_ALREADY_LEASED` first. `renew`, `release` and + `list` then name the lease by this ID, through a gateway too. - `--no-wait` — fail immediately with exit 11 instead of queueing. - `--allow-download` — permit downloading a missing runtime / system image (multi-GB; never implicit). Without it, a missing runtime is exit 12. @@ -789,9 +799,13 @@ The grant carries one additional block so you can see where it landed: {"lease":{"id":"3f81a2c4.lse_9f2c","worker":{"id":"3f81a2c4","label":"mac-studio-2"}}} ``` -The lease id names its worker (that is how renew, release, and reads route -with no gateway-side state to lose), but it is **opaque** — do not parse it. -`worker.label` is display-only. +A lease Simlock named has an id that names its worker (that is how renew, +release, and reads route), but it is **opaque** — do not parse it. A lease +you named with `--lease-id` keeps exactly that ID through a gateway, with no +worker in front of it; the gateway finds the worker from a table it keeps in +memory and rebuilds from its workers after a restart, so a renew or release +in the moment after a restart can answer `UNKNOWN_LEASE` until the worker has +reported. `worker.label` is display-only. **`lease renew`, `release`, and lease reads are forwarded** to the worker that owns the lease, and the `ttlDeadline` you see is that worker's own. diff --git a/docs/CLIENT.md b/docs/CLIENT.md index 423345cb..fb21ee0e 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -129,6 +129,25 @@ installer printed one. A request that joins a download already running hears that download's latest progress at once. A request that needs no download never hears this stage. +**Choosing the lease ID.** Pass `leaseId` when you already have your own ID +for the lease and want Simlock to use it, so there is nothing to map: + +```ts +const grant = await client.requestLease({ platform: "ios", leaseId: "ad-7f3a" }); +grant.lease.id; // "ad-7f3a" +grant.lease.idChosenByRequester; // true +``` + +The ID is 1 to 64 ASCII letters, digits, `-` and `_`, starts with a letter or +digit, and is case-sensitive; anything else is a `BAD_REQUEST`. `renewLease`, +`releaseLease` and the lease lists then name the lease by that ID, against a +gateway too, where it comes back with no worker in front of it. You keep the +ID unique for all time: it names one lease, and you do not send it again once +that lease has ended. Simlock refuses one an active lease or a waiting request +holds with `LEASE_ID_TAKEN` (`details.leaseId`), after the +`REQUESTER_ALREADY_LEASED` check, and keeps no record of IDs already used. +Without `leaseId` a request gets an ID from Simlock, as before. + **Keeping the lease alive is yours to do.** Every lease is TTL-bound: it expires at `grant.lease.ttlDeadline` unless a `renewLease` call lands first, and the daemon does nothing on its own to keep it. `requestLease` takes an optional @@ -160,7 +179,8 @@ the request is still waiting you join that wait, and once it has a result you get that result. Either way it never grants you a second lease. A result is never worked out again: a request that failed stays failed under its key, so use a new key to try again. Keys last for `lease.requestRetentionMs` after -the request finishes. The same key with a different device is +the request finishes. The same key with a different device, or a different +`leaseId` (sent on one call and not the other counts as different), is `IDEMPOTENCY_CONFLICT`. Keys belong to a requester id, and a repeat must come from the same connection principal that sent the request: the same key and requester id from a different principal is `FORBIDDEN`. A request still waiting when the daemon restarts @@ -737,7 +757,7 @@ difference, and neither does code written against it. mode (`"slim" | "full"`). Everything else you might reach for is a leaky inference rather than an answer: a lease from a gateway carries an additive `worker: { id, label }` block, but so might a future single-machine daemon's; -a lease id from a gateway names its worker, but ids are opaque and parsing +a lease id from a gateway can name its worker, but ids are opaque and parsing one is a bug waiting to happen. Two behaviours worth knowing when the daemon on the other end is a gateway, diff --git a/docs/EVENTS.md b/docs/EVENTS.md index b7b1bea5..8de511fe 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -15,7 +15,7 @@ through `simlock events` and `simlock events --follow`. | `lease.renewed` | lease id, new deadline | a `lease.renew` succeeded — whether it came from `simlock lease renew`, `POST /v1/leases/{id}/renew`, or the renew timer a running `simlock lease` / MCP session keeps over its own lease. There is one renew path and this is it | LeaseLifecycle | implemented | | `lease.released` | lease id, device id, reason (explicit/killed/device-lost), owner id | an explicit `lease.release` (which is what a `simlock lease` holder does on its way out), (killed) an operator `release --all` or `nuke`, or (device-lost) a leased device could not be recovered after it stopped running outside simlock, or a daemon start found its device not running, and the device of that lease was wiped and returned to the pool, left waiting in `reclaiming` on a platform the daemon could not list, or marked missing. Closing a connection is not a release and never emits this | LeaseLifecycle | implemented | | `lease.expired` | lease id, device id, owner id | the lease's deadline passed with no `lease.renew` behind it — the grant-time TTL, or the TTL of the last renew, simply ran out. This is the one way a lease ends without somebody asking, and the only bound on a holder that was killed outright | LeaseLifecycle | implemented | -| `lease.rejected` | request id, requester, request spec (as on `lease.requested`: a request that named a class or nothing has no model, and one that named an OS range carries it as typed), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused before it was stored (`killed`, `already-leased`) emits no `lease.requested`, and its `lease.rejected` carries the id it would have been stored under; on a gateway, `no-worker` is a request no worker that takes requests can serve, `NO_CAPACITY` at once, and `unresolvable-spec` is one no known worker has the platform, model, or runtime for, and such a request emits no `lease.queued`. A request a worker refused as unable to serve (`RUNTIME_MISSING`, `UNKNOWN_MODEL`, `NO_DRIVER`) while another worker was busy waits in the gateway queue, unless it is a `noWait` request, which is rejected at once with reason `no-wait` and emits no `lease.queued`; a request that did queue and then finds no worker left emits `unresolvable-spec` after its `lease.queued`, and one that never queued gets only the refusing worker's own `lease.rejected`; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again. The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter — one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | +| `lease.rejected` | request id, requester, request spec (as on `lease.requested`: a request that named a class or nothing has no model, and one that named an OS range carries it as typed), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused before it was stored (`killed`, `already-leased`, `lease-id-taken`) emits no `lease.requested`, and its `lease.rejected` carries the id it would have been stored under; on a gateway, `no-worker` is a request no worker that takes requests can serve, `NO_CAPACITY` at once, and `unresolvable-spec` is one no known worker has the platform, model, or runtime for, and such a request emits no `lease.queued`. A request a worker refused as unable to serve (`RUNTIME_MISSING`, `UNKNOWN_MODEL`, `NO_DRIVER`) while another worker was busy waits in the gateway queue, unless it is a `noWait` request, which is rejected at once with reason `no-wait` and emits no `lease.queued`; a request that did queue and then finds no worker left emits `unresolvable-spec` after its `lease.queued`, and one that never queued gets only the refusing worker's own `lease.rejected`; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again. The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter — one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | On a **gateway**, the first three of these are its own fleet queue's facts, emitted by `FleetLeaseCoordinator` and never by the worker whose device is @@ -23,7 +23,7 @@ eventually granted — `lease.granted`/`renewed`/`released`/`expired` for a flee lease arrive already relayed from the owning worker (see "Fleet (gateway mode)" below), so a gateway never emits those four itself. `already-leased` on a gateway is the fleet-wide one-lease-per-requester check, answered from -the gateway's own lease index before any worker is ever contacted. +the gateway's own lease index before any worker is ever contacted. `lease-id-taken` is a `leaseId` an active lease or a waiting request already holds; on a gateway it is the gateway's own leases and requests, checked the same way, and a worker's own refusal of it is that worker's `lease.rejected`. ## Capacity and queue diff --git a/docs/HTTP-API.md b/docs/HTTP-API.md index 193f35f0..a4149182 100644 --- a/docs/HTTP-API.md +++ b/docs/HTTP-API.md @@ -419,9 +419,21 @@ same tag, and a request without `imageTag` only one created for none. On iOS worker lists for that API level, an iOS one included, fails at once with `422 RUNTIME_MISSING`, with or without `noWait`. +`leaseId` (optional) is the ID you want the granted lease to have, in place of +one Simlock generates: 1 to 64 ASCII letters, digits, `-` and `_`, starting +with a letter or digit, case-sensitive (`ad-7f3a` and `Ad-7f3a` are two IDs). +It has no `.`, so it can never look like a gateway's `.`. +Everything that names the lease afterwards, `renew`, `release` and the reads, +uses that ID, on a worker and through a gateway. You promise it is unique for +all time: it names one lease, and you do not send it again once that lease has +ended. Simlock refuses an ID an active lease or a waiting request already holds +with `409 LEASE_ID_TAKEN` (the body carries `leaseId`) and does not remember +IDs that were used before. A requester that already holds a lease or a waiting +request gets `409 REQUESTER_ALREADY_LEASED` first, whatever `leaseId` it sends. + The body is strict: a key this route does not know, a `mode` other than -`"slim"` or `"full"`, or an `imageTag` that is not 1 to 64 letters, digits, -`_`, `.` or `-`, is `400 BAD_REQUEST`. +`"slim"` or `"full"`, an `imageTag` that is not 1 to 64 letters, digits, +`_`, `.` or `-`, or a `leaseId` outside the form above, is `400 BAD_REQUEST`. `allowDownload` is now clamped through `config.downloads.policy` the same way the socket protocol always was (**bug fix, 0.3.0**): before this @@ -440,7 +452,8 @@ changed since — a request that failed with `NO_CAPACITY` stays failed. To try again, use a new key. Repeating works across a daemon restart, for `lease.requestRetentionMs` after the request finished (see [CONFIGURATION.md](CONFIGURATION.md)). The same key with a different -`platform`, `device`, `os`, `mode`, or `imageTag` is `409 IDEMPOTENCY_CONFLICT`. Keys +`platform`, `device`, `os`, `mode`, `imageTag`, or `leaseId` (sent on one call and +not the other counts as different) is `409 IDEMPOTENCY_CONFLICT`. Keys belong to your token: another token sending the same key starts a request of its own. @@ -492,7 +505,7 @@ On a single host, with `allowDownload: true` the `201` is returned as soon as the request is stored — resolving a downloadable runtime can take minutes, so progress and any later failure surface on the request resource instead of on the `POST` itself. A refusal that comes before the request is stored -(`409 REQUESTER_ALREADY_LEASED`, `400` for a `ttlMs` above `lease.maxTtlMs`) +(`409 REQUESTER_ALREADY_LEASED`, `409 LEASE_ID_TAKEN`, `400` for a `ttlMs` above `lease.maxTtlMs`) still fails the `POST`. Through a gateway the flag changes nothing about the `POST`: it never downloads, so a request no worker can serve fails it as above. @@ -589,14 +602,18 @@ show it. `label` is display-only. A worker's network address is deliberately never here: clients reach the device through the gateway, with [`POST /v1/leases/{id}/exec`](#post-v1leasesidexec). -The lease `id` names its worker (`.`, split -on the **first** `.`), which is how a gateway routes renew, release, and -reads with no state of its own to lose across a restart. A worker id is a -UUID, so a real id reads -`3f81a2c4-9b7d-4e21-8a55-1c0e6f2d7b93.lse_9f2c`; the examples here and -elsewhere in these docs abbreviate it to its first segment for legibility. -**Treat the whole id as opaque** — pass it back verbatim in paths and bodies, -and read `worker.id` when you want the machine. +A lease whose ID Simlock generated has an `id` that names its worker +(`.`, split on the **first** `.`), which is +how a gateway routes renew, release, and reads. A worker id is a UUID, so a +real id reads `3f81a2c4-9b7d-4e21-8a55-1c0e6f2d7b93.lse_9f2c`; the examples +here and elsewhere in these docs abbreviate it to its first segment for +legibility. A lease whose requester chose the ID (`leaseId` on +[`POST /v1/lease-requests`](#post-v1lease-requests)) keeps exactly that ID, +with no worker in front of it: the gateway routes it from a table it keeps in +memory and rebuilds from its workers after a restart, so for a moment after a +restart, renew and release of such a lease can answer `404 UNKNOWN_LEASE` +until its worker has reported. **Treat the whole id as opaque** — pass it back +verbatim in paths and bodies, and read `worker.id` when you want the machine. `dataPlane` is **reserved** and always `null` in this version: streaming a device's screen, forwarding a port, or opening an interactive TTY is a @@ -1358,7 +1375,7 @@ Every failure is the same shape the daemon protocol uses: | 401 | `UNAUTHENTICATED` (missing or unrecognized token) | | 403 | `FORBIDDEN` (role doesn't permit the route — including a `worker` token on any `/v1` route other than `/v1/uplink`, and an `agent`/`operator` token at `/v1/uplink`; a `/v1/lease-requests/*` route whose request another token sent; or `POST /v1/leases/{id}/renew`/`DELETE /v1/leases/{id}`/`POST /v1/leases/{id}/exec` naming another requester's still-live lease), `DOWNLOADS_DISABLED` (`POST /v1/components/install` under `downloads.policy: "never"`) | | 404 | `UNKNOWN_WORKER` (`POST`/`DELETE /v1/workers/{id}/drain` naming a worker the gateway does not know), `UNKNOWN_LEASE_REQUEST` (unknown request id), `UNKNOWN_LEASE` (unknown lease id, expired/released, **or `GET /v1/leases/{id}`/`GET /v1/leases/{id}/events` naming another requester's lease** — see [`GET /v1/leases/{id}`](#get-v1leasesid)) | -| 409 | `REQUESTER_ALREADY_LEASED` (body names the existing lease id; fleet-wide on a gateway), `IDEMPOTENCY_CONFLICT` (an `Idempotency-Key` repeated with a different device), `REQUEST_NOT_CANCELLABLE` (body names the lease id if the request had already been granted), `WORKER_CONNECTED` (`DELETE /v1/workers/{id}` while its uplink is open), `COMPONENT_NOT_OWNED`, `COMPONENT_IN_USE` (body carries `devices` and `foreignDevices`), `COMPONENT_BUSY` (the three refusals of `DELETE /v1/components/{platform}/{version}`) | +| 409 | `REQUESTER_ALREADY_LEASED` (body names the existing lease id; fleet-wide on a gateway), `LEASE_ID_TAKEN` (a `leaseId` an active lease or a waiting request holds; body carries `leaseId`; on a gateway, one of the gateway's own leases or requests, or a worker that refused it), `IDEMPOTENCY_CONFLICT` (an `Idempotency-Key` repeated with a different device or `leaseId`), `REQUEST_NOT_CANCELLABLE` (body names the lease id if the request had already been granted), `WORKER_CONNECTED` (`DELETE /v1/workers/{id}` while its uplink is open), `COMPONENT_NOT_OWNED`, `COMPONENT_IN_USE` (body carries `devices` and `foreignDevices`), `COMPONENT_BUSY` (the three refusals of `DELETE /v1/components/{platform}/{version}`) | | 422 | `UNKNOWN_MODEL`, `RUNTIME_MISSING`, `NO_DRIVER`, `PASSTHROUGH_REFUSED` (a refused `exec` verb, a caller-supplied `--set`/`-P`, a bare `adb shell`), `UNKNOWN_PASSTHROUGH_TOOL` | | 501 | `UNSUPPORTED_IN_GATEWAY_MODE` (an operation that acts on one machine, asked of a gateway: `POST /v1/components/install`, `GET /v1/components`, `DELETE /v1/components/{platform}/{version}`), `UNSUPPORTED_IN_WORKER_MODE` (an operation on a gateway's workers, asked of a single host: `POST`/`DELETE /v1/workers/{id}/drain`, `DELETE /v1/workers/{id}`, `POST /v1/components/install` with `workers`) | | 503 | `NO_CAPACITY` (with `noWait: true`, or, on a gateway, when no worker that takes requests can serve the request; response carries `Retry-After`), `WORKER_UNREACHABLE` (a gateway could not reach the worker holding this lease or request) | diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index 7b0f4b31..625a3cf5 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -758,14 +758,27 @@ state at all. `lease.renew`, `lease.release`, and single-lease reads are worker's own. There is nothing to emulate and no timer to run: a client that stops renewing loses its lease on the worker's clock, gateway or no gateway. -- **The lease id names its worker.** A gateway lease id is the owning - worker's id, then a `.`, then the worker's own lease id — so renew, - release, and reads route by splitting on the **first** `.` rather than by - consulting state a restart could lose. A worker id is its instance - identity, a UUID, so a real one reads - `3f81a2c4-9b7d-4e21-8a55-1c0e6f2d7b93.lse_9f2c`; every example in these - docs abbreviates it to its first segment for legibility. Clients treat the - whole thing as opaque, exactly as they already treat `lse_9f2c`. +- **A generated lease id names its worker.** A gateway lease id for a lease + simlock generated is the owning worker's id, then a `.`, then the worker's + own lease id. A worker id is its instance identity, a UUID, so a real one + reads `3f81a2c4-9b7d-4e21-8a55-1c0e6f2d7b93.lse_9f2c`; every example in + these docs abbreviates it to its first segment for legibility. Clients treat + the whole thing as opaque, exactly as they already treat `lse_9f2c`. +- **A caller-chosen lease id crosses the gateway bare** (ADR 0020). A + requester that sent `leaseId` gets a lease with exactly that id, with no + worker prefix, because an id the caller made up cannot carry one. The + gateway routes renew, release and reads for it from its lease index, an + in-memory map it rebuilds from what workers report (a lease flagged + `idChosenByRequester` whose id matches the `leaseId` pattern is named + bare), so a gateway restart loses bare routes until each worker has + reported; a renew in that window is `UNKNOWN_LEASE`, and the gateway never + asks a worker on a miss. The gateway takes the id from what it forwarded, + never from the worker's echo; a grant that differs is released on the worker + and the request waits again. It refuses an id its own leases or its own + waiting requests hold, and passes a worker's `LEASE_ID_TAKEN` on without + trying another worker. If two workers ever report the same bare id, the + first stays routed and the other lease expires at its TTL (reviewed in + #412). - **The lease object gains `worker: { id, label }`** (additive) so a client and the console can say *where* the device lives. A worker's network address is never on it: clients reach devices through the gateway. diff --git a/docs/internal/COMPONENTS.md b/docs/internal/COMPONENTS.md index 1f7ac68d..b759861f 100644 --- a/docs/internal/COMPONENTS.md +++ b/docs/internal/COMPONENTS.md @@ -93,9 +93,9 @@ modules with. |---|---|---|---| | `createLeasing`, `LeaseStartup` | `src/leasing/create-leasing.ts`, `src/leasing/lease-startup.ts` | The composition root for leasing, built on `createCore`'s services: it wires the health monitor or leaves it out, and builds the ports core needs as `corePorts`, which the daemon hands to `core.connect`. The lease half of startup, around core's read: settle every request the previous process left open as failed, then, against the startup read, end the leases whose device is not running and restore the TTL timer of the rest. | Run a device operation: it calls core's services. | | `LeaseReconciler` | `src/leasing/lease-reconciler.ts` | Judging every lease on disk against the startup read by the device's registry `driverDeviceId`, and handing each lease whose device is not running to the release coordinator as `device-lost` or `expired` with the device outcome the read implies (reclaim, wait, missing). | Reclaim, mark a device missing or emit an event itself: the release path owns all three. | -| `LeaseRequestBook` | `src/leasing/lease-request-book.ts` | The lease-request rules: store before queueing, answer a repeat under the same idempotency key, write a result once (a daemon's grant is written with its lease, in one commit). Shared with the gateway over an in-memory store. | Queue or serve the request. | +| `LeaseRequestBook` | `src/leasing/lease-request-book.ts` | The lease-request rules: store before queueing, answer a repeat under the same idempotency key and `leaseId`, write a result once (a daemon's grant is written with its lease, in one commit), and say whether a request still open holds a caller-chosen lease ID. Shared with the gateway over an in-memory store. | Queue or serve the request. | | `WaitQueue` | `src/leasing/wait-queue.ts` | FIFO membership, timeouts, cancellation, progress, and settlement of pending requests. | Decide whether capacity exists or perform lease work. | -| `LeaseAcquisitionCoordinator` | `src/leasing/lease-acquisition-coordinator.ts` | Admission, resolving a request into a requirement and a create spec (class, OS range, default mode, image tag), driving plans to a grant, eviction by demand, maintenance fencing for nuke. Exposes the queue head's spec, the waiting requests and the demand the warm pool reads (`waitingDemand`), read only, and `resolve`, which turns a request into its spec with downloads off for the warm pool's targets. | Choose which device: the planner does. Reclaim: the release side does. | +| `LeaseAcquisitionCoordinator` | `src/leasing/lease-acquisition-coordinator.ts` | Admission (the one-lease-per-requester check, then the lease-ID clash check), resolving a request into a requirement and a create spec (class, OS range, default mode, image tag), driving plans to a grant, eviction by demand, maintenance fencing for nuke. Exposes the queue head's spec, the waiting requests and the demand the warm pool reads (`waitingDemand`), read only, and `resolve`, which turns a request into its spec with downloads off for the warm pool's targets. | Choose which device: the planner does. Reclaim: the release side does. | | `AcquisitionPlanner` | `src/leasing/acquisition-planner.ts` | The read-only plan for one request: grant a fitting ready device, wait for a fitting one the warm pool is booting, boot a fitting shut-down one, evict an idle running device, provision, wait, or refuse. Takes capacity reservations and claims and hands them to the caller. | Perform any side effect. | | `LeaseLifecycle` | `src/leasing/lease-lifecycle.ts` | Grant, renew, the registry half of a release, expiry scheduling through `LeaseExpiryScheduler`. | Reclaim; wake the queue. | | `LeaseExpiryScheduler` | `src/leasing/lease-expiry-scheduler.ts` | TTL timers, delivered to the release coordinator. | Decide what expiry means. | @@ -131,7 +131,7 @@ typed against and catalog matching (ARCHITECTURE.md, "Boundaries"). | `GatewayService` | `src/gateway/service.ts` | Lifecycle: the uplink listener, one `WorkerLink` per worker, feeding the registry, the periodic tick that backstops refreshes and sweeps expired views. | Answer an operation or define a view. | | `WorkerLink` | `src/gateway/worker-link.ts` | One worker's uplink from the gateway's side: the typed admin client over it, the reads on connect, refreshes on worker events, relaying worker events with the worker's id. | Decide what a view means. | | `WorkerRegistry`, drain store | `src/gateway/worker-registry.ts`, `src/gateway/drain-store.ts` | What a worker view is and when it changes; the facts emitted as views change; the one persisted bit, the drained set. | Talk to a worker. | -| `FleetLeaseCoordinator`, `FleetQueue`, `FleetLeaseIndex` | `src/gateway/fleet-coordinator.ts`, `src/gateway/queue.ts`, `src/gateway/lease-index.ts` | Admission and the fleet-wide one-lease rule, the fleet FIFO, dispatch of each queued request to the worker routing picks with `noWait`, retry on a cannot-serve refusal, forwarding lease and exec calls, projecting worker leases as fleet leases. | Provision, evict, or hold a device opinion: a worker grants or refuses. | +| `FleetLeaseCoordinator`, `FleetQueue`, `FleetLeaseIndex` | `src/gateway/fleet-coordinator.ts`, `src/gateway/queue.ts`, `src/gateway/lease-index.ts` | Admission, the fleet-wide one-lease rule and the lease-ID clash check, the fleet FIFO, dispatch of each queued request to the worker routing picks with `noWait`, retry on a cannot-serve refusal, forwarding lease and exec calls, projecting worker leases as fleet leases, naming a caller-chosen lease bare and routing it from the index. | Provision, evict, or hold a device opinion: a worker grants or refuses. | | Routing | `src/gateway/routing.ts`, `src/gateway/routing/` | The pure routing policy: an ordered list of stages (`takes-requests`, `can-serve`, `healthy`, `idle-queue`, `warm-hit`, `free-slot`, `ram-budget`, `free-capacity`) over worker views, selected by `gateway.routing`. | Read anything but views; order stages from config. | | `GatewayDispatcher` | `src/gateway/dispatcher.ts` | The second implementation of the contract's handlers (ADR 0005 §32): answered from the fleet, forwarded to one worker, or fanned out to all. | Define a second contract or role check. | | Aggregate, component relay | `src/gateway/aggregate.ts`, `src/gateway/component-relay.ts` | The fleet expressed in one machine's shapes, as pure functions over views; asking every worker to install a component and collecting one outcome each. | Decide anything a worker decides. | diff --git a/docs/internal/EVENTS.md b/docs/internal/EVENTS.md index d5bc3e85..f4fb7cc1 100644 --- a/docs/internal/EVENTS.md +++ b/docs/internal/EVENTS.md @@ -46,7 +46,7 @@ in short: `subject.past-tense-fact`, emitted post-commit, facts not commands. | `lease.renewed` | lease id, new deadline | a `lease.renew` succeeded — whether it came from `simlock lease renew`, `POST /v1/leases/{id}/renew`, or the renew timer a running `simlock lease` / MCP session keeps over its own lease. There is one renew path and this is it | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.released` | lease id, device id, reason (explicit/killed/device-lost), owner id | an explicit `lease.release` (which is what a `simlock lease` holder does on its way out), (killed) an operator `release --all` or `nuke`, or (device-lost) a leased device could not be recovered after it stopped running outside simlock, or a daemon start found its device not running, and the device of that lease was wiped and returned to the pool, left waiting in `reclaiming` on a platform the daemon could not list, or marked missing. Closing a connection is not a release and never emits this | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.expired` | lease id, device id, owner id | the lease's deadline passed with no `lease.renew` behind it — the grant-time TTL, or the TTL of the last renew, simply ran out. This is the one way a lease ends without somebody asking, and the only bound on a holder that was killed outright | LeaseLifecycle | implemented (payload per ADR 0004 pending) | -| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused at admission (`killed`, `already-leased`) was never stored and has no `lease.requested`; it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec` only if it had entered the gateway queue (its `lease.queued` needs a terminal fact); otherwise the worker's own `lease.rejected` is the terminal fact and the gateway emits none; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | +| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never stored and has no `lease.requested`; it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec` only if it had entered the gateway queue (its `lease.queued` needs a terminal fact); otherwise the worker's own `lease.rejected` is the terminal fact and the gateway emits none; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | On a **gateway**, these three are its own fleet queue's facts (ADR 0005 §11/§14), emitted by `FleetLeaseCoordinator` and never by the worker whose device is @@ -55,7 +55,7 @@ lease arrive already relayed from the owning worker (see "Fleet (gateway mode)" below), so a gateway never emits those four itself. `already-leased` on a gateway is the fleet-wide one-lease-per-requester check (§14, keyed on `requesterId`), answered from the gateway's own lease index before any -worker is ever contacted. +worker is ever contacted. `lease-id-taken` (ADR 0020; widens a published vocabulary, which events rule 6 allows as additive) is a `leaseId` an active lease or a waiting request already holds; on a gateway it is the gateway's own leases and requests, checked the same way, and a worker's own refusal of it is that worker's `lease.rejected`, as for `no-wait`. A gateway also emits it when a grant arrives for a bare id its index already routes to another worker: the new worker's lease is released and the request ends with it. ## Capacity and queue diff --git a/src/gateway/lease-index.ts b/src/gateway/lease-index.ts index be7f13f7..9f91394a 100644 --- a/src/gateway/lease-index.ts +++ b/src/gateway/lease-index.ts @@ -4,18 +4,19 @@ import { NoopLogger } from "../ports/index.js"; /** * `FleetLeaseIndex`: the gateway's own record of which leases *it* issued (ADR 0005 §14, §16, - * §27a, §30). Nothing here is persisted -- like every worker view, it is rebuilt from what a - * worker reports (`rebuildFromWorker`) whenever the gateway can see it, which is what makes a - * gateway restart lose nothing a worker restart would not also lose (Decision 5). + * §27a, §30; ADR 0020). Nothing here is persisted -- like every worker view, it is rebuilt from + * what a worker reports (`rebuildFromWorker`) whenever the gateway can see it. A gateway restart + * loses nothing a worker restart would not also lose (Decision 5) for a generated id, which names + * its worker; a caller-chosen id (ADR 0020) is bare and routed by this index alone, so for the + * moment before its worker has reported, the gateway does not know it. * - * A gateway lease id is minted once, at grant, as `${workerId}.${workerLeaseId}` (§16) -- or, for - * a lease whose requester chose its id (ADR 0020), as that id bare, routed by this index alone -- - * and never re-derived by splitting the string back apart -- every lookup in this class goes through the - * map this index keeps, keyed by the id it minted or by the `(workerId, workerLeaseId)` pair a - * relayed worker event carries. The "split on the first `.`" the ADR describes is what makes the - * id *routable in principle* (a worker id is a UUID, so it can never itself contain the - * separator); nothing in this codebase needs to actually perform that split, because this index - * always has the structured pair already. + * A generated gateway lease id is minted once, at grant, as `${workerId}.${workerLeaseId}` (§16) -- + * a caller-chosen one is that id as sent, bare -- and never re-derived by splitting the string back + * apart: every lookup in this class goes through the map this index keeps, keyed by the id it + * minted or by the `(workerId, workerLeaseId)` pair a relayed worker event carries. The "split on + * the first `.`" the ADR describes is what makes a generated id *routable in principle* (a worker + * id is a UUID, so it can never itself contain the separator); nothing in this codebase needs to + * actually perform that split, because this index always has the structured pair already. */ /** One lease this gateway knows it issued. `requesterId`/`ownerId` are the *fleet-level* From dcdee701080baf39db9b7d21d7249bb1140a9ff5 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:11:24 +0200 Subject: [PATCH 5/9] test: pin the lease ID paths on the wire, the dispatchers, the book and the registry (#410) --- docs/CLI.md | 2 +- src/contract/operations.test.ts | 50 ++++++++++++++ src/core/registry.test.ts | 92 +++++++++++++++++++++++++- src/daemon/dispatcher.test.ts | 14 ++++ src/daemon/error-code.test.ts | 8 ++- src/gateway/dispatcher.test.ts | 30 +++++++++ src/http/errors.test.ts | 21 +++++- src/leasing/create-leasing.test.ts | 3 +- src/leasing/lease-request-book.test.ts | 54 +++++++++++++++ src/mcp/server.ts | 2 +- 10 files changed, 269 insertions(+), 7 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index 7db78e7f..f7596111 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -68,7 +68,7 @@ command starts it again) to bring the platform up. | 2 | `UNKNOWN_REQUEST` | the daemon has no such operation — usually a client newer than the daemon | | 2 | `PASSTHROUGH_REFUSED` | a `simctl`/`adb` verb simlock refuses, a caller-supplied `--set`/`-P`, or a bare `adb shell` where there is no terminal to give it | | 2 | `UNKNOWN_PASSTHROUGH_TOOL` | a passthrough tool simlock does not wrap | -| 2 | `IDEMPOTENCY_CONFLICT` | a lease request reused an idempotency key its requester already sent for a different device; use a new key | +| 2 | `IDEMPOTENCY_CONFLICT` | a lease request reused an idempotency key its requester already sent for a different device or `--lease-id`; use a new key | | 10 | `QUEUE_TIMEOUT` | timed out waiting for a device (`--timeout` elapsed) | | 10 | `EXEC_TIMEOUT` | a `simctl`/`adb` command run through `device.exec` outlived `exec.timeoutMs` and was killed | | 10 | `DOWNLOAD_TIMEOUT` | a runtime download, including the time spent waiting for another download on the same platform, outlived `downloads.timeoutMs` | diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 5b511cff..2afb83c9 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -233,6 +233,56 @@ describe("lease.request osVersion", () => { ); }); +describe("lease.request leaseId", () => { + const parse = (leaseId: unknown) => + OPERATIONS["lease.request"].input.safeParse({ leaseId, model: "iPhone 17", platform: "ios" }); + + it.each([ + ["a plain ID", "myid"], + ["a case-sensitive one", "MyID"], + ["one with a hyphen and an underscore inside", "ad-7f3a_01"], + ["one that looks like a generated ID", "lse_123"], + ["a single character", "a"], + ["a digit first", "7"], + ["64 characters", "a".repeat(64)], + ])("accepts %s", (_label, leaseId) => { + expect(parse(leaseId).success).toBe(true); + }); + + it.each([ + ["an empty string", ""], + ["65 characters", "a".repeat(65)], + ["a dot, kept for gateway IDs", "w1.myid"], + ["a space", "my id"], + ["a hyphen first", "-s"], + ["an option", "--help"], + ["an underscore first", "_x"], + ["a character outside ASCII", "ząb"], + ["a trailing newline", "myid\n"], + ["a fullwidth digit", "\uFF11abc"], + ["a number", 7], + ["null", null], + ])("refuses %s", (_label, leaseId) => { + expect(parse(leaseId).success).toBe(false); + }); + + it("is optional, and an input without it parses to one without it", () => { + expect( + OPERATIONS["lease.request"].input.parse({ model: "iPhone 17", platform: "ios" }), + ).toEqual({ model: "iPhone 17", platform: "ios" }); + }); + + it("is not part of the device the request names", () => { + const input = OPERATIONS["lease.request"].input.parse({ + leaseId: "myid", + model: "iPhone 17", + platform: "ios", + }); + + expect(requestedDevice(input)).toEqual({ model: "iPhone 17", platform: "ios" }); + }); +}); + describe("operation input/output round trips", () => { it("lease.request: round-trips a representative request and rejects legacy aliases", () => { const input = OPERATIONS["lease.request"].input.parse({ diff --git a/src/core/registry.test.ts b/src/core/registry.test.ts index a7640361..8d70dc14 100644 --- a/src/core/registry.test.ts +++ b/src/core/registry.test.ts @@ -1629,6 +1629,96 @@ describe("Registry", () => { ]); }); + it("keeps the leaseId a request stored across a reload, and loads a record without one as having none", async () => { + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + const options = { + clock, + eventBus: new EventBus(clock), + filesystem, + idGenerator: { generate: () => "unexpected" }, + statePath, + }; + const registry = await Registry.load(options); + await registry.createLeaseRequest({ + id: "req_1", + ownerId: "agent-1", + request: { platform: "ios" }, + requesterId: "agent-1", + leaseId: "ad-7f3a", + }); + await registry.createLeaseRequest({ + id: "req_2", + ownerId: "agent-2", + request: { platform: "ios" }, + requesterId: "agent-2", + }); + + const reloaded = await Registry.load(options); + + expect(reloaded.leaseRequests()).toMatchObject([ + { id: "req_1", leaseId: "ad-7f3a" }, + { id: "req_2" }, + ]); + expect(reloaded.leaseRequests()[1]).not.toHaveProperty("leaseId"); + }); + + it("loads the lease of a stored grant written before idChosenByRequester as one simlock named", async () => { + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + const lease = { + deviceId: "dev_1", + grantedAt: 1_000, + id: "lse_1", + lastRenewedAt: 1_000, + ownerId: "agent-1", + requesterId: "agent-1", + ttlDeadline: 2_000, + ttlMs: 60_000, + }; + const device = { + createdAt: 1_000, + driverData: {}, + driverDeviceId: "driver_dev_1", + id: "dev_1", + spec, + state: "leased", + }; + await filesystem.writeFileAtomic( + statePath, + JSON.stringify({ + devices: [device], + leaseRequests: [ + { + createdAt: 1_000, + grant: { device, environment: {}, lease, timing: {} }, + id: "req_1", + ownerId: "agent-1", + request: { platform: "ios" }, + requesterId: "agent-1", + settledAt: 1_000, + state: "granted", + }, + ], + leases: [lease], + }), + ); + + const registry = await Registry.load({ + clock, + eventBus: new EventBus(clock), + filesystem, + idGenerator: { generate: () => "unexpected" }, + statePath, + }); + + expect(registry.leaseRequests()[0]?.grant?.lease).toMatchObject({ + id: "lse_1", + idChosenByRequester: false, + }); + }); + it("writes idChosenByRequester true for a lease created with a leaseId, false without one, and keeps both across a reload", async () => { const clock = new FakeClock(1_000); const filesystem = new MemoryFilesystem(); @@ -1660,7 +1750,7 @@ describe("Registry", () => { ...base, deviceId: await ready("one"), leaseId: "ad-7f3a", - } as Parameters[0]); + }); const generated = await registry.createLease({ ...base, deviceId: await ready("two") }); expect(chosen).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 4b006647..cd4c9bab 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -447,6 +447,20 @@ describe("Dispatcher: parsing", () => { ]); }); + it("grants lease.request's leaseId as the lease ID, and answers an ID that is in use with LEASE_ID_TAKEN carrying it in details", async () => { + const { dispatcher } = await buildDispatcher(); + const input = { leaseId: "ad-7f3a", model: "iPhone 17 Pro", platform: "ios" } as const; + + const first = await dispatcher.dispatch("lease.request", input, session()); + const clash = dispatcher.dispatch("lease.request", input, session({ principal: "other" })); + + expect(first.lease).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); + await expect(clash).rejects.toMatchObject({ + code: "LEASE_ID_TAKEN", + details: { leaseId: "ad-7f3a" }, + }); + }); + it("rejects an operation this dispatcher has no handler for with UNKNOWN_REQUEST", async () => { const { dispatcher } = await buildDispatcher(); // "daemon.stop" is ADR §6's frozen exception -- `DaemonServer#dispatchLine` intercepts it diff --git a/src/daemon/error-code.test.ts b/src/daemon/error-code.test.ts index 1b8de99e..07bedef4 100644 --- a/src/daemon/error-code.test.ts +++ b/src/daemon/error-code.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { ComponentInstallerClosedError, UnsupportedRequestOptionError } from "../core/index.js"; -import { NoCapacityError, ReplayedLeaseRequestError } from "../leasing/index.js"; +import { LeaseIdTakenError, NoCapacityError, ReplayedLeaseRequestError } from "../leasing/index.js"; import { classifyError, describeLeaseRequestFailure } from "./error-code.js"; describe("describeLeaseRequestFailure", () => { @@ -49,3 +49,9 @@ describe("classifyError on a request option the driver does not have", () => { }); }); }); + +describe("classifyError on a lease ID that is in use", () => { + it("answers LEASE_ID_TAKEN for LeaseIdTakenError", () => { + expect(classifyError(new LeaseIdTakenError("ad-7f3a"))).toBe("LEASE_ID_TAKEN"); + }); +}); diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 9bde395b..72c7c827 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -1061,6 +1061,36 @@ describe("GatewayDispatcher", () => { expect(client.lastRequestLeaseInput).toMatchObject({ imageTag: "google_apis_playstore" }); }); + it("forwards a lease.request's leaseId to the worker, and answers the grant under that ID with no worker in front of it", async () => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ + grant: grantFixture({ + lease: { ...grantFixture().lease, id: "ad-7f3a", idChosenByRequester: true }, + }), + kind: "grant", + }); + + const grant = await dispatcher.dispatch( + "lease.request", + { leaseId: "ad-7f3a", model: "iPhone 17", noWait: true, platform: "ios" }, + session({ role: "agent" }), + ); + + expect(client.lastRequestLeaseInput).toMatchObject({ leaseId: "ad-7f3a" }); + expect(grant.lease).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); + }); + it("forwards device.exec to the worker that holds the lease, gated on the caller owning it", async () => { const { coordinator, directory, dispatcher, workers } = harness(); const client = new ScriptedWorkerClient(); diff --git a/src/http/errors.test.ts b/src/http/errors.test.ts index 8e1087cd..25084b24 100644 --- a/src/http/errors.test.ts +++ b/src/http/errors.test.ts @@ -10,7 +10,11 @@ import { UnknownLeaseError, UnknownModelError, } from "../core/index.js"; -import { NoCapacityError, RequesterAlreadyLeasedError } from "../leasing/index.js"; +import { + LeaseIdTakenError, + NoCapacityError, + RequesterAlreadyLeasedError, +} from "../leasing/index.js"; import { classifyError, StartupFailedError } from "../daemon/error-code.js"; import { DispatchError, @@ -47,6 +51,21 @@ describe("mapError", () => { expect(withoutLease.extra).toBeUndefined(); }); + it("maps LeaseIdTakenError and a DispatchError of LEASE_ID_TAKEN to 409, the DispatchError naming the ID in extra", () => { + expect(mapError(new LeaseIdTakenError("ad-7f3a"))).toMatchObject({ + code: "LEASE_ID_TAKEN", + status: 409, + }); + expect(mapError(new DispatchError("LEASE_ID_TAKEN", "in use", { leaseId: "ad-7f3a" }))).toEqual( + { + code: "LEASE_ID_TAKEN", + extra: { leaseId: "ad-7f3a" }, + message: "in use", + status: 409, + }, + ); + }); + it("maps NoCapacityError to 503", () => { expect(mapError(new NoCapacityError())).toMatchObject({ code: "NO_CAPACITY", status: 503 }); }); diff --git a/src/leasing/create-leasing.test.ts b/src/leasing/create-leasing.test.ts index 4f52bd22..a29bf017 100644 --- a/src/leasing/create-leasing.test.ts +++ b/src/leasing/create-leasing.test.ts @@ -3571,8 +3571,7 @@ describe("createLeasing a lease ID chosen by the requester", () => { maxRunning: 4, }; - /** What a caller that sends `leaseId` passes: the field is spread in so this file still builds - * against options that do not name it. */ + /** What a caller that sends `leaseId` passes. */ function asking(requesterId: string, leaseId?: string, more: Record = {}) { return { ownerId: requesterId, diff --git a/src/leasing/lease-request-book.test.ts b/src/leasing/lease-request-book.test.ts index 119b605a..d10c227f 100644 --- a/src/leasing/lease-request-book.test.ts +++ b/src/leasing/lease-request-book.test.ts @@ -459,6 +459,60 @@ describe("LeaseRequestBook", () => { expect(() => book.replay(different, keyed)).toThrow(IdempotencyConflictError); }); + it.each([ + ["a different leaseId", "other"], + ["no leaseId where one was sent", undefined], + ])("refuses a repeat naming %s as an idempotency conflict", async (_label, leaseId) => { + const book = bookOver(memoryStore()); + await book.admit(request, { ...keyed, leaseId: "myid" }, () => granted("myid")); + await settled(); + + expect(() => + book.replay(request, { ...keyed, ...(leaseId === undefined ? {} : { leaseId }) }), + ).toThrow(IdempotencyConflictError); + }); + + it("refuses a repeat that adds a leaseId the first request did not send as an idempotency conflict, and replays one that sends the same", async () => { + const book = bookOver(memoryStore()); + await book.admit(request, keyed, () => granted("lse_1")); + await settled(); + + expect(() => book.replay(request, { ...keyed, leaseId: "myid" })).toThrow( + IdempotencyConflictError, + ); + await expect(book.replay(request, keyed)).resolves.toMatchObject({ lease: { id: "lse_1" } }); + }); + + it("says an open request holds its leaseId, and a settled or cancelled one, or one that sent none, does not", async () => { + const book = bookOver(memoryStore()); + let finish: (grant: { lease: { id: string } }) => void = () => undefined; + await book.admit(request, { ...keyed, leaseId: "waits" }, () => ({ + promise: new Promise((resolve) => { + finish = resolve; + }), + })); + await book.admit( + request, + { idempotencyKey: "key-2", ownerId: "other", requesterId: "other", leaseId: "gone" }, + () => ({ promise: Promise.reject(new RequestCancelledError("req_x")) }), + ); + await book.admit(request, { ownerId: "third", requesterId: "third" }, () => ({ + promise: new Promise(() => undefined), + })); + await settled(); + + expect([ + book.holdsLeaseId("waits"), + book.holdsLeaseId("gone"), + book.holdsLeaseId("third"), + book.holdsLeaseId("never-sent"), + ]).toEqual([true, false, false, false]); + + finish({ lease: { id: "waits" } }); + await settled(); + expect(book.holdsLeaseId("waits")).toBe(false); + }); + it("keeps a settled record as it is when a second result arrives for it, and settles nothing for an unknown id", async () => { const store = memoryStore(); const created = await store.createLeaseRequest(newRequest("agent")); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index e15d044a..2753fa7d 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -123,7 +123,7 @@ export function createMcpServer(session: McpSession): McpServer { { title: "Lease simulator", description: - "Lease one simulator or emulator for this MCP session. Name an exact model (`model`), or a device class (`class`: phone, tablet, watch, tv, vision, auto or desktop), or name neither to get a phone; naming both is an error. A class or no-model request is served by a fitting idle device before a new one is created. The lease is held until released or this MCP connection closes; provisioning can block unless noWait is true. `osVersion` is an exact version (`18.4`) or a range: `>=`, `>`, `<=` or `<` followed by a version, joined by single spaces (`>=18 <26`), or a hyphen range (`18 - 26`); a range is served by a fitting device or the newest installed runtime in it, and never downloads. Downloads are disabled by default and require allowDownload: true. On Android, imageTag picks the system image type (a tag from list_devices' images, such as google_apis_playstore); a request with imageTag uses only an installed image and never downloads.", + "Lease one simulator or emulator for this MCP session. Name an exact model (`model`), or a device class (`class`: phone, tablet, watch, tv, vision, auto or desktop), or name neither to get a phone; naming both is an error. A class or no-model request is served by a fitting idle device before a new one is created. The lease is held until released or this MCP connection closes; provisioning can block unless noWait is true. `osVersion` is an exact version (`18.4`) or a range: `>=`, `>`, `<=` or `<` followed by a version, joined by single spaces (`>=18 <26`), or a hyphen range (`18 - 26`); a range is served by a fitting device or the newest installed runtime in it, and never downloads. Downloads are disabled by default and require allowDownload: true. On Android, imageTag picks the system image type (a tag from list_devices' images, such as google_apis_playstore); a request with imageTag uses only an installed image and never downloads. Pass `leaseId` to choose the lease's ID instead of getting one generated: 1 to 64 ASCII letters, digits, `-` or `_`, starting with a letter or digit; you keep it unique for all time, and an ID that an active lease or a waiting request already holds is refused with LEASE_ID_TAKEN.", inputSchema: leaseSimulatorInputSchema, outputSchema: leaseSimulatorOutputSchema, }, From bc2f39e60592d804905ce4b8f771a101029cad81 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:14:10 +0200 Subject: [PATCH 6/9] test: name the assertion for a refused ID that waits instead, and pin the retry after a wrong ID (#410) 2 survivors killed locally; unit lane green --- src/gateway/fleet-coordinator.test.ts | 38 +++++++++++++++++++++++++++ src/leasing/create-leasing.test.ts | 8 ++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index bfef956a..51736441 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -3607,6 +3607,20 @@ describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { return (client.lastRequestLeaseInput as Record | undefined)?.leaseId; } + /** A new answer from the worker `oneWorker` connected, as a periodic refresh brings one. */ + function refreshWorker(workers: WorkerRegistry, capacity: ReturnType): void { + workers.refresh("wrk_a", { + capacity, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + devices: [], + downloads: { policy: "on-request" }, + health: "running", + leases: [], + queueDepth: 0, + }); + } + function oneWorker(overrides: Parameters[0] = {}) { const fleet = harness(overrides); const client = new ScriptedWorkerClient(); @@ -3743,6 +3757,30 @@ describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { expect(JSON.stringify(logger.warnings)).toContain("wrk_a"); }); + it("a worker that answered a grant under another ID is not asked again until its view changes, then is, and the right grant settles the request", async () => { + const { client, coordinator, workers } = oneWorker(); + client.requestLeaseQueue.push( + { grant: chosenGrant("not-what-was-sent"), kind: "grant" }, + { grant: chosenGrant("myid"), kind: "grant" }, + ); + const request = coordinator.request(REQUEST, chosen("myid")); + const outcome = promiseState(request); + await tick(); + expect(leaseRequests(client)).toHaveLength(1); + + // The same view again: nothing changed, so the worker is left alone. + refreshWorker(workers, statusFixture().capacity); + await tick(); + expect(leaseRequests(client)).toHaveLength(1); + + // A changed view: it is asked again, and this time answers with the ID it was given. + refreshWorker(workers, roomierIos()); + await tick(); + expect(outcome.state).toBe("fulfilled"); + await expect(request).resolves.toMatchObject({ lease: { id: "myid" } }); + expect(leaseRequests(client)).toHaveLength(2); + }); + it("the gateway answers UNKNOWN_LEASE for a renew of an ID missing from its index", async () => { const { coordinator } = oneWorker(); diff --git a/src/leasing/create-leasing.test.ts b/src/leasing/create-leasing.test.ts index a29bf017..8d0f12bd 100644 --- a/src/leasing/create-leasing.test.ts +++ b/src/leasing/create-leasing.test.ts @@ -3654,7 +3654,11 @@ describe("createLeasing a lease ID chosen by the requester", () => { it("a second request for an ID held by a waiting request fails with LEASE_ID_TAKEN", async () => { const harness = await withWaiting("agent-2", "myid"); - await expect(harness.engine.request(request, asking("agent-3", "myid"))).rejects.toMatchObject({ + const clash = harness.engine.request(request, asking("agent-3", "myid")); + void clash.catch(() => undefined); + + // Settled at once: a request that was let in would wait for the one device instead. + await expect(settledOrPending(clash)).resolves.toMatchObject({ leaseId: "myid", name: "LeaseIdTakenError", }); @@ -3689,7 +3693,7 @@ describe("createLeasing a lease ID chosen by the requester", () => { it("lease.requested and lease.rejected requestSpec, and the status waiting spec, carry no leaseId", async () => { const harness = await withWaiting("agent-2", "myid"); - await harness.engine.request(request, asking("agent-3", "myid")).catch(() => undefined); + await settledOrPending(harness.engine.request(request, asking("agent-3", "myid"))); const requested = harness.bus.replay().filter((event) => event.event === "lease.requested"); const rejected = harness.bus.replay().filter((event) => event.event === "lease.rejected"); From 39bab6a3a636c3e7db7ed12fe59bc113de5ae295 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:15:05 +0200 Subject: [PATCH 7/9] refactor: keep the lease request option helpers under Fallow's complexity limit (#410) --- src/core/registry.ts | 4 ++-- src/http/app.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/core/registry.ts b/src/core/registry.ts index 517b81b2..3e74442b 100644 --- a/src/core/registry.ts +++ b/src/core/registry.ts @@ -1254,8 +1254,8 @@ function hasLeaseRequestFields(value: unknown): value is Record typeof value.id === "string" && typeof value.requesterId === "string" && typeof value.ownerId === "string" && - (value.idempotencyKey === undefined || typeof value.idempotencyKey === "string") && - (value.leaseId === undefined || typeof value.leaseId === "string") && + isOptionalString(value.idempotencyKey) && + isOptionalString(value.leaseId) && isDeviceRequest(value.request) && typeof value.createdAt === "number" && isLeaseRequestState(value.state) diff --git a/src/http/app.ts b/src/http/app.ts index 77c6a022..7cec56ae 100644 --- a/src/http/app.ts +++ b/src/http/app.ts @@ -171,6 +171,15 @@ function toLeaseRequestInput(body: z.infer): Leas ...(body.class === undefined ? {} : { class: body.class }), platform: body.platform, ...leaseRequestOptionFields(body), + ...leaseRequestNameFields(body), + }; +} + +/** The body's fields that name who owns the lease and what its ID is, not how to wait. */ +function leaseRequestNameFields(body: z.infer) { + return { + ...(body.owner === undefined ? {} : { owner: body.owner }), + ...(body.leaseId === undefined ? {} : { leaseId: body.leaseId }), }; } @@ -183,8 +192,6 @@ function leaseRequestOptionFields(body: z.infer) ...(body.allowDownload === undefined ? {} : { allowDownload: body.allowDownload }), ...(body.mode === undefined ? {} : { mode: body.mode }), ...(body.imageTag === undefined ? {} : { imageTag: body.imageTag }), - ...(body.owner === undefined ? {} : { owner: body.owner }), - ...(body.leaseId === undefined ? {} : { leaseId: body.leaseId }), }; } From 4524004462af760f989bab01f0523f4af3da816d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:21:04 +0200 Subject: [PATCH 8/9] test: kill the surviving mutants of the lease ID paths (#410) 35 alive -> fewer; the rest are log text and omitted-versus-undefined spreads --- src/cli/index.test.ts | 4 +- src/contract/errors.test.ts | 15 +++++ src/core/registry.test.ts | 54 +++++++++++++++++ src/gateway/fleet-coordinator.test.ts | 83 ++++++++++++++++++-------- src/gateway/lease-index.test.ts | 12 ++++ src/http/app.test.ts | 47 +++++++++++++++ src/leasing/create-leasing.test.ts | 2 + src/leasing/lease-request-book.test.ts | 21 +++++++ 8 files changed, 212 insertions(+), 26 deletions(-) diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index ef5969b4..b41d0f00 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -3992,7 +3992,7 @@ describe("CLI: holder renew and release (ADR 0004 §2)", () => { connectAdmin: async () => fakeClient({ requestLease: (input) => { - requested.push((input as Record).leaseId); + requested.push("leaseId" in input ? input.leaseId : "absent"); return answer === "grant" ? Promise.resolve(detachedGrant) : Promise.reject( @@ -4012,7 +4012,7 @@ describe("CLI: holder renew and release (ADR 0004 §2)", () => { // The flag's value goes out as typed and an omitted flag sends none: the contract decides // what an ID may look like. - expect(requested).toEqual(["ad-7f3a", undefined, "ad-7f3a"]); + expect(requested).toEqual(["ad-7f3a", "absent", "ad-7f3a"]); expect(exitCode).toBe(13); expect(output.stderr).toContain('"code":"LEASE_ID_TAKEN"'); }); diff --git a/src/contract/errors.test.ts b/src/contract/errors.test.ts index c59742c8..c044c2e6 100644 --- a/src/contract/errors.test.ts +++ b/src/contract/errors.test.ts @@ -38,6 +38,21 @@ describe("SimlockError", () => { }, ); + it("narrows LEASE_ID_TAKEN's details to the ID, and answers it with exit 13 and HTTP 409 as a domain error", () => { + const error = fromWireError("LEASE_ID_TAKEN", "lease ID ad-7f3a is already in use", { + leaseId: "ad-7f3a", + }); + if (error.code !== "LEASE_ID_TAKEN") throw new Error("expected LEASE_ID_TAKEN"); + + expect(error.details.leaseId).toBe("ad-7f3a"); + expect(ERROR_TABLE.LEASE_ID_TAKEN).toEqual({ + cliExitCode: 13, + code: "LEASE_ID_TAKEN", + httpStatus: 409, + kind: "domain", + }); + }); + it("isSimlockError rejects a plain Error", () => { expect(isSimlockError(new Error("boom"))).toBe(false); }); diff --git a/src/core/registry.test.ts b/src/core/registry.test.ts index 8d70dc14..7ddca2d6 100644 --- a/src/core/registry.test.ts +++ b/src/core/registry.test.ts @@ -1663,6 +1663,60 @@ describe("Registry", () => { expect(reloaded.leaseRequests()[1]).not.toHaveProperty("leaseId"); }); + it("keeps idChosenByRequester true on the lease of a stored grant across a reload", async () => { + const clock = new FakeClock(1_000); + const options = { + clock, + eventBus: new EventBus(clock), + filesystem: new MemoryFilesystem(), + idGenerator: { generate: () => "x" }, + statePath, + }; + const registry = await Registry.load(options); + const device = await registry.registerDevice({ + driverData: {}, + driverDeviceId: "driver_device", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + await registry.createLeaseRequest({ + id: "req_1", + leaseId: "ad-7f3a", + ownerId: "agent-1", + request: { platform: "ios" }, + requesterId: "agent-1", + }); + await registry.createLease({ + deviceId: device.id, + leaseId: "ad-7f3a", + ownerId: "agent-1", + request: { + environment: {}, + id: "req_1", + timing: { + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReadyMs: 0, + estimatedReclaimMs: 0, + }, + }, + requesterId: "agent-1", + ttlDeadline: 2_000, + ttlMs: 60_000, + }); + + const reloaded = await Registry.load(options); + + expect(reloaded.leaseRequests()[0]?.grant?.lease).toMatchObject({ + id: "ad-7f3a", + idChosenByRequester: true, + }); + }); + it("loads the lease of a stored grant written before idChosenByRequester as one simlock named", async () => { const clock = new FakeClock(1_000); const filesystem = new MemoryFilesystem(); diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index 51736441..3f55ca77 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -3655,7 +3655,7 @@ describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { const grant = await coordinator.request(REQUEST, requestOptions()); - expect(forwardedLeaseId(client)).toBeUndefined(); + expect(client.lastRequestLeaseInput).not.toHaveProperty("leaseId"); expect(grant.lease.id).toBe("wrk_a.lse_1"); }); @@ -3689,7 +3689,11 @@ describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { await tick(); expect(secondState.state).toBe("rejected"); - expect(await refusal).toMatchObject({ code: "LEASE_ID_TAKEN", details: { leaseId: "myid" } }); + expect(await refusal).toMatchObject({ + code: "LEASE_ID_TAKEN", + details: { leaseId: "myid" }, + message: "lease ID myid is already in use", + }); expect(rejected).toEqual([ expect.objectContaining({ reason: "lease-id-taken", requester: "agent-2" }), ]); @@ -3790,30 +3794,61 @@ describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { }); }); - it("a grant for a bare ID the index maps to another worker is released on the new worker and answers LEASE_ID_TAKEN, and the first entry stays routed", async () => { - const { client, coordinator, leaseIndex } = oneWorker(); - client.requestLeaseQueue.push({ - beforeGrant: () => { - // While wrk_a's grant is on its way, wrk_b reports a lease with the same ID. - leaseIndex.rebuildFromWorker("wrk_b", [ - { - grantedAt: 1, - id: "myid", - idChosenByRequester: true, - ownerId: "agent-9", - requesterId: `${GATEWAY_PREFIX}agent-9`, - } as WorkerReportedLease, - ]); - }, - grant: chosenGrant("myid"), - kind: "grant", - }); + describe("a grant for a bare ID the index maps to another worker", () => { + /** wrk_a's grant of `myid` is on its way when wrk_b reports a lease with the same ID. */ + function grantRacingAReport(overrides: Parameters[0] = {}) { + const fleet = oneWorker(overrides); + fleet.client.requestLeaseQueue.push({ + beforeGrant: () => { + fleet.leaseIndex.rebuildFromWorker("wrk_b", [ + { + grantedAt: 1, + id: "myid", + idChosenByRequester: true, + ownerId: "agent-9", + requesterId: `${GATEWAY_PREFIX}agent-9`, + } as WorkerReportedLease, + ]); + }, + grant: chosenGrant("myid"), + kind: "grant", + }); + return fleet; + } - await expect(coordinator.request(REQUEST, chosen("myid"))).rejects.toMatchObject({ - code: "LEASE_ID_TAKEN", + it("is released on the new worker and answers LEASE_ID_TAKEN, and the first entry stays routed", async () => { + const { client, coordinator, eventBus, leaseIndex } = grantRacingAReport(); + const rejected: unknown[] = []; + eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + + await expect(coordinator.request(REQUEST, chosen("myid"))).rejects.toMatchObject({ + code: "LEASE_ID_TAKEN", + details: { leaseId: "myid" }, + }); + + expect(client.calls).toContain("lease.release:myid"); + expect(leaseIndex.resolve("myid")).toMatchObject({ workerId: "wrk_b" }); + expect(rejected).toEqual([ + expect.objectContaining({ reason: "lease-id-taken", requester: "agent-1" }), + ]); }); - expect(client.calls).toContain("lease.release:myid"); - expect(leaseIndex.resolve("myid")).toMatchObject({ workerId: "wrk_b" }); + it("logs a release the worker refuses, naming the worker and the lease, and still answers LEASE_ID_TAKEN", async () => { + const logger = new RecordingLogger(); + const { client, coordinator } = grantRacingAReport({ logger }); + client.releaseLeaseQueue.push({ error: new Error("worker said no"), kind: "error" }); + + await expect(coordinator.request(REQUEST, chosen("myid"))).rejects.toMatchObject({ + code: "LEASE_ID_TAKEN", + }); + await tick(); + + expect(logger.warnings).toEqual([ + expect.objectContaining({ + fields: expect.objectContaining({ workerId: "wrk_a", workerLeaseId: "myid" }), + message: "Failed to release a lease the gateway will not route", + }), + ]); + }); }); }); diff --git a/src/gateway/lease-index.test.ts b/src/gateway/lease-index.test.ts index a4823f65..4f7e06cc 100644 --- a/src/gateway/lease-index.test.ts +++ b/src/gateway/lease-index.test.ts @@ -339,6 +339,18 @@ describe("FleetLeaseIndex", () => { ]); }); + it("logs a flagged lease with an invalid ID once, not on every snapshot that repeats it", () => { + const logger = new RecordingLogger(); + const index = new FleetLeaseIndex(PREFIX, logger); + + index.rebuildFromWorker("wrk_1", [chosen("not.a-valid id")]); + index.rebuildFromWorker("wrk_1", [chosen("not.a-valid id")]); + index.rebuildFromWorker("wrk_1", [chosen("not.a-valid id")]); + + expect(logger.warnings).toHaveLength(1); + expect(index.all()).toHaveLength(1); + }); + it("when two workers report the same caller-chosen ID on rebuild, the first stays routed and a warning names both workers", () => { const logger = new RecordingLogger(); const index = new FleetLeaseIndex(PREFIX, logger); diff --git a/src/http/app.test.ts b/src/http/app.test.ts index 8c56c79d..63fc3b49 100644 --- a/src/http/app.test.ts +++ b/src/http/app.test.ts @@ -643,6 +643,53 @@ describe("POST /v1/lease-requests", () => { }); }); +describe("POST /v1/lease-requests leaseId", () => { + it("passes a caller-chosen leaseId onto the dispatch input as sent, and no leaseId key when the body names none", async () => { + const { app, dispatcher } = buildHarness(); + const withId = postLeaseRequest(app, { ...defaultBody, leaseId: "ad-7f3a" }); + const call = await waitForDispatch(dispatcher, "lease.request"); + expect(call.input).toMatchObject({ leaseId: "ad-7f3a" }); + call.resolve(makeGrant({ lease: { id: "ad-7f3a" } })); + await withId; + + const withoutId = postLeaseRequest(app, defaultBody, otherAgentAuth); + const second = await waitForDispatch(dispatcher, "lease.request", 1); + expect(second.input).not.toHaveProperty("leaseId"); + second.resolve(makeGrant({ lease: { id: "lse_2" } })); + await withoutId; + }); + + it("answers 400 BAD_REQUEST for a leaseId outside the pattern, without dispatching", async () => { + const { app, dispatcher } = buildHarness(); + + const response = await postLeaseRequest(app, { ...defaultBody, leaseId: "w1.myid" }); + + expect(response.status).toBe(400); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe("BAD_REQUEST"); + expect(dispatcher.calls).toEqual([]); + }); + + it("answers 409 LEASE_ID_TAKEN with the ID in the body when the dispatcher refuses it", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["lease.request"] = () => { + throw new DispatchError("LEASE_ID_TAKEN", "lease ID ad-7f3a is already in use", { + leaseId: "ad-7f3a", + }); + }; + + const response = await postLeaseRequest(app, { ...defaultBody, leaseId: "ad-7f3a" }); + + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: { + code: "LEASE_ID_TAKEN", + leaseId: "ad-7f3a", + message: "lease ID ad-7f3a is already in use", + }, + }); + }); +}); + describe("full lease-request lifecycle via GET / long-poll / SSE", () => { it("progresses queued -> booting -> granted, observable through GET", async () => { const { app, dispatcher } = buildHarness(); diff --git a/src/leasing/create-leasing.test.ts b/src/leasing/create-leasing.test.ts index 8d0f12bd..1679e756 100644 --- a/src/leasing/create-leasing.test.ts +++ b/src/leasing/create-leasing.test.ts @@ -3646,6 +3646,7 @@ describe("createLeasing a lease ID chosen by the requester", () => { await expect(harness.engine.request(request, asking("agent-2", "myid"))).rejects.toMatchObject({ leaseId: "myid", + message: "lease ID myid is already in use", name: "LeaseIdTakenError", }); expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual(["myid"]); @@ -3689,6 +3690,7 @@ describe("createLeasing a lease ID chosen by the requester", () => { requestSpec: request, }, ]); + expect(rejected.map((event) => event.module)).toEqual(["lease-acquisition-coordinator"]); }); it("lease.requested and lease.rejected requestSpec, and the status waiting spec, carry no leaseId", async () => { diff --git a/src/leasing/lease-request-book.test.ts b/src/leasing/lease-request-book.test.ts index d10c227f..86b9b706 100644 --- a/src/leasing/lease-request-book.test.ts +++ b/src/leasing/lease-request-book.test.ts @@ -513,6 +513,27 @@ describe("LeaseRequestBook", () => { expect(book.holdsLeaseId("waits")).toBe(false); }); + it("stores the leaseId a request sent beside it, and no leaseId key for a request that sent none", async () => { + const store = memoryStore(); + const book = bookOver(store); + await book.admit(request, { ownerId: "a", requesterId: "a", leaseId: "myid" }, () => + granted("myid"), + ); + await book.admit(request, { ownerId: "b", requesterId: "b" }, () => granted("lse_2")); + + const [withId, without] = store.leaseRequests(); + expect(withId).toMatchObject({ leaseId: "myid", request }); + expect(without).not.toHaveProperty("leaseId"); + }); + + it("names the lease ID in the message of an idempotency conflict over it", async () => { + const book = bookOver(memoryStore()); + await book.admit(request, { ...keyed, leaseId: "myid" }, () => granted("myid")); + await settled(); + + expect(() => book.replay(request, keyed)).toThrow(/different device request or lease ID/); + }); + it("keeps a settled record as it is when a second result arrives for it, and settles nothing for an unknown id", async () => { const store = memoryStore(); const created = await store.createLeaseRequest(newRequest("agent")); From 3bca6533163c79b2e7c9455fd588f5d7bd759558 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 22:26:20 +0200 Subject: [PATCH 9/9] fix: prove the lease ID paths the review found unproven, and correct the docs that described them (#410) Tests: 0 failing before the fixes (each new test goes red when its code is broken). --- docs/CLI.md | 12 +++---- docs/EVENTS.md | 2 +- docs/HTTP-API.md | 11 +++--- docs/internal/ARCHITECTURE.md | 6 ++-- docs/internal/EVENTS.md | 2 +- e2e/http-api.test.ts | 25 ++++++++++---- src/bus/index.ts | 3 +- src/contract/errors.ts | 3 +- src/core/registry.test.ts | 33 ++++++++++++++++++ src/gateway/fleet-coordinator.test.ts | 50 +++++++++++++++++++++++++-- src/gateway/fleet-coordinator.ts | 11 +++--- src/gateway/lease-index.ts | 6 ++-- src/http/app.test.ts | 3 ++ 13 files changed, 131 insertions(+), 36 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index f7596111..bcf6d609 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -799,12 +799,12 @@ The grant carries one additional block so you can see where it landed: {"lease":{"id":"3f81a2c4.lse_9f2c","worker":{"id":"3f81a2c4","label":"mac-studio-2"}}} ``` -A lease Simlock named has an id that names its worker (that is how renew, -release, and reads route), but it is **opaque** — do not parse it. A lease -you named with `--lease-id` keeps exactly that ID through a gateway, with no -worker in front of it; the gateway finds the worker from a table it keeps in -memory and rebuilds from its workers after a restart, so a renew or release -in the moment after a restart can answer `UNKNOWN_LEASE` until the worker has +A lease Simlock named has an id that names its worker, but it is **opaque** +— do not parse it. A lease you named with `--lease-id` keeps exactly that ID +through a gateway, with no worker in front of it. The gateway finds the +worker of every lease, either kind, from a table it keeps in memory and +rebuilds from its workers after a restart, so a renew or release in the +moment after a restart can answer `UNKNOWN_LEASE` until the worker has reported. `worker.label` is display-only. **`lease renew`, `release`, and lease reads are forwarded** to the worker diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 8de511fe..6821c381 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -15,7 +15,7 @@ through `simlock events` and `simlock events --follow`. | `lease.renewed` | lease id, new deadline | a `lease.renew` succeeded — whether it came from `simlock lease renew`, `POST /v1/leases/{id}/renew`, or the renew timer a running `simlock lease` / MCP session keeps over its own lease. There is one renew path and this is it | LeaseLifecycle | implemented | | `lease.released` | lease id, device id, reason (explicit/killed/device-lost), owner id | an explicit `lease.release` (which is what a `simlock lease` holder does on its way out), (killed) an operator `release --all` or `nuke`, or (device-lost) a leased device could not be recovered after it stopped running outside simlock, or a daemon start found its device not running, and the device of that lease was wiped and returned to the pool, left waiting in `reclaiming` on a platform the daemon could not list, or marked missing. Closing a connection is not a release and never emits this | LeaseLifecycle | implemented | | `lease.expired` | lease id, device id, owner id | the lease's deadline passed with no `lease.renew` behind it — the grant-time TTL, or the TTL of the last renew, simply ran out. This is the one way a lease ends without somebody asking, and the only bound on a holder that was killed outright | LeaseLifecycle | implemented | -| `lease.rejected` | request id, requester, request spec (as on `lease.requested`: a request that named a class or nothing has no model, and one that named an OS range carries it as typed), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused before it was stored (`killed`, `already-leased`, `lease-id-taken`) emits no `lease.requested`, and its `lease.rejected` carries the id it would have been stored under; on a gateway, `no-worker` is a request no worker that takes requests can serve, `NO_CAPACITY` at once, and `unresolvable-spec` is one no known worker has the platform, model, or runtime for, and such a request emits no `lease.queued`. A request a worker refused as unable to serve (`RUNTIME_MISSING`, `UNKNOWN_MODEL`, `NO_DRIVER`) while another worker was busy waits in the gateway queue, unless it is a `noWait` request, which is rejected at once with reason `no-wait` and emits no `lease.queued`; a request that did queue and then finds no worker left emits `unresolvable-spec` after its `lease.queued`, and one that never queued gets only the refusing worker's own `lease.rejected`; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again. The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter — one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | +| `lease.rejected` | request id, requester, request spec (as on `lease.requested`: a request that named a class or nothing has no model, and one that named an OS range carries it as typed), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused before it was stored (`killed`, `already-leased`, `lease-id-taken`) emits no `lease.requested`, and its `lease.rejected` carries the id it would have been stored under (on a gateway, `lease-id-taken` can also end a request that was stored, when a worker's grant carries an ID the gateway already routes elsewhere: that one has its `lease.requested`); on a gateway, `no-worker` is a request no worker that takes requests can serve, `NO_CAPACITY` at once, and `unresolvable-spec` is one no known worker has the platform, model, or runtime for, and such a request emits no `lease.queued`. A request a worker refused as unable to serve (`RUNTIME_MISSING`, `UNKNOWN_MODEL`, `NO_DRIVER`) while another worker was busy waits in the gateway queue, unless it is a `noWait` request, which is rejected at once with reason `no-wait` and emits no `lease.queued`; a request that did queue and then finds no worker left emits `unresolvable-spec` after its `lease.queued`, and one that never queued gets only the refusing worker's own `lease.rejected`; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again. The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter — one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | On a **gateway**, the first three of these are its own fleet queue's facts, emitted by `FleetLeaseCoordinator` and never by the worker whose device is diff --git a/docs/HTTP-API.md b/docs/HTTP-API.md index a4149182..14a2af17 100644 --- a/docs/HTTP-API.md +++ b/docs/HTTP-API.md @@ -603,16 +603,15 @@ never here: clients reach the device through the gateway, with [`POST /v1/leases/{id}/exec`](#post-v1leasesidexec). A lease whose ID Simlock generated has an `id` that names its worker -(`.`, split on the **first** `.`), which is -how a gateway routes renew, release, and reads. A worker id is a UUID, so a +(`.`, split on the **first** `.`). A worker id is a UUID, so a real id reads `3f81a2c4-9b7d-4e21-8a55-1c0e6f2d7b93.lse_9f2c`; the examples here and elsewhere in these docs abbreviate it to its first segment for legibility. A lease whose requester chose the ID (`leaseId` on [`POST /v1/lease-requests`](#post-v1lease-requests)) keeps exactly that ID, -with no worker in front of it: the gateway routes it from a table it keeps in -memory and rebuilds from its workers after a restart, so for a moment after a -restart, renew and release of such a lease can answer `404 UNKNOWN_LEASE` -until its worker has reported. **Treat the whole id as opaque** — pass it back +with no worker in front of it. A gateway routes every lease, generated or +chosen, from a table it keeps in memory and rebuilds from its workers after a +restart, so for a moment after a restart, renew and release of any lease can +answer `404 UNKNOWN_LEASE` until its worker has reported. **Treat the whole id as opaque** — pass it back verbatim in paths and bodies, and read `worker.id` when you want the machine. `dataPlane` is **reserved** and always `null` in this version: streaming a diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index 625a3cf5..32e936dc 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -274,8 +274,8 @@ event envelope an `id`, taking it to 10, and ADR 0009 makes `atRamBudget` a required capacity field, taking it to 11, and ADR 0015 §3 makes `modelClasses` a required catalog field, taking it to 12, and ADR 0015 §4 makes `classDefaults` one too, taking it to 13, and a device's `servesDefaultMode`, required too, takes it to 16 (ADR 0015 §1 and §2 took it to 14 and 15), and `config.get`'s required `warmPool.reserveRunning` takes it to 17, and a starting daemon's `status.get`, which -answers `daemon` and `host` only, takes it to 18, and `config.get`'s required `warmPool.targets` and `warmPool.maxConcurrentBoots` take it to 19, and `status.get`'s `warmPool` block takes it to 20. So the range both -sides advertise is `{min: 20, max: 20}`, an older client and a current daemon simply +answers `daemon` and `host` only, takes it to 18, and `config.get`'s required `warmPool.targets` and `warmPool.maxConcurrentBoots` take it to 19, and `status.get`'s `warmPool` block takes it to 20, and a lease request's optional `leaseId` takes it to 21. So the range both +sides advertise is `{min: 21, max: 21}`, an older client and a current daemon simply do not overlap, and `hello` fails with `PROTOCOL_VERSION_UNSUPPORTED` naming both ranges. The same negotiation runs over a worker's uplink, which is why a worker older than this shows up in a gateway's views as `incompatible` @@ -955,7 +955,7 @@ emits its own facts — `worker.connected`, `worker.disconnected`, ADR 0014 to `{min: 10, max: 10}`, because every event envelope has an `id`, and ADR 0009 to `{min: 11, max: 11}`, because `atRamBudget` is required, and ADR 0015 to `{min: 12, max: 12}`, because the catalog's `modelClasses` is required, then - to `{min: 13, max: 13}`, because its `classDefaults` is, and ADR 0009 to `{min: 16, max: 16}`, because a device's `servesDefaultMode` is, then to `{min: 17, max: 17}`, because `config.get`'s `warmPool.reserveRunning` is, and a starting `status.get` to `{min: 18, max: 18}`, because its `devices`, `leases`, `capacity` and `queueDepth` are optional, then to `{min: 19, max: 19}`, because `config.get`'s `warmPool.targets` and `warmPool.maxConcurrentBoots` are required, and to `{min: 20, max: 20}`, because `status.get` gains `warmPool`; a + to `{min: 13, max: 13}`, because its `classDefaults` is, and ADR 0009 to `{min: 16, max: 16}`, because a device's `servesDefaultMode` is, then to `{min: 17, max: 17}`, because `config.get`'s `warmPool.reserveRunning` is, and a starting `status.get` to `{min: 18, max: 18}`, because its `devices`, `leases`, `capacity` and `queueDepth` are optional, then to `{min: 19, max: 19}`, because `config.get`'s `warmPool.targets` and `warmPool.maxConcurrentBoots` are required, and to `{min: 20, max: 20}`, because `status.get` gains `warmPool`, and to `{min: 21, max: 21}`, because a lease request may carry `leaseId`; a worker on an older version is `incompatible` the same way. That is the ordinary upgrade path, not a failure mode: upgrade the worker. An incompatible worker is marked `incompatible` in its view with both ranges shown and is never dispatched to, and it is not diff --git a/docs/internal/EVENTS.md b/docs/internal/EVENTS.md index f4fb7cc1..a4427df8 100644 --- a/docs/internal/EVENTS.md +++ b/docs/internal/EVENTS.md @@ -46,7 +46,7 @@ in short: `subject.past-tense-fact`, emitted post-commit, facts not commands. | `lease.renewed` | lease id, new deadline | a `lease.renew` succeeded — whether it came from `simlock lease renew`, `POST /v1/leases/{id}/renew`, or the renew timer a running `simlock lease` / MCP session keeps over its own lease. There is one renew path and this is it | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.released` | lease id, device id, reason (explicit/killed/device-lost), owner id | an explicit `lease.release` (which is what a `simlock lease` holder does on its way out), (killed) an operator `release --all` or `nuke`, or (device-lost) a leased device could not be recovered after it stopped running outside simlock, or a daemon start found its device not running, and the device of that lease was wiped and returned to the pool, left waiting in `reclaiming` on a platform the daemon could not list, or marked missing. Closing a connection is not a release and never emits this | LeaseLifecycle | implemented (payload per ADR 0004 pending) | | `lease.expired` | lease id, device id, owner id | the lease's deadline passed with no `lease.renew` behind it — the grant-time TTL, or the TTL of the last renew, simply ran out. This is the one way a lease ends without somebody asking, and the only bound on a holder that was killed outright | LeaseLifecycle | implemented (payload per ADR 0004 pending) | -| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never stored and has no `lease.requested`; it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec` only if it had entered the gateway queue (its `lease.queued` needs a terminal fact); otherwise the worker's own `lease.rejected` is the terminal fact and the gateway emits none; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | +| `lease.rejected` | request id, requester (both required on every reason, additive, ADR 0016 §2), request spec (as on `lease.requested`; `full` replaced by `mode`, ADR 0007 §13; `model` optional and `class` added, ADR 0015 §9; `osVersion` may be a range as typed, ADR 0015 §2), reason (timeout/no-wait/unresolvable-spec/no-worker/already-leased/lease-id-taken/boot-timeout/killed/cancelled/daemon-restarted) | a request ended without a grant. A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never stored and has no `lease.requested` (except a gateway's `lease-id-taken` after a grant, below); it carries the id minted for it before the check, the one the stored request would have had; on a gateway (ADR 0009 §4, §8) `no-worker` is rows 1 and 5 of the fast-fail table (`NO_CAPACITY`: no worker takes requests, or none that does can serve it) and `unresolvable-spec` rows 2 to 4 (`NO_DRIVER`, `UNKNOWN_MODEL`, `RUNTIME_MISSING`), emitted in the dispatch walk before the request is queued, so a request rejected on arrival emits no `lease.queued`, and a waiting one is rejected when the views change (additive, events rule 6). A request a worker refused with one of those codes (ADR 0009 §5) and the table later ends with that stored refusal gets a gateway `unresolvable-spec` only if it had entered the gateway queue (its `lease.queued` needs a terminal fact); otherwise the worker's own `lease.rejected` is the terminal fact and the gateway emits none; `daemon-restarted` is a request still waiting when the daemon stopped, settled as failed when it starts again (#72; widens a published vocabulary, which events rule 6 allows as additive). The reason list can grow: a consumer must tolerate a reason it does not know; `cancelled` is an explicit single-request cancel (`cancelPending` on the leasing module, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue / LeaseStartup (worker) / FleetLeaseCoordinator (gateway) | implemented | On a **gateway**, these three are its own fleet queue's facts (ADR 0005 §11/§14), emitted by `FleetLeaseCoordinator` and never by the worker whose device is diff --git a/e2e/http-api.test.ts b/e2e/http-api.test.ts index dd56b885..43741982 100644 --- a/e2e/http-api.test.ts +++ b/e2e/http-api.test.ts @@ -513,12 +513,13 @@ describe("HTTP API", () => { driverScript: { ios: { availableOsVersions: ["18.4"], knownModels: ["iPhone 16"] } }, }); const baseUrl = `http://127.0.0.1:${port}`; - const tokens = await Promise.all( - [0, 1].map(async () => { - const result = await env.cli(["token", "create", "--role", "agent"]); - return `Bearer ${(result.json as { secret: string }).secret}`; - }), - ); + // One at a time: the token file is read, extended and rewritten per create, so two creates + // at once can lose a token (#424). + const tokens: string[] = []; + for (const _ of [0, 1]) { + const result = await env.cli(["token", "create", "--role", "agent"]); + tokens.push(`Bearer ${(result.json as { secret: string }).secret}`); + } await waitFor( async () => { try { @@ -538,6 +539,18 @@ describe("HTTP API", () => { const first = await post(tokens[0] ?? ""); expect(first.status).toBe(201); + // The 201 comes with the first progress push; wait for the grant so an active lease holds the ID. + const firstId = ((await first.json()) as { request: { id: string } }).request.id; + await waitFor( + async () => { + const response = await fetch(`${baseUrl}/v1/lease-requests/${firstId}?wait=10`, { + headers: { authorization: tokens[0] ?? "" }, + }); + const body = (await response.json()) as { request: { state: string } }; + return body.request.state === "granted"; + }, + { timeout: 30_000, label: "the first request is granted" }, + ); const second = await post(tokens[1] ?? ""); expect(second.status).toBe(409); expect(await second.json()).toMatchObject({ diff --git a/src/bus/index.ts b/src/bus/index.ts index 6b5bd239..e7250d6f 100644 --- a/src/bus/index.ts +++ b/src/bus/index.ts @@ -61,7 +61,8 @@ export interface EventMap { "lease.rejected": { /** The request's id. A request refused at admission (`killed`, `already-leased`, `lease-id-taken`) was never * stored and has no `lease.requested`, but it carries the id it would have been stored - * under. */ + * under. A gateway's `lease-id-taken` can also follow a grant (a worker's lease whose ID the + * gateway already routes elsewhere): that request was stored and has its `lease.requested`. */ readonly requestId: string; readonly requester: string; /** The request as it arrived, as on `lease.requested`: `class` when it named one, `model` diff --git a/src/contract/errors.ts b/src/contract/errors.ts index 5ec36812..4961aa6e 100644 --- a/src/contract/errors.ts +++ b/src/contract/errors.ts @@ -47,7 +47,8 @@ export interface ErrorDetailsMap { * stored request is untouched; a new request needs a new key. */ IDEMPOTENCY_CONFLICT: Record; /** `lease.request` named a `leaseId` an active lease or a waiting request already holds - * (ADR 0020). Nothing was stored. */ + * (ADR 0020). The daemon stores nothing for it; a gateway can also meet it after storing the + * request, when a worker's grant carries an ID the gateway already routes elsewhere. */ LEASE_ID_TAKEN: { readonly leaseId: string }; NO_DRIVER: { readonly platform: Platform }; RUNTIME_MISSING: { diff --git a/src/core/registry.test.ts b/src/core/registry.test.ts index 7ddca2d6..edb8edd4 100644 --- a/src/core/registry.test.ts +++ b/src/core/registry.test.ts @@ -1663,6 +1663,39 @@ describe("Registry", () => { expect(reloaded.leaseRequests()[1]).not.toHaveProperty("leaseId"); }); + it("drops a stored lease request whose leaseId is not a string and keeps the others", async () => { + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + const open = (id: string, extra: Record) => ({ + createdAt: 1_000, + id, + ownerId: "agent-1", + request: { platform: "ios" }, + requesterId: "agent-1", + state: "open", + ...extra, + }); + await filesystem.writeFileAtomic( + statePath, + JSON.stringify({ + devices: [], + leaseRequests: [open("req_bad", { leaseId: 7 }), open("req_ok", { leaseId: "ad-7f3a" })], + leases: [], + }), + ); + + const registry = await Registry.load({ + clock, + eventBus: new EventBus(clock), + filesystem, + idGenerator: { generate: () => "unexpected" }, + statePath, + }); + + expect(registry.leaseRequests().map((record) => record.id)).toEqual(["req_ok"]); + }); + it("keeps idChosenByRequester true on the lease of a stored grant across a reload", async () => { const clock = new FakeClock(1_000); const options = { diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index 3f55ca77..45c802f5 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -3660,19 +3660,63 @@ describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { }); it("through a gateway, renew and release by a caller-chosen ID reach the right worker", async () => { - const { client, coordinator, leaseIndex } = oneWorker(); - client.requestLeaseQueue.push({ grant: chosenGrant("ad-7f3a"), kind: "grant" }); + const { coordinator, directory, leaseIndex, workers } = harness(); + const clientA = new ScriptedWorkerClient(); + const clientB = new ScriptedWorkerClient(); + directory.add("wrk_a", clientA); + directory.add("wrk_b", clientB); + connectWorker(workers, "wrk_a"); + // wrk_b has more free slots, so the request goes there and not to the first worker. + connectWorker(workers, "wrk_b", { capacity: roomierIos() }); + clientB.requestLeaseQueue.push({ grant: chosenGrant("ad-7f3a"), kind: "grant" }); await coordinator.request(REQUEST, chosen("ad-7f3a")); + expect(leaseIndex.resolve("ad-7f3a")).toMatchObject({ workerId: "wrk_b" }); await coordinator.renew("ad-7f3a", undefined); await coordinator.release("ad-7f3a"); - expect(client.calls).toEqual( + expect(clientB.calls).toEqual( expect.arrayContaining(["lease.renew:ad-7f3a", "lease.release:ad-7f3a"]), ); + expect(clientA.calls.filter((call) => /^lease\.(renew|release)/.test(call))).toEqual([]); expect(leaseIndex.resolve("ad-7f3a")).toBeUndefined(); }); + it("when two workers report the same caller-chosen ID on rebuild, renew of that ID reaches the first worker", async () => { + const { coordinator, directory, leaseIndex, workers } = harness(); + const clientA = new ScriptedWorkerClient(); + const clientB = new ScriptedWorkerClient(); + directory.add("wrk_a", clientA); + directory.add("wrk_b", clientB); + connectWorker(workers, "wrk_a"); + connectWorker(workers, "wrk_b"); + const reported = (ownerId: string) => ({ + grantedAt: 1, + id: "myid", + idChosenByRequester: true, + ownerId, + requesterId: `${GATEWAY_PREFIX}${ownerId}`, + }); + + leaseIndex.rebuildFromWorker("wrk_a", [reported("agent-1")]); + leaseIndex.rebuildFromWorker("wrk_b", [reported("agent-2")]); + await coordinator.renew("myid", undefined); + + expect(clientA.calls).toContain("lease.renew:myid"); + expect(clientB.calls.filter((call) => call.startsWith("lease.renew"))).toEqual([]); + }); + + it("through a gateway, a request whose requester already holds a lease fails with REQUESTER_ALREADY_LEASED, not LEASE_ID_TAKEN, when it names that lease's own ID", async () => { + const { client, coordinator } = oneWorker(); + client.requestLeaseQueue.push({ grant: chosenGrant("myid"), kind: "grant" }); + await coordinator.request(REQUEST, chosen("myid")); + + const refusal = await coordinator.request(REQUEST, chosen("myid")).catch((e: unknown) => e); + + expect(refusal).toBeInstanceOf(RequesterAlreadyLeasedError); + expect((refusal as RequesterAlreadyLeasedError).existingLeaseId).toBe("myid"); + }); + it("through a gateway, a second request for an ID held by a gateway lease fails with LEASE_ID_TAKEN, emits lease.rejected with reason lease-id-taken, and is not forwarded", async () => { const { client, coordinator, eventBus } = oneWorker(); client.requestLeaseQueue.push({ grant: chosenGrant("myid"), kind: "grant" }); diff --git a/src/gateway/fleet-coordinator.ts b/src/gateway/fleet-coordinator.ts index b709c329..0d0bdc6e 100644 --- a/src/gateway/fleet-coordinator.ts +++ b/src/gateway/fleet-coordinator.ts @@ -876,8 +876,8 @@ export class FleetLeaseCoordinator { * * C1 (round 3 review): the two branches that leave `waiter` *terminal* -- `#settleGrant`, and * the catch block's own `queue.reject` -- also each call `#dispatch()` themselves, right after. - * `#staleView`'s own branches (the unreachable-target check above, and the catch's immediate - * `NO_CAPACITY`) do not need to: both already call a worker's `refresh()`, and a real snapshot + * `#staleView`'s own branches (the unreachable-target check above, the catch's immediate + * `NO_CAPACITY`, and the mismatched-grant retry) do not need to: all already call a worker's `refresh()`, and a real snapshot * landing for it fires `#onViewsChanged` -> `#dispatch()`. That pass offers the waiter to every * other worker; it offers it to a worker that answered `NO_CAPACITY` only once that worker's * view has changed (ADR 0009 §5), so a worker that keeps refusing is asked once per change of @@ -1061,9 +1061,10 @@ export class FleetLeaseCoordinator { } /** ADR §11: "an immediate `NO_CAPACITY` is the only answer that leaves it queued ... the - * request waits", because this is a stale view, not a real refusal. A `noWait` caller reaches - * this only from a target that turned out unreachable; a `noWait` caller refused with - * `NO_CAPACITY` is settled in `#attempt` instead (ADR 0009 §5). */ + * request waits", because this is a stale view, not a real refusal. A waiter reaches + * this from a target that turned out unreachable, and from a grant whose id was not the one + * asked for (`noWait` included, in both); a `noWait` caller refused with `NO_CAPACITY` is + * settled in `#attempt` instead (ADR 0009 §5). */ #staleView(waiter: FleetWaiter, workerId: string): void { this.#enqueue(waiter); this.#refreshView(workerId); diff --git a/src/gateway/lease-index.ts b/src/gateway/lease-index.ts index 9f91394a..aa6ea7a6 100644 --- a/src/gateway/lease-index.ts +++ b/src/gateway/lease-index.ts @@ -6,9 +6,9 @@ import { NoopLogger } from "../ports/index.js"; * `FleetLeaseIndex`: the gateway's own record of which leases *it* issued (ADR 0005 §14, §16, * §27a, §30; ADR 0020). Nothing here is persisted -- like every worker view, it is rebuilt from * what a worker reports (`rebuildFromWorker`) whenever the gateway can see it. A gateway restart - * loses nothing a worker restart would not also lose (Decision 5) for a generated id, which names - * its worker; a caller-chosen id (ADR 0020) is bare and routed by this index alone, so for the - * moment before its worker has reported, the gateway does not know it. + * loses nothing a worker restart would not also lose (Decision 5): every id, generated or + * caller-chosen (ADR 0020), is looked up in this one map, so for the moment before its worker + * has reported, the gateway does not know it. * * A generated gateway lease id is minted once, at grant, as `${workerId}.${workerLeaseId}` (§16) -- * a caller-chosen one is that id as sent, bare -- and never re-derived by splitting the string back diff --git a/src/http/app.test.ts b/src/http/app.test.ts index 63fc3b49..79293d71 100644 --- a/src/http/app.test.ts +++ b/src/http/app.test.ts @@ -661,6 +661,9 @@ describe("POST /v1/lease-requests leaseId", () => { it("answers 400 BAD_REQUEST for a leaseId outside the pattern, without dispatching", async () => { const { app, dispatcher } = buildHarness(); + // A handler takes the call without the fake's own input parse, so only the route's schema + // can keep a bad ID from reaching the dispatcher. + dispatcher.handlers["lease.request"] = () => makeGrant({ lease: { id: "w1.myid" } }); const response = await postLeaseRequest(app, { ...defaultBody, leaseId: "w1.myid" });