From fcaf681234b83c55556bd1a49236d75ca3dcf765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 11:52:00 +0100 Subject: [PATCH 01/11] fix(limrun): give one lease one owner for its instance and its teardown A lease's instance, keep-alive timer, network capture and app alias were recorded in four maps, none of which held anything until the instance create had answered. Two allocations of one lease therefore rented two instances, and a release or daemon close arriving mid-create found nothing to end and left an instance billed. One owner now carries the create and every resource beside it, is published in the same step that starts the create, and is dropped only once its one disposal confirms. A route that arrives late joins that disposal or is refused with what it came to, and an unconfirmed delete keeps the lease held so the next release retries it rather than reporting an instance released. The owner is published before its allocation can await anything, so a release that arrives at the create's first suspension point already owns the instance that create goes on renting, and a create refuses a profile flag this provider reads no field for as its first step rather than after it has billed an instance. An instance is named to its owner the moment Limrun answers for it, before any session is built on it, so a create that rented one and then failed still owes its end, and a delete Limrun refused is retried by the next route rather than forgotten. Both instance POSTs ask the SDK for a single attempt: the SDK's own retries would answer an ambiguous failure by renting a second instance. Every wait a teardown takes is now bounded, so nothing an SDK started without a deadline can hold a release open. A create, a destination tunnel still being opened, or a local teardown that outlives its budget is reported instead of waited on; the work is never cancelled and its owner still disposes what it makes, and the lease is given back with a warning naming what was left, because the instance itself is gone. Capture startup publishes its pending result, so two allocations that ask for traffic logs open one destination tunnel and a teardown finds the tunnel its open had not answered for. The shared host ADB endpoint is retired before teardown yields, so a command that arrives while cleanup is starting raises no tunnel, and one already being raised is ended where it answers instead of being published to nobody. --- packages/provider-limrun/src/android.test.ts | 158 +++- packages/provider-limrun/src/android.ts | 81 +- .../provider-limrun/src/capture-lifecycle.ts | 206 +++++ .../provider-limrun/src/client-options.ts | 7 + .../provider-limrun/src/lease-ownership.ts | 118 +++ packages/provider-limrun/src/lease-release.ts | 94 +++ .../provider-limrun/src/lifecycle-budget.ts | 55 ++ packages/provider-limrun/src/network-har.ts | 91 +- packages/provider-limrun/src/runtime.test.ts | 530 +++++++++++- packages/provider-limrun/src/runtime.ts | 792 ++++++++++++------ .../src/session-allocation.test.ts | 174 ++++ .../provider-limrun/src/session-allocation.ts | 159 ++-- src/__tests__/limrun-runtime.test.ts | 14 +- 13 files changed, 2104 insertions(+), 375 deletions(-) create mode 100644 packages/provider-limrun/src/capture-lifecycle.ts create mode 100644 packages/provider-limrun/src/lease-ownership.ts create mode 100644 packages/provider-limrun/src/lease-release.ts create mode 100644 packages/provider-limrun/src/lifecycle-budget.ts create mode 100644 packages/provider-limrun/src/session-allocation.test.ts diff --git a/packages/provider-limrun/src/android.test.ts b/packages/provider-limrun/src/android.test.ts index 08983d7ab3..95ebac68d6 100644 --- a/packages/provider-limrun/src/android.test.ts +++ b/packages/provider-limrun/src/android.test.ts @@ -1,15 +1,49 @@ -import { expect, test, vi } from 'vitest'; +import { beforeEach, expect, test, vi } from 'vitest'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import type { DeviceInfo } from '@agent-device/kernel/device'; import { androidAdbHostTarget, androidAdbInvocation, + androidAdbSerialTarget, serializeAndroidAdbInvocation, type AndroidAdbInvocation, } from '@agent-device/platform-android/mechanics'; -import { cleanupLimrunAndroidAdbTunnel, type LimrunAndroidSession } from './android.ts'; -import type { LimrunAdbCommandOptions, LimrunAdbProvider } from './runtime-dependencies.ts'; +import { + cleanupLimrunAndroidAdbTunnel, + createLimrunAndroidSession, + type LimrunAndroidSession, +} from './android.ts'; +import type { + LimrunAdbCommandOptions, + LimrunAdbProvider, + LimrunRuntimeDependencies, +} from './runtime-dependencies.ts'; const ok = { exitCode: 0, stdout: '', stderr: '' }; +const tunnels = vi.hoisted(() => ({ + /** + * Held by a case that has to keep a tunnel in flight, which is the window a session's cleanup has to + * cover: the endpoint the SDK is still raising can answer after that cleanup answered. + */ + gate: undefined as Promise | undefined, + start: vi.fn(async () => { + await tunnels.gate; + return { address: { address: '127.0.0.1', port: 62_001 }, close: tunnels.close }; + }), + close: vi.fn(), +})); + +vi.mock('@limrun/api/instance-client', () => ({ + createInstanceClient: vi.fn(async () => ({ startAdbTunnel: tunnels.start })), +})); + +beforeEach(() => { + tunnels.gate = undefined; + tunnels.start.mockClear(); + tunnels.close.mockClear(); +}); + test('cleanup asks the platform to address the disconnect as a server-level command and hands it over unchanged', async () => { const calls: Array<{ argv: string[]; options: LimrunAdbCommandOptions | undefined }> = []; const close = vi.fn(); @@ -48,3 +82,121 @@ test('cleanup asks the platform to address the disconnect as a server-level comm expect(session.adbTunnel).toBeUndefined(); expect(session.adbTunnelPromise).toBeUndefined(); }); + +/** + * Cleanup yields before it can look at the session's endpoint, so it has to retire that endpoint in the step + * that started it: a command arriving inside that window would otherwise raise a tunnel the cleanup had no way + * to see, which then answers into a session nothing owns any more. + */ +test('a command that arrives while cleanup is starting raises no tunnel', async () => { + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, androidDependencies()); + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + // The cleanup has not looked at the endpoint yet, and this command has no retired flag to stop it. + const command = session.adbProvider.exec(['shell', 'echo', 'again']); + + await expect(command).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + await cleanup; + + expect(tunnels.start).not.toHaveBeenCalled(); + expect(session.adbSerial).toBeUndefined(); + expect(session.adbTunnel).toBeUndefined(); +}); + +/** + * A tunnel already being raised when cleanup arrives answers after cleanup returned, and the endpoint is + * process-wide: the session has to end it where it lands rather than publish it to nobody. + */ +test('a tunnel still being raised when cleanup arrives is ended and its command refused', async () => { + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, androidDependencies()); + let reportTunnel!: () => void; + tunnels.gate = new Promise((resolve) => { + reportTunnel = resolve; + }); + try { + const command = session.adbProvider.exec(['shell', 'echo', 'held']); + await vi.waitFor(() => expect(tunnels.start).toHaveBeenCalledOnce()); + + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + reportTunnel(); + + await expect(command).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + await cleanup; + } finally { + tunnels.gate = undefined; + } + // The tunnel this session raised was closed, and nothing published it for a later command to use. + expect(tunnels.close).toHaveBeenCalledOnce(); + expect(session.adbSerial).toBeUndefined(); + expect(session.adbTunnel).toBeUndefined(); +}); + +/** + * The host ADB endpoint is process-wide, so an interactor kept from a cleaned-up session has to be refused + * rather than handed a fresh tunnel: nothing owns that tunnel any more, and it survives the daemon's work + * on the device. + */ +test('an interactor kept from a cleaned-up session is refused instead of raising a second tunnel', async () => { + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, androidDependencies()); + // The first command raises the tunnel this session owns. + await session.adbProvider.exec(['shell', 'echo', 'ready']); + expect(tunnels.start).toHaveBeenCalledOnce(); + + await cleanupLimrunAndroidAdbTunnel(session); + + await expect(session.adbProvider.exec(['shell', 'echo', 'again'])).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + expect(tunnels.start).toHaveBeenCalledOnce(); +}); + +const ANDROID_SESSION_OPTIONS = { + lease: { + leaseId: 'lease-android', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease, + instanceId: 'android-instance-1', + ownership: 'created' as const, + device: { + platform: 'android', + id: 'limrun:android:lease-android', + name: 'Android 14', + kind: 'emulator', + target: 'mobile', + booted: true, + } satisfies DeviceInfo, + apiUrl: 'https://android.example', + adbUrl: 'wss://adb.example', + token: 'instance-token', +}; + +function androidDependencies(): Pick { + return { + android: { + deviceAdbInvocation: (serial: string, command: readonly string[]) => + androidAdbInvocation(androidAdbSerialTarget(serial), command), + hostAdbInvocation: (command: readonly string[]) => + androidAdbInvocation(androidAdbHostTarget(), command), + adbError: async (message: string) => new Error(message), + createPortReverse: async () => ({ + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + list: async () => [], + }), + } as unknown as LimrunRuntimeDependencies['android'], + host: { runAdb: async () => ok } as unknown as LimrunRuntimeDependencies['host'], + }; +} diff --git a/packages/provider-limrun/src/android.ts b/packages/provider-limrun/src/android.ts index a30c243b1b..e7c221c7f7 100644 --- a/packages/provider-limrun/src/android.ts +++ b/packages/provider-limrun/src/android.ts @@ -42,6 +42,17 @@ type LimrunAndroidAdbSession = { adbTunnel?: LimrunAdbTunnel; adbSerial?: string; adbTunnelPromise?: Promise; + /** + * Set once teardown has begun for this session, before it yields. The host ADB endpoint is + * process-wide, so neither a caller that kept an interactor from this session nor a tunnel this session had + * already started may leave one no owner can tear down any more. + */ + adbTunnelRetired?: boolean; + /** + * The endpoint one teardown is ending, recorded so that teardown keeps its own way to reach the device + * whose reverse mappings it is clearing. No other route reads it. + */ + adbTeardownSerial?: string; readonly dependencies: Pick; }; @@ -150,21 +161,44 @@ export async function configureLimrunAndroidPortReverse( } export async function cleanupLimrunAndroidAdbTunnel(session: LimrunAndroidSession): Promise { - await session.adbTunnelPromise?.catch(() => {}); - const serial = session.adbSerial; - if (serial) { - await cleanupAndroidPortReverse(session); - await session.dependencies.host - .runAdb(session.dependencies.android.hostAdbInvocation(['disconnect', serial]), { - allowFailure: true, - timeoutMs: 10_000, - }) - .catch(() => {}); + // Retire first, before yielding: the host ADB endpoint is process-wide, so from this step on no retained + // interactor can raise a tunnel, and a tunnel already in flight ends where it lands. + session.adbTunnelRetired = true; + // Cleanup keeps the serial it finds, or the one the in-flight start publishes while this step waits on it: + // a start answering into a retired session records its endpoint here first, because this cleanup is the only + // route left that can reach it. That serial is this session's only access to the host once the endpoint is + // retired, and retirement still lets this step through — it has to clear the reverse mappings off the device + // it is about to disconnect. + const started = await session.adbTunnelPromise?.catch(() => undefined); + session.adbTunnelPromise = undefined; + const serial = session.adbSerial ?? started; + session.adbTeardownSerial = serial; + try { + if (serial) { + await cleanupAndroidPortReverse(session); + await session.dependencies.host + .runAdb(session.dependencies.android.hostAdbInvocation(['disconnect', serial]), { + allowFailure: true, + timeoutMs: 10_000, + }) + .catch(() => {}); + } + } finally { + session.adbTeardownSerial = undefined; + closeLimrunAndroidAdbTunnel(session); } - session.adbTunnel?.close(); +} + +/** + * Ends the tunnel this session holds, once. A tunnel that answered only after its session was retired + * lands here, because retirement alone cannot reach it; nothing else is left to end it. + */ +function closeLimrunAndroidAdbTunnel(session: LimrunAndroidSession): void { + const tunnel = session.adbTunnel; + if (tunnel === undefined) return; session.adbTunnel = undefined; session.adbSerial = undefined; - session.adbTunnelPromise = undefined; + tunnel.close(); } async function cleanupAndroidPortReverse(session: LimrunAndroidSession): Promise { @@ -236,6 +270,13 @@ async function requireSuccessfulLimrunAndroidAdb( } async function ensurePersistentAndroidAdbSerial(session: LimrunAndroidAdbSession): Promise { + // The teardown's own read comes first, and is the only one retirement lets through: clearing an endpoint's + // reverse mappings means asking the device this cleanup is about to disconnect what was mapped on it. + const teardownSerial = session.adbTeardownSerial; + if (teardownSerial !== undefined) return teardownSerial; + // The retired check comes before the serial fast path: a session whose cleanup answered while an + // earlier command was still in flight holds a serial nothing owns any more. + if (session.adbTunnelRetired) throw alreadyCleanedUp(session); if (session.adbSerial) return session.adbSerial; const pending = session.adbTunnelPromise ?? startAndroidAdbTunnel(session); session.adbTunnelPromise = pending; @@ -250,11 +291,27 @@ async function ensurePersistentAndroidAdbSerial(session: LimrunAndroidAdbSession async function startAndroidAdbTunnel(session: LimrunAndroidAdbSession): Promise { const tunnel = await session.client.startAdbTunnel(); const serial = `${tunnel.address.address}:${tunnel.address.port}`; + // Nothing between this call's start and the publication below lets a second tunnel be raised, so a session + // retired while this tunnel was being raised records it all the same: its cleanup is the one route that can + // still reach it, and it is awaiting this promise. Refusal alone would leave the endpoint on the host with + // nobody left to disconnect it. session.adbTunnel = tunnel; session.adbSerial = serial; + if (session.adbTunnelRetired) throw alreadyCleanedUp(session); return serial; } +function alreadyCleanedUp(session: LimrunAndroidAdbSession): AppError { + return new AppError( + 'SESSION_NOT_FOUND', + `Limrun ADB tunnel for instance '${session.instanceId}' was already cleaned up`, + { + reason: 'limrun_adb_tunnel_cleaned_up', + hint: `Lease '${session.lease.leaseId}' is closed. Reopen the lease instead of driving this interactor.`, + }, + ); +} + function tcpEndpoint(port: number): LimrunPortReverseEndpoint { if (!Number.isInteger(port) || port < 1 || port > 65_535) { throw new AppError('INVALID_ARGS', `Invalid Android tcp reverse port: ${port}`); diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts new file mode 100644 index 0000000000..7e3f99d78c --- /dev/null +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -0,0 +1,206 @@ +import type { + LimrunLeaseOwner, + LimrunOwnerCapture, + LimrunRuntimeSession, +} from './lease-ownership.ts'; +import type { NetworkCapture, SettledNetworkCapture } from './network-har.ts'; +import { awaitLimrunWithinBudget } from './lifecycle-budget.ts'; +import { errorMessage } from '@agent-device/kernel/errors'; + +/** + * What one lease's network capture goes through: starting it, ending it, and answering what `artifacts` can + * say about it. The owner is where a capture lives, including a start whose destination tunnel is still being + * opened, so the allocation that started one, a release that took the lease over, and an artifacts read all + * look at the same thing and none of them can end up owning a tunnel nobody can close. + */ + +/** + * How long a read or a teardown waits for a destination tunnel that is still being opened. The SDK gives this + * handshake fifteen seconds of its own. + */ +const CAPTURE_OPEN_BUDGET_MS = 20_000; + +/** Released captures stay listable; the oldest settled records are forgotten past this count. */ +const SETTLED_NETWORK_CAPTURE_LIMIT = 32; + +export type LimrunCaptureHost = Readonly<{ + /** Where each lease's HAR goes. Without it a capture is unavailable and nothing is opened. */ + harDir: string | undefined; + /** The live session of an owner, and nothing once a teardown has taken the lease. */ + liveSessionOf(owner: LimrunLeaseOwner): LimrunRuntimeSession | undefined; + /** The leases this runtime holds, which is what the record limit spares from forgetting. */ + heldOwners(): Iterable; +}>; + +export class LimrunCaptureLifecycle { + /** The capture records this lifecycle settled, which outlive their lease so `artifacts` can still name them. */ + private readonly settled = new Map(); + private readonly host: LimrunCaptureHost; + + constructor(host: LimrunCaptureHost) { + this.host = host; + } + + /** + * Starts this lease's capture, or joins the start its owner is already running. The owner records that start + * before anything is awaited, so a repeat allocation cannot open a second destination tunnel whose + * `already_active` refusal would be reported over the capture that is working, and a teardown that takes the + * lease meanwhile finds a tunnel that is not there yet and ends it where it lands. + */ + async start(owner: LimrunLeaseOwner, hostList: unknown): Promise { + const held = owner.capture; + if (held?.phase === 'starting') { + await held.opening; + return; + } + const harDir = this.host.harDir; + if (!harDir) { + owner.capture = { phase: 'unavailable', reason: 'capture-dir-unset' }; + return; + } + // Only a live lease can capture: a release or a shutdown that took this owner is not a caller a new + // destination tunnel could be handed to, and nothing here would own it afterwards. + const session = this.host.liveSessionOf(owner); + if (!session) return; + let reportOpened!: (capture: NetworkCapture) => void; + const opening = new Promise((opened) => { + reportOpened = opened; + }); + owner.capture = { phase: 'starting', opening }; + const hosts = hostsToCapture(hostList); + // Started after the publication above, and nothing between the two lets a second start through. A start + // that could not be attempted at all is reported the same way, so nothing waits on a tunnel that is never + // coming, and the outcome is published before the waiting allocation is answered. + const publish = (capture: NetworkCapture): void => { + // A teardown took the capture over, and what this tunnel came to is that teardown's to end: nothing + // here hands a destination tunnel to a lease that is on its way out. + const current = owner.capture; + if (current !== undefined && current.phase === 'starting' && current.opening === opening) { + owner.capture = capture; + } + reportOpened(capture); + }; + void openCapture(session.client, harDir, owner.lease.leaseId, hosts).then( + publish, + (error: unknown) => + publish({ + phase: 'unavailable', + reason: 'tunnel-start-failed', + message: errorMessage(error), + }), + ); + await opening; + } + + /** + * Takes this owner's capture off it and ends it, keeping the record it came to so `artifacts` can still + * answer for the HAR after the lease is gone. Whatever its phase, the capture leaves the owner here: this is + * the disposal route, and the one route that owns a tunnel whose open had not answered yet. + */ + async end(owner: LimrunLeaseOwner): Promise { + const leaseId = owner.lease.leaseId; + const state = owner.capture; + if (state === undefined) return this.settled.get(leaseId); + owner.capture = undefined; + if (state.phase !== 'starting') return await endCapture(leaseId, state, this.record); + // A destination tunnel still being opened is waited for within a budget. A tunnel that answers later is + // written and ended by the continuation below all the same, so it is never left without an owner. + const opened = await awaitLimrunWithinBudget(state.opening, CAPTURE_OPEN_BUDGET_MS); + if (opened.ok) return await endCapture(leaseId, opened.value, this.record); + void state.opening.then(async (late) => { + await endCapture(leaseId, late, this.record); + }); + return this.settled.get(leaseId); + } + + /** + * What `artifacts` answers for one lease: its capture written now while the lease is held, and the record a + * release settled after the lease is gone. A lease this lifecycle never held answers nothing. + */ + async list( + leaseId: string, + held: LimrunOwnerCapture | undefined, + ): Promise { + if (held === undefined) return this.settled.get(leaseId); + if (held.phase !== 'starting') { + const { writeCaptureRecord } = await import('./network-har.ts'); + return await writeCaptureRecord(held); + } + // A capture whose tunnel is still being opened has no traffic to report yet; that tunnel stays this + // lease's own to end, which is its release's route, not this read. + const opened = await awaitLimrunWithinBudget(held.opening, CAPTURE_OPEN_BUDGET_MS); + if (!opened.ok) return undefined; + const { writeCaptureRecord } = await import('./network-har.ts'); + return await writeCaptureRecord(opened.value); + } + + /** Keeps one released capture's record, forgetting the oldest once the record limit is passed. */ + private readonly record = ( + leaseId: string, + capture: SettledNetworkCapture, + ): SettledNetworkCapture => { + // Re-insertion keeps the records in settle order, so the oldest released one is the first to go. + this.settled.delete(leaseId); + this.settled.set(leaseId, capture); + this.forgetRecordsOverLimit(); + return capture; + }; + + /** + * Forgets the oldest released capture once every record this lifecycle can answer for passes the limit. A + * lease that is still held keeps its own record: what is still being captured is not the oldest thing to + * throw away. + */ + private forgetRecordsOverLimit(): void { + while (this.settled.size + this.heldCaptureCount() > SETTLED_NETWORK_CAPTURE_LIMIT) { + const held = new Set([...this.host.heldOwners()].map((owner) => owner.lease.leaseId)); + const released = [...this.settled.keys()].find((leaseId) => !held.has(leaseId)); + if (released === undefined) return; + this.settled.delete(released); + } + } + + private heldCaptureCount(): number { + let count = 0; + for (const owner of this.host.heldOwners()) count += owner.capture === undefined ? 0 : 1; + return count; + } +} + +/** Opens one capture's destination tunnel, or says why it could not. */ +async function openCapture( + client: LimrunRuntimeSession['client'], + harDir: string, + leaseId: string, + hosts: readonly string[], +): Promise { + const { limrunHarPath, startLimrunHarCapture } = await import('./network-har.ts'); + const result = await startLimrunHarCapture(client, hosts.length > 0 ? hosts : undefined); + return result.ok + ? { phase: 'capturing', harPath: limrunHarPath(harDir, leaseId), capture: result.capture } + : { phase: 'unavailable', reason: result.reason, message: result.message }; +} + +/** + * Ends one capture: writes its HAR, closes its tunnel, and keeps the record listable. The record is written + * before the tunnel is closed, because its failure counts are read from the capture that is about to go. + */ +async function endCapture( + leaseId: string, + capture: NetworkCapture, + record: (leaseId: string, capture: SettledNetworkCapture) => SettledNetworkCapture, +): Promise { + const { writeCaptureRecord } = await import('./network-har.ts'); + const settled = await writeCaptureRecord(capture); + if (capture.phase === 'capturing') capture.capture.close(); + return record(leaseId, settled); +} + +function hostsToCapture(hostList: unknown): string[] { + return typeof hostList === 'string' + ? hostList + .split(',') + .map((host) => host.trim()) + .filter(Boolean) + : []; +} diff --git a/packages/provider-limrun/src/client-options.ts b/packages/provider-limrun/src/client-options.ts index b424486e1b..1de4d47db1 100644 --- a/packages/provider-limrun/src/client-options.ts +++ b/packages/provider-limrun/src/client-options.ts @@ -8,6 +8,13 @@ export function limrunUserAgent(clientVersion: string): string { return `${LIMRUN_CLIENT_ID}/${clientVersion}`; } +/** + * Asks the SDK for one attempt at a request whose failure this runtime cannot tell apart from Limrun having + * done the work: the SDK retries a lost answer by POSTing again, and a repeated instance create rents a + * second instance. The SDK defaults to two retries, so a request that must not repeat carries this. + */ +export const LIMRUN_SINGLE_ATTEMPT = Object.freeze({ maxRetries: 0 }); + /** The organization client, or a typed refusal for an operation that needs the API key. */ export function requireLimrunOrgClient( limrun: Limrun | undefined, diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts new file mode 100644 index 0000000000..388adedc05 --- /dev/null +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -0,0 +1,118 @@ +import type { DeviceLease } from '@agent-device/contracts/device'; +import type { LimrunPlatform } from './device.ts'; +import type { LimrunAndroidSession } from './android.ts'; +import type { LimrunIosSession } from './ios.ts'; +import type { NetworkCapture, SettledNetworkCapture } from './network-har.ts'; + +/** + * One lease's resources in this runtime, and the phase it is in: the create that is making its instance + * (`allocating`), the instance that create made (`live`), or the one disposal that is ending it + * (`releasing`). This is the only place a lease's instance, keep-alive timer, network capture and app alias + * are recorded, so every route that can end one — release, shutdown, expired-lease recovery, and the + * allocation still waiting on the create — reads it here. + * + * The owner is published before its allocation can await anything, so a duplicate allocation joins that + * create instead of making a second instance, and a release or shutdown arriving while the create runs + * finds the instance it is on its way to making. `releasing` is entered in the same synchronous step that + * starts the disposal, and the owner is dropped only once that disposal has confirmed, so a lease is never + * between owners while one of its instances may be live: an allocation that arrives mid-release is refused, + * a second release joins the disposal in flight, and shutdown awaits it rather than starting another. + * + * Only `LimrunRuntimeImplementation` holds owners, and only its routes write them, so the fields stay + * mutable and no owner reaches a caller. + */ +export type LimrunLeaseOwner = { + readonly lease: DeviceLease; + /** Which platform's instance this lease's create is making, and which one its delete must name. */ + readonly platform: LimrunPlatform; + /** + * The create that is making this lease's instance; never rejects unobserved — the runtime reads it. It + * carries its own outcome, including a refusal to bill anything for this lease's profile flags. + */ + readonly creation: Promise; + state: LimrunLeaseState; + /** + * The instance this lease's create rented, recorded the moment Limrun answered, before any session was + * built on it. A create that could not finish still owes an end for it, and its disposal reads it here. + */ + rentedInstanceId: string | undefined; + /** The keep-alive interval for the live instance; started when it went live, cleared by its disposal. */ + keepAlive: NodeJS.Timeout | undefined; + /** What this lease has in capture terms, including a start whose tunnel is still being opened. */ + capture: LimrunOwnerCapture | undefined; + /** The uploaded app the instance was created with, and the identifier Limrun installed it under. */ + appAlias: LimrunAppAlias | undefined; + /** The instance Limrun has already confirmed deleted, so a retried disposal never bills a second delete. */ + deletedInstanceId: string | undefined; +}; + +/** What one lease has live in this runtime, whatever its platform. */ +export type LimrunRuntimeSession = LimrunIosSession | LimrunAndroidSession; + +/** Whichever route can end one lease: a release, a daemon close, or the create that failed. */ +export type LimrunTeardownRoute = 'release' | 'shutdown' | 'create'; + +/** + * What one lease's app alias maps: the uploaded asset's name, and what Limrun installed it under. */ +export type LimrunAppAlias = Readonly<{ assetName: string; installedAppId: string }>; + +/** + * What one lease has in capture terms: the start that is still opening its destination tunnel + * (`starting`), the capture that start came to, or what a settled capture was written as. `starting` is + * published before the start can answer, so a duplicate allocation joins it instead of opening a second + * tunnel, and a teardown finds a tunnel that is not there yet and ends it where it lands. + */ +export type LimrunOwnerCapture = + | Readonly<{ phase: 'starting'; readonly opening: Promise }> + | NetworkCapture; + +/** + * What ending one lease came to. `cleanup` answers whether this lease's instance is certainly gone: + * `confirmed` once Limrun deleted it, `attached` when the instance was never this runtime's to delete, + * `uncertain` when deleting it failed or is still owed, and `none` when the lease never had an instance of + * its own. + */ +export type LimrunDisposalOutcome = Readonly<{ + cleanup: 'confirmed' | 'attached' | 'uncertain' | 'none'; + /** The instance the lease held, known as soon as the create named it. */ + instanceId: string | undefined; + /** Why a disposal could not be confirmed; absent once the instance is gone. */ + failure: string | undefined; + /** The released capture's record, so a refusal can still carry the HAR that was written. */ + capture: SettledNetworkCapture | undefined; +}>; + +/** + * What one create amounted to: the session it made, or why it refused or failed. An instance the create + * rented before it failed is not carried here — the owner records it as soon as Limrun names it, which is + * earlier than any outcome could be reported. + */ +export type LimrunCreateOutcome = + | Readonly<{ + status: 'ready'; + session: LimrunRuntimeSession; + appAlias: LimrunAppAlias | undefined; + }> + | Readonly<{ status: 'failed'; error: unknown }>; + +type LimrunLeaseState = + | Readonly<{ phase: 'allocating' }> + | Readonly<{ phase: 'live'; session: LimrunRuntimeSession }> + | LimrunReleasingState; + +/** + * The state of a lease one teardown route owns. Only the take writes it, and only while that route's + * disposal runs: `settled` is what that disposal reported, and `instanceId` grows to name the instance the + * create announced while the teardown was already under way. + */ +type LimrunReleasingState = { + phase: 'releasing'; + /** Which route took the lease from its allocation; one route owns the owner until it confirms. */ + by: LimrunTeardownRoute; + /** The instance being ended, known as soon as the create named it. */ + instanceId: string | undefined; + /** The one disposal of this owner, joined by every caller that arrives while it runs. */ + readonly disposal: Promise; + /** What that disposal reported; missing while it runs, so a caller can tell gone from unseen. */ + settled: LimrunDisposalOutcome | undefined; +}; diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts new file mode 100644 index 0000000000..ac6fbd4715 --- /dev/null +++ b/packages/provider-limrun/src/lease-release.ts @@ -0,0 +1,94 @@ +import type Limrun from '@limrun/api'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend } from './device.ts'; +import type { LimrunDisposalOutcome } from './lease-ownership.ts'; + +/** + * The route that ends one lease: the release a caller asked for, or the recovery of a lease this runtime lost + * sight of. Both answer for the same thing — what the lease has running, and whether it stopped — and neither + * reports a lease as released on anything short of Limrun confirming the delete. + */ + +/** What a release answers for one lease: the instance it ended, with what it left and wrote. */ +type LimrunReleaseRecord = Record; + +/** + * Ends one lease from the disposal its runtime already started for it, or from what the lease's own labels + * name when the runtime no longer holds an owner for it. A delete Limrun did not confirm keeps the lease held: + * the release is refused, and the next release, recovery or shutdown retries the same instance rather than + * reporting an instance released. + * + * The disposal is handed in rather than started here because the caller took the lease in the same step the + * release was asked for, and no loading in between may let the allocation still waiting on its create answer + * first. + */ +export async function releaseLimrunLease( + limrun: Limrun | undefined, + lease: DeviceLease, + disposal: Promise | undefined, +): Promise { + // No disposal means this runtime holds nothing for the lease: it was never allocated here, or its disposal + // confirmed and gave it back. What the lease has under its labels is the recovery route's. + if (disposal === undefined) return await releaseLabeledInstances(limrun, lease); + const outcome = await disposal; + if (outcome.cleanup === 'uncertain') throw releaseLeftInstanceRunning(lease.leaseId, outcome); + if (outcome.instanceId === undefined) return undefined; + const { networkArtifacts } = await import('./network-har.ts'); + return { + limrunInstanceId: outcome.instanceId, + // The instance is gone, so the lease is released; what the release stopped waiting for is a warning the + // caller still has to see, because it is a host resource this process left behind. + ...(outcome.failure === undefined + ? {} + : { + warnings: [ + { + code: 'LIMRUN_TEARDOWN_INCOMPLETE', + message: outcome.failure, + limrunInstanceId: outcome.instanceId, + }, + ], + }), + ...(outcome.capture ? { cloudArtifacts: networkArtifacts(outcome.capture) } : {}), + }; +} + +/** Ends every instance a lease's own labels name, for a lease this runtime no longer holds an owner for. */ +async function releaseLabeledInstances( + limrun: Limrun | undefined, + lease: DeviceLease, +): Promise { + const platform = platformForLimrunLeaseBackend(lease.backend); + if (!platform || !limrun) return undefined; + const labelSelector = `provider=${LIMRUN_PROVIDER},leaseId=${lease.leaseId}`; + const instances = + platform === 'ios' + ? await limrun.iosInstances.list({ labelSelector }) + : await limrun.androidInstances.list({ labelSelector }); + const instanceIds = instances.getPaginatedItems().map((instance) => instance.metadata.id); + for (const instanceId of instanceIds) { + if (platform === 'ios') { + await limrun.iosInstances.delete(instanceId); + } else { + await limrun.androidInstances.delete(instanceId); + } + } + if (instanceIds.length === 0) return undefined; + return { limrunInstanceId: instanceIds[0], limrunInstanceCount: instanceIds.length }; +} + +/** What a release reports when Limrun did not confirm the delete, so the lease is still held. */ +function releaseLeftInstanceRunning(leaseId: string, outcome: LimrunDisposalOutcome): never { + throw new AppError( + 'COMMAND_FAILED', + `Limrun instance for lease '${leaseId}' was not confirmed deleted`, + { + reason: 'limrun_instance_delete_unconfirmed', + retriable: true, + ...(outcome.instanceId === undefined ? {} : { limrunInstanceId: outcome.instanceId }), + ...(outcome.failure === undefined ? {} : { cause: outcome.failure }), + hint: `Release lease '${leaseId}' again, or ask Limrun to delete instance '${outcome.instanceId ?? leaseId}'.`, + }, + ); +} diff --git a/packages/provider-limrun/src/lifecycle-budget.ts b/packages/provider-limrun/src/lifecycle-budget.ts new file mode 100644 index 0000000000..84b3ea9115 --- /dev/null +++ b/packages/provider-limrun/src/lifecycle-budget.ts @@ -0,0 +1,55 @@ +/** + * One bounded wait, for the work a lease's lifecycle awaits but does not own: a create that may be + * retrying, a session connection still handshaking, an ADB `connect` the SDK started without a deadline, + * or a capture still opening its destination tunnel. Every one of these can answer later than a release + * or a daemon close may wait, and none of them may keep an instance billed while its caller is unresolved. + * + * Expiry is a reported outcome rather than a thrown one, and the work being waited on is never cancelled: + * it runs to its end, and its owner disposes what it eventually makes. + */ + +/** How the wait ended, kept so a caller states which work was still outstanding rather than guessing. */ +export type LimrunBudgetOutcome = + | Readonly<{ ok: true; value: T }> + | Readonly<{ ok: false; expired: true }>; + +/** + * Waits for `work` within `budgetMs`, unref'd so a daemon close is never held open by the timer alone. + * A work item that was too slow to be awaited is kept observed, so the outcome it reports later cannot + * surface as an unhandled rejection. + */ +export async function awaitLimrunWithinBudget( + work: Promise, + budgetMs: number, +): Promise> { + void Promise.resolve(work).catch(() => undefined); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + work.then((value): LimrunBudgetOutcome => ({ ok: true, value })), + new Promise>((expired) => { + timer = setTimeout(() => expired({ ok: false, expired: true }), budgetMs); + timer.unref?.(); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +/** + * Why an outcome is not confirmed: the work this runtime stopped waiting for. It is reported rather than + * dropped, because whoever reads it has to say whether the resource may still be running. + */ +export function workOutstandingFailure(work: string, budgetMs: number): string { + return `${work} had not answered within ${String(budgetMs / 1000)} s`; +} + +/** Joins why a delete failed with the local work that outlived its budget, delete first. */ +export function joinOutcomeFailures( + deleteFailure: string | undefined, + outstanding: string | undefined, +): string | undefined { + if (deleteFailure === undefined) return outstanding; + return outstanding === undefined ? deleteFailure : `${deleteFailure}; ${outstanding}`; +} diff --git a/packages/provider-limrun/src/network-har.ts b/packages/provider-limrun/src/network-har.ts index cde0178378..dbe4d9b5a6 100644 --- a/packages/provider-limrun/src/network-har.ts +++ b/packages/provider-limrun/src/network-har.ts @@ -4,8 +4,10 @@ import { DestinationTunnelSessionError } from '@limrun/api/destination-tunnel-di import type { DestinationTunnelHAREntry } from '@limrun/api/destination-tunnel-har'; import type { DestinationTunnelInspectionEvent } from '@limrun/api/destination-tunnel-inspection'; import type { DestinationTcpTunnel } from '@limrun/api/destination-tunnel-dialer'; +import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import fs from 'node:fs'; import path from 'node:path'; +import { LIMRUN_PROVIDER } from './device.ts'; /** * The Limrun network capture: one destination tunnel with inspection. Completed requests stay in @@ -148,7 +150,7 @@ export async function startLimrunHarCapture( } /** Writes the entries as a HAR 1.2 file that only the daemon's user can read. */ -export async function writeLimrunHar( +async function writeLimrunHar( harPath: string, entries: readonly DestinationTunnelHAREntry[], ): Promise { @@ -182,3 +184,90 @@ function defaultPort(url: URL | null): number | undefined { if (url.port) return Number(url.port); return url.protocol === 'https:' ? 443 : url.protocol === 'http:' ? 80 : undefined; } + +/** What one lease's capture came to once it was written or refused. */ +export type SettledNetworkCapture = + | Readonly<{ phase: 'written'; harPath: string; failures: LimrunNetworkFailures }> + | Readonly<{ + phase: 'unavailable'; + reason: 'capture-dir-unset' | 'tunnel-in-use' | 'tunnel-start-failed' | 'har-write-failed'; + message?: string; + }>; + +/** What one lease has in capture terms: what it is capturing now, or what its capture came to. */ +export type NetworkCapture = + | SettledNetworkCapture + | Readonly<{ phase: 'capturing'; harPath: string; capture: LimrunHarCapture }>; + +/** Where one lease's HAR goes, so the lease id is the only name a capture's file needs. */ +export function limrunHarPath(harDir: string, leaseId: string): string { + return path.join(harDir, `${leaseId}.har`); +} + +/** + * Writes one capture's HAR and answers with the record it came to. The capture itself is left alone, so an + * artifacts read can snapshot a live lease's traffic without ending the capture. A settled capture is + * already its own record. + */ +export async function writeCaptureRecord(capture: NetworkCapture): Promise { + if (capture.phase !== 'capturing') return capture; + const failures = await capture.capture.failures(); + try { + await writeLimrunHar(capture.harPath, capture.capture.entries()); + return { phase: 'written', harPath: capture.harPath, failures }; + } catch (error) { + return { + phase: 'unavailable', + reason: 'har-write-failed', + message: error instanceof Error ? error.message : String(error), + }; + } +} + +const NETWORK_ARTIFACT_NAME = 'Network traffic (HAR on the daemon host)'; + +/** What `artifacts` answers for one lease's capture record. */ +export function networkArtifacts(capture: SettledNetworkCapture): CloudArtifactsResult { + if (capture.phase === 'unavailable') { + return { + provider: LIMRUN_PROVIDER, + status: 'unavailable', + message: `Network capture unavailable (${capture.reason})${capture.message ? `: ${capture.message}` : ''}`, + cloudArtifacts: [ + { + provider: LIMRUN_PROVIDER, + kind: 'network', + name: NETWORK_ARTIFACT_NAME, + availability: 'unavailable', + metadata: { reason: capture.reason }, + }, + ], + }; + } + const { count, last, lastDialFailure, truncated } = capture.failures; + const summary = [ + `Network capture failures: ${count}`, + ...(last + ? [`last ${last.kind}${last.host ? ` ${last.host}${last.port ? `:${last.port}` : ''}` : ''}`] + : []), + ...(lastDialFailure ? [`last dial failure ${lastDialFailure}`] : []), + ...(truncated > 0 ? [`${truncated} oldest requests left the HAR`] : []), + ].join('; '); + return { + provider: LIMRUN_PROVIDER, + status: 'ready', + ...(count > 0 || lastDialFailure || truncated > 0 ? { message: summary } : {}), + cloudArtifacts: [ + { + provider: LIMRUN_PROVIDER, + kind: 'network', + name: NETWORK_ARTIFACT_NAME, + url: capture.harPath, + extension: 'har', + contentType: 'application/json', + availability: 'ready', + metadata: { failures: capture.failures }, + }, + ], + }; +} diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index 108d4946f8..48347b4acb 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -15,7 +15,11 @@ import { import { createLimrunRuntime } from './runtime.ts'; import { unusedLimrunRuntimeHost } from './runtime.fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; -import type { LimrunAdbExecutor, LimrunRuntimeDependencies } from './runtime-dependencies.ts'; +import type { + LimrunAdbExecutor, + LimrunAdbProvider, + LimrunRuntimeDependencies, +} from './runtime-dependencies.ts'; /** * The provider's platform module is the daemon's real load path: `createLimrunRuntime(..., @@ -32,50 +36,506 @@ const state = vi.hoisted(() => ({ tunnelClose: vi.fn(), captureTunnelClose: vi.fn(), onInspectionEvent: undefined as ((event: unknown) => void) | undefined, + androidCreate: vi.fn(async () => ({ + metadata: { id: 'android-instance-1' }, + status: { + token: 'instance-token', + apiUrl: 'https://android.example', + adbWebSocketUrl: 'wss://adb.example', + }, + })), + androidDelete: vi.fn(async (_instanceId: string) => { + if (state.deleteFails) throw new Error('Limrun refused to delete the instance'); + }), + /** Made to reject by a case whose instance delete is refused, the one failure a lease may not swallow. */ + deleteFails: false, + keepAlive: vi.fn(), + /** Instance-client connections the provider made, so one lease can be shown to make one. */ + connectCount: 0, + /** + * Held by a case that has to keep a create in flight after its instance was rented, which is the window + * in which a release or a daemon close has to find the instance. + */ + createGate: undefined as Promise | undefined, + /** + * Held by a case that has to keep a session's destination tunnel or ADB tunnel in flight, which is the + * window a release has to cover for a tunnel that has not been opened yet. + */ + tunnelGate: undefined as Promise | undefined, + /** Made to throw by a case whose session connection fails after its instance was rented. */ + connectFails: false, + /** Destination tunnels and ADB tunnels this runtime asked the SDK to open. */ + captureStarts: 0, + adbTunnelStarts: 0, + /** + * The ADB provider last handed to an interactor, which is the handle a caller keeps for a session whose + * lease has since ended. + */ + heldAdbProvider: undefined as LimrunAdbProvider | undefined, })); vi.mock('@limrun/api', () => ({ default: class MockLimrun { readonly iosInstances = { create: vi.fn(), list: vi.fn(), delete: vi.fn() }; readonly androidInstances = { - create: vi.fn(async () => ({ - metadata: { id: 'android-instance-1' }, - status: { - token: 'instance-token', - apiUrl: 'https://android.example', - adbWebSocketUrl: 'wss://adb.example', - }, - })), - list: vi.fn(), - delete: vi.fn(async () => undefined), + create: async () => { + const instance = await state.androidCreate(); + await state.createGate; + return instance; + }, + // Expired-lease recovery finds nothing under this lease's labels, so a release of a lease the + // runtime lost answers nothing rather than reaching an unstubbed SDK call. + list: vi.fn(async () => ({ getPaginatedItems: () => [] })), + delete: state.androidDelete, }; readonly assets = { getOrUpload: vi.fn(), list: vi.fn() }; }, })); vi.mock('@limrun/api/instance-client', () => ({ - createInstanceClient: vi.fn(async () => ({ - disconnect: state.disconnect, - keepAlive: vi.fn(), - openUrl: vi.fn(), - sendAsset: vi.fn(), - pressKey: vi.fn(), - setText: vi.fn(), - tap: vi.fn(), - startAdbTunnel: vi.fn(async () => ({ - address: { address: '127.0.0.1', port: 62_001 }, - close: state.tunnelClose, - })), - startTunnel: vi.fn(async (options: { onInspectionEvent: (event: unknown) => void }) => { - state.onInspectionEvent = options.onInspectionEvent; - return { tunnelId: 't1', selectors: [], close: state.captureTunnelClose }; - }), - getTunnelStatus: vi.fn(async () => ({ - lastDialFailure: { tunnelId: 't1', reason: 'dns_not_found' }, - })), - })), + createInstanceClient: vi.fn(async () => { + state.connectCount += 1; + if (state.connectFails) throw new Error('instance connection refused'); + return { + disconnect: state.disconnect, + keepAlive: state.keepAlive, + openUrl: vi.fn(), + sendAsset: vi.fn(), + pressKey: vi.fn(), + setText: vi.fn(), + tap: vi.fn(), + startAdbTunnel: vi.fn(async () => { + state.adbTunnelStarts += 1; + await state.tunnelGate; + return { address: { address: '127.0.0.1', port: 62_001 }, close: state.tunnelClose }; + }), + startTunnel: vi.fn(async (options: { onInspectionEvent: (event: unknown) => void }) => { + state.captureStarts += 1; + state.onInspectionEvent = options.onInspectionEvent; + await state.tunnelGate; + return { tunnelId: 't1', selectors: [], close: state.captureTunnelClose }; + }), + getTunnelStatus: vi.fn(async () => ({ + lastDialFailure: { tunnelId: 't1', reason: 'dns_not_found' }, + })), + }; + }), })); +/** + * One lease has one set of resources, however many routes arrive at it: a leased instance cannot be rented + * twice, left untracked by a release or a daemon close that raced its create, or kept alive after it is + * gone. The mock `state` is shared by this file, so each case measures from its own baseline. + */ + +test('two allocations of one lease rent one instance and start one keep-alive timer', async () => { + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const { connects, creates, deletes, disconnects } = limrunActivity(); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', keepAlive: true }, + testDependencies(), + ); + let started: ReturnType | undefined; + try { + // The second call arrives while the first one's create is still running. + const first = runtime.leaseLifecycle.allocate?.(androidLease('lease-race')); + const second = runtime.leaseLifecycle.allocate?.(androidLease('lease-race')); + await waitForCreate(creates + 1); + started = vi.spyOn(globalThis, 'setInterval'); + reportCreate(); + + assert.equal((await first)?.limrunInstanceId, 'android-instance-1'); + assert.equal((await second)?.limrunInstanceId, 'android-instance-1'); + assert.equal(state.connectCount - connects, 1); + assert.equal(started.mock.calls.length, 1); + } finally { + started?.mockRestore(); + await runtime.shutdown(); + state.createGate = undefined; + } + // The second allocation is not an orphaned session behind the first: one create, one connection, one delete. + const activity = limrunActivity(); + assert.equal(activity.creates - creates, 1); + assert.equal(activity.connects - connects, 1); + assert.deepEqual( + state.androidDelete.mock.calls.slice(deletes).map(([instanceId]) => instanceId), + ['android-instance-1'], + ); + assert.equal(activity.disconnects - disconnects, 1); +}); + +test('a release that arrives before the create answered ends the instance that create made', async () => { + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const { creates, deletes, disconnects } = limrunActivity(); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', keepAlive: true }, + testDependencies(), + ); + const lease = androidLease('lease-race-release'); + let started: ReturnType | undefined; + try { + const allocating = runtime.leaseLifecycle.allocate?.(lease); + const refused = refusalOf(allocating); + await waitForCreate(creates + 1); + started = vi.spyOn(globalThis, 'setInterval'); + + const released = runtime.leaseLifecycle.release?.(lease); + reportCreate(); + + // The release cannot answer before the instance it is waiting for exists, so it names what it ended. + assert.deepEqual(await released, { limrunInstanceId: 'android-instance-1' }); + // The allocation whose instance was taken from it is refused rather than handed a dead session. + const error = await refused; + assert.ok( + error instanceof AppError && + error.details?.reason === 'limrun_lease_released_mid_allocation' && + error.details.limrunInstanceId === 'android-instance-1', + `expected the allocation to be refused for the release, got ${String(error)}`, + ); + assert.equal(started.mock.calls.length, 0); + assert.deepEqual( + state.androidDelete.mock.calls.slice(deletes).map(([instanceId]) => instanceId), + ['android-instance-1'], + ); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + } finally { + started?.mockRestore(); + await runtime.shutdown(); + state.createGate = undefined; + } + // One instance, so one delete: the allocation's own refusal does not end a second one. + assert.equal(state.androidDelete.mock.calls.length - deletes, 1); +}); + +test('a shutdown while the create runs ends the instance that create made', async () => { + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const { creates, deletes } = limrunActivity(); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', keepAlive: true }, + testDependencies(), + ); + let started: ReturnType | undefined; + try { + const allocating = runtime.leaseLifecycle.allocate?.(androidLease('lease-race-shutdown')); + const refused = refusalOf(allocating); + await waitForCreate(creates + 1); + started = vi.spyOn(globalThis, 'setInterval'); + + const closing = runtime.shutdown(); + reportCreate(); + await closing; + + const error = await refused; + assert.ok( + error instanceof AppError && error.details?.reason === 'limrun_lease_released_mid_allocation', + `expected the allocation to be refused for the shutdown, got ${String(error)}`, + ); + assert.equal(started.mock.calls.length, 0); + // Nothing may allocate into a runtime that was told to end every instance it holds. + await assert.rejects( + async () => await runtime.leaseLifecycle.allocate?.(androidLease('lease-after-shutdown')), + (error) => error instanceof AppError && error.details?.reason === 'limrun_runtime_shut_down', + ); + assert.deepEqual( + state.androidDelete.mock.calls.slice(deletes).map(([instanceId]) => instanceId), + ['android-instance-1'], + ); + } finally { + started?.mockRestore(); + state.createGate = undefined; + } +}); + +/** Lets an allocation reach the instance create, which is a few module loads away, without clock waits. */ +async function waitForCreate(calls: number): Promise { + await vi.waitFor(() => assert.equal(state.androidCreate.mock.calls.length, calls)); +} + +test('a release that arrives before the allocation yields ends the instance its create rents', async () => { + const { creates, deletes } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-release-before-yield'); + try { + // Neither call is awaited between: the release arrives at the allocation's first suspension point. + const allocating = runtime.leaseLifecycle.allocate?.(lease); + const refused = refusalOf(allocating); + const released = runtime.leaseLifecycle.release?.(lease); + + // The release had an owner to take, so it answers for the instance that create went on renting. + assert.deepEqual(await released, { limrunInstanceId: 'android-instance-1' }); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + const error = await refused; + assert.ok( + error instanceof AppError && + error.details?.reason === 'limrun_lease_released_mid_allocation' && + error.details.limrunInstanceId === 'android-instance-1', + `expected the allocation to be refused for the release, got ${String(error)}`, + ); + } finally { + await runtime.shutdown(); + } + // One instance was rented and one was ended; the refused allocation did not rent another. + assert.equal(state.androidCreate.mock.calls.length - creates, 1); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); +}); + +test('the create of a new lease refuses a profile flag it is given no field for', async () => { + const { creates } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + try { + await assert.rejects( + async () => + await runtime.leaseLifecycle.allocate?.(androidLease('lease-refused-flag'), { + flags: { providerNetworkLogs: true, providerOsVersion: '15' }, + }), + (error) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); + } finally { + await runtime.shutdown(); + } + // Nothing was billed before the refused flag was answered for. + assert.equal(state.androidCreate.mock.calls.length - creates, 0); +}); + +test('a create that failed after renting keeps its unconfirmed delete for the next route', async () => { + const { deletes } = limrunActivity(); + state.connectFails = true; + state.deleteFails = true; + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + try { + // The instance was rented before the session connection failed, and Limrun refused this delete. + const error = await refusalOf( + runtime.leaseLifecycle.allocate?.(androidLease('lease-rented-then-failed')), + ); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.ok( + error instanceof AppError && + error.details?.reason === 'limrun_instance_delete_unconfirmed' && + error.details.limrunInstanceId === 'android-instance-1', + `expected the allocation to report the unended instance, got ${String(error)}`, + ); + + // The lease is still held, so the daemon close retries the delete it could not confirm. + state.deleteFails = false; + await runtime.shutdown(); + } finally { + state.connectFails = false; + state.deleteFails = false; + await runtime.shutdown(); + } + // The retried delete ended the same instance the create rented, rather than forgetting it was owed. + assert.deepEqual(deletedInstances(deletes), ['android-instance-1', 'android-instance-1']); +}); + +test('two allocations that ask for traffic logs open one destination tunnel', async () => { + const { captures, captureCloses } = limrunActivity(); + let reportTunnel!: () => void; + state.tunnelGate = new Promise((resolve) => { + reportTunnel = resolve; + }); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', networkHarDir: mkdtempForTestSync('limrun-har-') }, + testDependencies(), + ); + const lease = androidLease('lease-capture-race'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + // Both repeat allocations ask for the capture while the first one's tunnel is still being opened. + const first = runtime.leaseLifecycle.allocate?.(lease, { + flags: { providerNetworkLogs: true }, + }); + const second = runtime.leaseLifecycle.allocate?.(lease, { + flags: { providerNetworkLogs: true }, + }); + await waitForNewOpen(captures, () => state.captureStarts); + reportTunnel(); + await Promise.all([first, second]); + + // The second allocation joined the start in flight rather than opening a tunnel whose `already_active` + // refusal would be reported over the capture that is working. + assert.equal(state.captureStarts - captures, 1); + } finally { + state.tunnelGate = undefined; + await runtime.shutdown(); + } + assert.equal(state.captureTunnelClose.mock.calls.length - captureCloses, 1); +}); + +test('a release that arrives while a tunnel is opening ends the tunnel that answers late', async () => { + const { captures, captureCloses, deletes } = limrunActivity(); + let reportTunnel!: () => void; + state.tunnelGate = new Promise((resolve) => { + reportTunnel = resolve; + }); + const harDir = mkdtempForTestSync('limrun-har-'); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', networkHarDir: harDir }, + testDependencies(), + ); + const lease = androidLease('lease-capture-teardown'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + + // The capture's destination tunnel is still being opened when the release arrives. + const capturing = refusalOf( + runtime.leaseLifecycle.allocate?.(lease, { flags: { providerNetworkLogs: true } }), + ); + await waitForNewOpen(captures, () => state.captureStarts); + const released = runtime.leaseLifecycle.release?.(lease); + reportTunnel(); + + const record = (await released) as Record; + assert.equal(record.limrunInstanceId, 'android-instance-1'); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + // The release wrote the capture's record as well, so `artifacts` can still name the HAR. + assert.equal( + (record.cloudArtifacts as { cloudArtifacts: { url?: string }[] }).cloudArtifacts[0]?.url, + path.join(harDir, `${lease.leaseId}.har`), + ); + const error = await capturing; + assert.ok( + error instanceof AppError && error.details?.reason === 'limrun_lease_released_mid_allocation', + `expected the capture allocation to be refused for the release, got ${String(error)}`, + ); + } finally { + state.tunnelGate = undefined; + await runtime.shutdown(); + } + // The release owned a tunnel whose open had not answered, and ended it where it landed. + assert.equal(state.captureTunnelClose.mock.calls.length - captureCloses, 1); +}); + +test('a release stops waiting on a stuck ADB tunnel, deletes the instance and says what it left', async () => { + const { deletes, disconnects, adbTunnels, tunnelCloses } = limrunActivity(); + let reportTunnel!: () => void; + state.tunnelGate = new Promise((resolve) => { + reportTunnel = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-stuck-tunnel'); + let refusedCommand: Promise | undefined; + vi.useFakeTimers(); + try { + const device = await allocateAndroidDevice(runtime, lease.leaseId); + if (!runtime.getInteractor(device)) throw new Error('Limrun runtime must return an interactor'); + // A retained interactor raises this session's ADB tunnel, an `adb connect` the SDK starts without a + // deadline; it is still stuck when the release arrives. + const command = heldAdbCommand(['shell', 'echo', 'held']); + refusedCommand = refusalOf(command); + await vi.advanceTimersByTimeAsync(0); + assert.equal(state.adbTunnelStarts - adbTunnels, 1); + + const released = runtime.leaseLifecycle.release?.(lease); + await vi.advanceTimersByTimeAsync(0); + // The delete does not queue behind the stuck tunnel: it was asked for while that tunnel was still open. + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + + // The release's own budget for the local teardown elapses; the delete it ran beside is already answered. + await vi.advanceTimersByTimeAsync(LOCAL_TEARDOWN_WAIT_MS); + const [warning] = warningsOf(await released); + assert.equal(warning?.code, 'LIMRUN_TEARDOWN_INCOMPLETE'); + // The report names the work this route stopped waiting for, and the instance it left a host endpoint on. + assert.match(String(warning?.message), /Local teardown/); + assert.equal(warning?.limrunInstanceId, 'android-instance-1'); + // The lease is given back: the stuck tunnel is reported, not held over the next allocation. + await runtime.leaseLifecycle.allocate?.(lease); + + reportTunnel(); + await vi.advanceTimersByTimeAsync(0); + const refused = await refusedCommand; + assert.ok( + refused instanceof AppError && refused.details?.reason === 'limrun_adb_tunnel_cleaned_up', + `expected the retained command to be refused, got ${String(refused)}`, + ); + } finally { + reportTunnel(); + vi.useRealTimers(); + state.tunnelGate = undefined; + await runtime.shutdown(); + await (refusedCommand ?? Promise.resolve()); + } + // The tunnel that answered into a retired session was closed where it landed, and no second one was raised. + assert.equal(state.adbTunnelStarts - adbTunnels, 1); + assert.equal(state.tunnelClose.mock.calls.length - tunnelCloses, 1); +}); + +/** The retained interactor's own ADB handle, driven outside any request this runtime is serving. */ +function heldAdbCommand(args: readonly string[]): Promise { + const provider = state.heldAdbProvider; + if (!provider) throw new Error('Expected the runtime to hand an interactor an ADB provider'); + return provider.exec(args); +} + +function warningsOf(record: Record | undefined): WarningRecord[] { + return Array.isArray(record?.warnings) ? (record.warnings as WarningRecord[]) : []; +} + +type WarningRecord = Readonly<{ + code?: string; + message?: string; + limrunInstanceId?: string; +}>; + +/** + * Longer than the release's own budget for the local resources it started beside the delete, which is what + * the case waits out. The delete is asked for independently of it. + */ +const LOCAL_TEARDOWN_WAIT_MS = 60_000; + +/** + * Why one racing call refused, captured from the moment it starts. Two routes can answer in either order, + * and the one that answers first must not leave the other's refusal unobserved. + */ +function refusalOf(call: Promise | undefined): Promise { + return (call ?? Promise.resolve()).then( + () => undefined, + (error: unknown) => error, + ); +} + +/** Where the shared instance-control mocks stand, so a case can measure only what it caused. */ +function limrunActivity(): { + creates: number; + deletes: number; + disconnects: number; + connects: number; + captures: number; + captureCloses: number; + adbTunnels: number; + tunnelCloses: number; +} { + return { + creates: state.androidCreate.mock.calls.length, + deletes: state.androidDelete.mock.calls.length, + disconnects: state.disconnect.mock.calls.length, + connects: state.connectCount, + captures: state.captureStarts, + captureCloses: state.captureTunnelClose.mock.calls.length, + adbTunnels: state.adbTunnelStarts, + tunnelCloses: state.tunnelClose.mock.calls.length, + }; +} + +/** The instance ids the runtime asked Limrun to delete after a case's own baseline. */ +function deletedInstances(from: number): string[] { + return state.androidDelete.mock.calls.slice(from).map(([instanceId]) => instanceId as string); +} + +/** Lets a case reach a resource a call is on its way to opening, without clock waits. */ +async function waitForNewOpen(starts: number, opened: () => number): Promise { + await vi.waitFor(() => assert.ok(opened() > starts, 'expected a new tunnel to be attempted')); +} + test('the platform module answers app state from the live Android provider session', async () => { const registration = createLimrunRuntime( { apiKey: 'lim_test_key', runtimeInstance: 'test-instance' }, @@ -187,6 +647,7 @@ test('the platform module surfaces an aborted request inside the app-state reade }); test('a network capture writes a private HAR with its failures on artifacts and at release', async () => { + const captureCloses = state.captureTunnelClose.mock.calls.length; const harDir = mkdtempForTestSync('limrun-har-'); const runtime = createLimrunRuntime( { apiKey: 'lim_test_key', networkHarDir: harDir }, @@ -228,7 +689,7 @@ test('a network capture writes a private HAR with its failures on artifacts and cloudArtifacts?: { cloudArtifacts: { url?: string }[]; message?: string }; }; assert.equal(released.cloudArtifacts?.cloudArtifacts[0]?.url, harPath); - assert.equal(state.captureTunnelClose.mock.calls.length, 1); + assert.equal(state.captureTunnelClose.mock.calls.length - captureCloses, 1); assert.match( String(released.cloudArtifacts?.message), /last request-failed missing\.example\.com:443/, @@ -339,7 +800,10 @@ function testDependencies(): LimrunRuntimeDependencies { return { clientVersion: 'test-version', android: { - createInteractor: () => ({}) as Interactor, + createInteractor: (_device, adb) => { + state.heldAdbProvider = adb; + return {} as Interactor; + }, createPortReverse: async () => ({ ensure: async () => {}, remove: async () => {}, diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index 7a0f704f72..949916f9a7 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -13,7 +13,7 @@ import type { ProviderPortReverseOptions, } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, errorMessage } from '@agent-device/kernel/errors'; import { cleanupLimrunAndroidAdbTunnel, configureLimrunAndroidPortReverse, @@ -30,12 +30,7 @@ import { platformForLimrunLeaseBackend, type LimrunPlatform, } from './device.ts'; -import { - createLimrunIosInteractor, - createLimrunIosSession, - installLimrunIosApp, - type LimrunIosSession, -} from './ios.ts'; +import { createLimrunIosInteractor, createLimrunIosSession, installLimrunIosApp } from './ios.ts'; import { createLimrunDeviceSession, type LimrunDeviceSession } from './device-session.ts'; import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; import type { @@ -55,13 +50,34 @@ import type { CloudArtifactProvider, CloudArtifactsResult, } from '@agent-device/contracts/observability'; -import type { LimrunHarCapture, LimrunNetworkFailures } from './network-har.ts'; -import path from 'node:path'; - -type LimrunRuntimeSession = LimrunIosSession | LimrunAndroidSession; +import type { + LimrunAppAlias, + LimrunCreateOutcome, + LimrunDisposalOutcome, + LimrunLeaseOwner, + LimrunRuntimeSession, + LimrunTeardownRoute, +} from './lease-ownership.ts'; +import type { LimrunCaptureLifecycle } from './capture-lifecycle.ts'; const KEEP_ALIVE_INTERVAL_MS = 30_000; +/** + * How long a teardown waits for the create to say what it rented. The create itself keeps Limrun's own + * timeouts: this bounds only the teardown's wait, so an instance is never held open by a create that + * stopped answering, and the owner is kept for the next route to end what that create rents. + */ +const CREATE_ANSWER_BUDGET_MS = 120_000; + +/** + * How long a teardown waits for the local resources it started beside the delete: the instance connection + * and the Android ADB endpoint, whose `adb connect` the SDK runs without a deadline. + */ +const LOCAL_TEARDOWN_BUDGET_MS = 45_000; + +/** How long a teardown waits for a capture's HAR record, which is reporting and never a reason to hold a lease. */ +const CAPTURE_RECORD_BUDGET_MS = 15_000; + export type LimrunRuntimeOptions = { /** Organization API key. It creates and deletes instances for platforms without `instances`. */ apiKey?: string; @@ -114,13 +130,19 @@ export function createLimrunRuntime( class LimrunRuntimeImplementation implements ProviderDeviceRuntime { private readonly limrun: Limrun | undefined; - private readonly sessions = new Map(); - private readonly keepAliveTimers = new Map(); - private readonly networkCaptures = new Map(); - private readonly appAliases = new Map< - string, - Readonly<{ assetName: string; installedAppId: string }> - >(); + /** + * A lease has an owner from the step it is allocated until the one disposal of it confirms. Allocate, + * join, release, recovery, shutdown and every reader of a live session go through this one map, so an + * in-flight create is never invisible to the route that has to end what it rents. + */ + private readonly leases = new Map(); + /** + * What each lease's network capture goes through, including the records a release settled. Absent until a + * lease asks for a capture, which is what keeps the capture's module out of a runtime that captures nothing. + */ + private capturesInstance?: LimrunCaptureLifecycle; + /** Whether this runtime has shut down; it then holds nothing, so it must create nothing. */ + private shutDown = false; private readonly options: LimrunRuntimeOptions; private readonly dependencies: LimrunRuntimeDependencies; readonly provider = LIMRUN_PROVIDER; @@ -153,7 +175,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { readonly deviceInventoryProvider: DeviceInventoryProvider = async (request) => { if (request.leaseProvider !== this.provider || !request.leaseId) return null; - const session = this.sessions.get(request.leaseId); + const session = this.liveSessionForLease(request.leaseId); if (!session) return null; if (request.platform && request.platform !== session.platform) return []; return [session.device]; @@ -173,6 +195,24 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { : undefined; } + /** + * The capture lifecycle, made the first time a lease asks for a capture. The records a release settled live + * there, so an artifacts read for a lease this runtime no longer holds goes through the same instance that + * settled them, and a runtime that never captures anything never evaluates that module. + */ + private async captures(): Promise { + if (this.capturesInstance === undefined) { + const { LimrunCaptureLifecycle } = await import('./capture-lifecycle.ts'); + // Two allocations can be inside this load at once, and the settled records have to be one lifecycle's. + this.capturesInstance ??= new LimrunCaptureLifecycle({ + harDir: this.options.networkHarDir, + liveSessionOf: (owner) => this.liveSession(owner), + heldOwners: () => this.leases.values(), + }); + } + return this.capturesInstance; + } + ownsDevice(device: DeviceInfo): boolean { return parseLimrunDeviceId(device.id) !== undefined; } @@ -246,11 +286,25 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { return portReverseResult(options); } + /** + * Ends every instance this runtime holds or is on its way to holding, and says so when it could not. + * Each lease is taken, or joined, in one synchronous step, so a create that answers afterwards is + * disposed by the one disposal of that lease and by no other route, and a release already running is + * awaited rather than raced. + */ async shutdown(): Promise { - const sessions = [...this.sessions.values()]; - await Promise.allSettled(sessions.map(async (session) => await this.terminateSession(session))); - this.sessions.clear(); - this.appAliases.clear(); + this.shutDown = true; + // Snapshot before anything is awaited: `allocate` refuses once this flag is set and publishes an + // owner only where none exists, and nothing between the flag and this loop awaits. So no create can + // start here, and every create already running has an owner for this loop to take. + const outcomes = await Promise.all( + [...this.leases.values()].map(async (owner) => await this.take(owner, 'shutdown')), + ); + // An instance Limrun would not give back is still billed, and this process will not stop it. The + // plugin's close is where an operator learns what a daemon close left running, so the instance and + // the provider's refusal travel on the error rather than being dropped. + const unended = outcomes.filter((outcome) => outcome.cleanup === 'uncertain'); + if (unended.length > 0) throw cleanupLeftInstancesRunning(unended); } private async allocate( @@ -260,88 +314,394 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (lease.leaseProvider !== this.provider) return undefined; const platform = platformForLimrunLeaseBackend(lease.backend); if (!platform) return undefined; - const { rejectRefusedLimrunProfileFields } = await import('./session-allocation.ts'); - // Before the reuse below, since a repeat allocation of a live lease carries flags of its own, and - // ahead of attach as well as create, since an attached instance reads none of them either. - rejectRefusedLimrunProfileFields(context); - const existing = this.sessions.get(lease.leaseId); - const session = - existing ?? - (await this.attachSession(platform, lease)) ?? - (await this.createSession(platform, lease, context)); - if (!existing) { - this.sessions.set(lease.leaseId, session); - if (this.options.keepAlive) this.startKeepAlive(lease.leaseId, session); + // Joining a lease somebody else allocated, so this call answers for flags of its own, and it settles + // them before it reads any state. Nothing else awaits on this route: the lookup below is the last one. + if (this.leases.has(lease.leaseId)) await this.rejectRefusedProfileFields(context); + // Published without awaiting anything, so a lease is never unowned while its allocation is at its first + // await: a release or a shutdown arriving there finds this owner and takes it, and the create that owner + // stands on checks this call's own profile flags before it bills anything. + let owner = this.leases.get(lease.leaseId); + if (owner === undefined) { + // Nothing between this lookup and the publication below awaits, so two allocations of one lease can + // never both pass it, and this runtime can be shut down only before or after this step. + if (this.shutDown) throw allocationRefusedAfterShutdown(lease); + owner = this.beginAllocation(lease, platform, context); + } + if (owner.state.phase === 'allocating') { + const create = await owner.creation; + if (create.status === 'failed') { + // A create that failed has no release coming for it, so this call is the only route left to end an + // instance it made before failing. What it came to outranks why the create did not finish: an + // instance Limrun is still billing is the caller's problem first. + const outcome = await this.settleFailedCreate(owner); + if (outcome?.cleanup === 'uncertain') throw createLeftInstanceRunning(lease, outcome); + throw create.error; + } + this.goLive(owner, create.session, create.appAlias); } - // A repeat allocation can ask for the capture, or retry one that could not start. - if ( - context?.flags?.providerNetworkLogs === true && - this.networkCaptures.get(lease.leaseId)?.phase !== 'capturing' - ) { - await this.startNetworkCapture(lease.leaseId, session, context.flags.providerNetworkHosts); + // A repeat allocation can ask for the capture, or retry one that could not start. A lease a teardown has + // already taken starts nothing: `captures.start` reads the owner's own live session, and what this lease + // has is that teardown's to end. + if (context?.flags?.providerNetworkLogs === true && owner.capture?.phase !== 'capturing') { + const captures = await this.captures(); + await captures.start(owner, context.flags.providerNetworkHosts); } + // A release or shutdown that took the lease while this call waited owns the instance the create made, and + // answers for it. This caller is refused for that teardown, with what it came to, rather than being handed + // an instance nothing will release for it. The refusal waits for the teardown to answer, so this caller + // never reports before the instance it was promised is gone. + const taken = owner.state; + if (taken.phase === 'releasing') { + const outcome = await taken.disposal; + throw allocationRefusedForTake(lease, taken.by, outcome); + } + const session = this.requireLiveSession(owner); return { limrunInstanceId: session.instanceId, device: session.device }; } - private async startNetworkCapture( - leaseId: string, + /** The profile flags this provider reads nothing for, refused for a lease this runtime already holds. */ + private async rejectRefusedProfileFields(context?: LeaseLifecycleContext): Promise { + // Loaded here rather than eagerly so this package entry's declared closure stays unchanged. The create + // of a new allocation refuses the same flags as its first step, inside the owner it published. + const { rejectRefusedLimrunProfileFields } = await import('./session-allocation.ts'); + rejectRefusedLimrunProfileFields(context); + } + + /** + * Publishes the owner one allocation's create is standing on, in the same step that starts the create + * and before it can answer, so a duplicate allocation joins that create instead of making a second + * instance, and a release or shutdown arriving mid-create has somewhere to find the instance that create + * is on its way to making. The owner records that instance the moment Limrun names it, so whichever route + * takes the lease has what it has to end. + */ + private beginAllocation( + lease: DeviceLease, + platform: LimrunPlatform, + context?: LeaseLifecycleContext, + ): LimrunLeaseOwner { + let report!: (create: LimrunCreateOutcome) => void; + const creation = new Promise((settled) => { + report = settled; + }); + const owner: LimrunLeaseOwner = { + lease, + platform, + creation, + state: { phase: 'allocating' }, + rentedInstanceId: undefined, + keepAlive: undefined, + capture: undefined, + appAlias: undefined, + deletedInstanceId: undefined, + }; + this.leases.set(lease.leaseId, owner); + // The create is started after the publication above, so its answer can never arrive without an owner + // beside it. Whichever route takes the lease in the meantime ends what this answer names. + void this.createForLease(owner, context).then(report); + return owner; + } + + /** + * Attaches or makes this lease's instance and answers with the whole outcome rather than with a promise + * that resolves or rejects: a refused caller still has to say whether anything was left running, and the + * owner is where an instance this create rented gets recorded. Nothing here ends an instance, and nothing + * retries a create: a create whose answer was lost may have made an instance anyway, and that instance is + * reported and ended, never replaced by a second one. + */ + private async createForLease( + owner: LimrunLeaseOwner, + context?: LeaseLifecycleContext, + ): Promise { + try { + const { + allocateLimrunAndroidSession, + allocateLimrunIosSession, + rejectRefusedLimrunProfileFields, + resolvePreinstalledAppId, + resolveRequestedLimrunAppAsset, + } = await import('./session-allocation.ts'); + // The first step of the create, so an allocation that published its owner before it could await + // anything still bills nothing for flags this provider reads no field for. + rejectRefusedLimrunProfileFields(context); + const platform = owner.platform; + // Attach first, and only ask for the organization client if there is nothing to attach: an attached + // instance needs no API key, and is not created with an app asset of ours. + const attached = await this.attachSession(platform, owner.lease); + if (attached) { + return { status: 'ready', session: attached, appAlias: undefined }; + } + const limrun = this.orgClient('Creating an instance'); + const requestedAsset = await resolveRequestedLimrunAppAsset(limrun, platform, context); + const params = this.sessionAllocationParams(limrun, owner.lease, requestedAsset); + const rent = (instanceId: string): void => { + owner.rentedInstanceId = instanceId; + }; + const session = + platform === 'ios' + ? await allocateLimrunIosSession(params, rent) + : await allocateLimrunAndroidSession(params, rent); + const appAlias: LimrunAppAlias | undefined = requestedAsset && { + assetName: requestedAsset.name, + installedAppId: await resolvePreinstalledAppId(session, requestedAsset), + }; + return { status: 'ready', session, appAlias }; + } catch (error) { + return { status: 'failed', error }; + } + } + + /** + * Puts the instance one create made, and the app alias that came with it, onto its owner. Another + * allocation that joined the same create can get here first, and a route that took the lease while the + * create ran owns the instance instead — either way this owner is already where it belongs, and nothing + * is published for a lease that no longer has an allocation. + */ + private goLive( + owner: LimrunLeaseOwner, session: LimrunRuntimeSession, - hostList: unknown, - ): Promise { - const harDir = this.options.networkHarDir; - if (!harDir) { - this.networkCaptures.set(leaseId, { phase: 'unavailable', reason: 'capture-dir-unset' }); - return; + appAlias: LimrunAppAlias | undefined, + ): void { + if (owner.state.phase !== 'allocating') return; + owner.appAlias = appAlias; + owner.state = { phase: 'live', session }; + if (this.options.keepAlive) this.startKeepAlive(owner, session); + } + + /** + * Closes an owner whose create made nothing, and ends the instance one that made an instance before it + * failed. A create that failed has no release coming for it — its caller is being refused — so this is + * the route that ends what it left running, through the same take every other teardown goes through. + * The outcome is reported to that caller, which is the only one left to hear it. + */ + private async settleFailedCreate( + owner: LimrunLeaseOwner, + ): Promise { + const state = owner.state; + if (state.phase === 'allocating') return await this.take(owner, 'create'); + // Another route took the lease while this call waited, and its disposal owns what the create made. + return state.phase === 'releasing' ? await state.disposal : undefined; + } + + /** + * Hands one lease's resources to the one route that may end them, or joins the disposal already ending + * them. The take and the disposal it starts are recorded before this step returns, so no route can reach + * the lease in between and find it free, and the owner leaves the map only once that disposal confirmed. + * A disposal that could not confirm keeps the owner here, so the next release, recovery or shutdown + * retries the delete rather than reporting an instance Limrun may still be billing. + */ + private take(owner: LimrunLeaseOwner, by: LimrunTeardownRoute): Promise { + const taken = owner.state; + const settled = taken.phase === 'releasing' ? taken.settled : undefined; + if (taken.phase === 'releasing' && !(settled?.cleanup === 'uncertain')) return taken.disposal; + let reportDisposal!: (outcome: LimrunDisposalOutcome) => void; + const disposal = new Promise((settledOutcome) => { + reportDisposal = settledOutcome; + }); + owner.state = { + phase: 'releasing', + by, + instanceId: + this.liveSession(owner)?.instanceId ?? owner.rentedInstanceId ?? settled?.instanceId, + disposal, + settled: undefined, + }; + const report = (outcome: LimrunDisposalOutcome): void => { + const state = owner.state; + if (state.phase === 'releasing') { + state.settled = outcome; + state.instanceId = outcome.instanceId ?? state.instanceId; + // Whatever the create made is now ended, or was never made, so this owner holds nothing and the + // lease is free again. An unconfirmed delete keeps the owner so its outcome can be retried. + if (outcome.cleanup !== 'uncertain') this.closeOwner(owner); + } + reportDisposal(outcome); + }; + // `disposeOwner` reports the failures it can name, so the rejection branch is only for what it could not + // report at all. The disposal still answers, because a caller may be waiting on this lease's instance. + void this.disposeOwner(owner).then(report, (error: unknown) => { + report({ + cleanup: 'uncertain', + instanceId: owner.rentedInstanceId, + failure: errorMessage(error), + capture: undefined, + }); + }); + return disposal; + } + + /** + * Ends what one taken owner stands for: the instance its create made or rented, with every resource the + * owner holds beside it. This is the only place an instance is ended, and an owner is taken by exactly one + * route at a time, so one instance is ended exactly once. Every wait here is bounded: work that outlives + * its budget keeps running and cleans up what it makes, and nothing of it stands between this route and the + * delete that stops the instance from being billed. + */ + private async disposeOwner(owner: LimrunLeaseOwner): Promise { + const { awaitLimrunWithinBudget, joinOutcomeFailures } = await import('./lifecycle-budget.ts'); + // A create that outlives this budget is reported rather than waited on: the owner is kept, so the next + // release, recovery or daemon close ends whatever that create goes on renting. + const create = await awaitLimrunWithinBudget(owner.creation, CREATE_ANSWER_BUDGET_MS); + const session = create.ok && create.value.status === 'ready' ? create.value.session : undefined; + const instanceId = session?.instanceId ?? owner.rentedInstanceId; + if (instanceId === undefined) return await this.disposeUnrented(owner, create.ok); + // The capture's record is optional reporting, and local teardown can outlive its own steps: neither is + // allowed to stand between this route and the delete that stops the instance from being billed, so both + // are started here and joined after it. A lease that never asked for a capture has no lifecycle to end, + // so its disposal never loads one. + const capture = this.capturesInstance?.end(owner); + const local = this.releaseOwnerResources(owner, session); + const instance = await this.endInstance(owner, session, instanceId); + const stuck = await this.awaitLocalTeardown(local); + // The record is optional reporting joined last: a capture that refuses to be written cannot keep a + // release from answering for the instance it ended, and is still listable once it does come to. + const joined = + capture === undefined + ? undefined + : await awaitLimrunWithinBudget(capture, CAPTURE_RECORD_BUDGET_MS); + return { + cleanup: instance.cleanup, + instanceId, + failure: joinOutcomeFailures(instance.failure, stuck), + capture: joined?.ok ? joined.value : undefined, + }; + } + + /** + * Ends what a disposal holds when no instance is named for the lease: its capture, and an outcome that + * keeps the lease held only while a create this route lost sight of may still rent something. + */ + private async disposeUnrented( + owner: LimrunLeaseOwner, + createAnswered: boolean, + ): Promise { + const { workOutstandingFailure } = await import('./lifecycle-budget.ts'); + const capture = await this.capturesInstance?.end(owner); + if (createAnswered) { + return { cleanup: 'none', instanceId: undefined, failure: undefined, capture }; } - const hosts = - typeof hostList === 'string' - ? hostList - .split(',') - .map((host) => host.trim()) - .filter(Boolean) - : []; - const { startLimrunHarCapture } = await import('./network-har.ts'); - const result = await startLimrunHarCapture( - session.client, - hosts.length > 0 ? hosts : undefined, - ); - this.networkCaptures.set( - leaseId, - result.ok - ? { - phase: 'capturing', - harPath: path.join(harDir, `${leaseId}.har`), - capture: result.capture, - } - : { phase: 'unavailable', reason: result.reason, message: result.message }, + return { + cleanup: 'uncertain', + instanceId: undefined, + failure: workOutstandingFailure( + `Instance create for lease '${owner.lease.leaseId}'`, + CREATE_ANSWER_BUDGET_MS, + ), + capture, + }; + } + + /** + * Ends one owner's instance. A delete Limrun already confirmed answers for itself, an attached instance is + * the caller's own to delete, and a created one is deleted here, with that answer deciding whether the + * lease can be given back. + */ + private async endInstance( + owner: LimrunLeaseOwner, + session: LimrunRuntimeSession | undefined, + instanceId: string, + ): Promise> { + if (owner.deletedInstanceId === instanceId) return { cleanup: 'confirmed', failure: undefined }; + if ((session?.ownership ?? 'created') === 'attached') { + // An attached instance is the caller's own, and this runtime was never given the right to delete it. + return { cleanup: 'attached', failure: undefined }; + } + const failure = await this.deleteInstance(owner.platform, instanceId); + if (failure === undefined) owner.deletedInstanceId = instanceId; + return { cleanup: failure === undefined ? 'confirmed' : 'uncertain', failure }; + } + + /** + * Ends the resources one owner holds besides its instance: the keep-alive beat that would otherwise ping a + * deleted instance forever, the instance connection, and the Android ADB tunnel and its reverse mappings. + * Each is released once, whichever route took the lease. + */ + private async releaseOwnerResources( + owner: LimrunLeaseOwner, + session: LimrunRuntimeSession | undefined, + ): Promise { + if (owner.keepAlive) clearInterval(owner.keepAlive); + owner.keepAlive = undefined; + if (session === undefined) return; + session.client.disconnect(); + // The tunnel's start is an `adb connect` the SDK runs without a deadline. Its cleanup retires the + // endpoint first, so a tunnel that answers late ends itself and no caller can reach it meanwhile. + if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); + } + + /** + * Reports the local teardown this route had to stop waiting for. A step that refused is reported as one + * that never answered: neither is something this runtime can still fix, and the delete that ran beside it + * has already been answered. + */ + private async awaitLocalTeardown(local: Promise): Promise { + const { awaitLimrunWithinBudget, workOutstandingFailure } = + await import('./lifecycle-budget.ts'); + const stuck = workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS); + const ended = await awaitLimrunWithinBudget( + local.then( + (): string | undefined => undefined, + (): string => stuck, + ), + LOCAL_TEARDOWN_BUDGET_MS, ); + return ended.ok ? ended.value : stuck; } - /** Writes a live capture's HAR now; a released capture reports the file it wrote at release. */ - private async writeNetworkCapture(leaseId: string): Promise { - const state = this.networkCaptures.get(leaseId); - if (state?.phase !== 'capturing') return state; - const failures = await state.capture.failures(); + /** One instance delete. `undefined` is Limrun's confirmation that the instance is gone. */ + private async deleteInstance( + platform: LimrunPlatform, + instanceId: string, + ): Promise { try { - const { writeLimrunHar } = await import('./network-har.ts'); - await writeLimrunHar(state.harPath, state.capture.entries()); - return { phase: 'written', harPath: state.harPath, failures }; + const limrun = this.orgClient('Deleting an instance'); + if (platform === 'ios') { + await limrun.iosInstances.delete(instanceId); + } else { + await limrun.androidInstances.delete(instanceId); + } + return undefined; } catch (error) { - return { - phase: 'unavailable', - reason: 'har-write-failed', - message: error instanceof Error ? error.message : String(error), - }; + return errorMessage(error); } } + /** Removes an owner once it holds nothing, and only while it is still the map's owner for that lease. */ + private closeOwner(owner: LimrunLeaseOwner): void { + if (this.leases.get(owner.lease.leaseId) === owner) this.leases.delete(owner.lease.leaseId); + } + + private startKeepAlive(owner: LimrunLeaseOwner, session: LimrunRuntimeSession): void { + const timer = setInterval(() => { + try { + session.client.keepAlive(); + } catch { + // A dead connection surfaces on the next real command. + } + }, KEEP_ALIVE_INTERVAL_MS); + timer.unref(); + owner.keepAlive = timer; + } + readonly cloudArtifacts: CloudArtifactProvider = { - listCloudArtifacts: async (query) => { - const capture = query.leaseId ? await this.writeNetworkCapture(query.leaseId) : undefined; - return capture && networkArtifacts(capture); - }, + listCloudArtifacts: async (query) => await this.listCloudArtifacts(query.leaseId), }; + /** + * What `artifacts` answers for one lease: its capture written now while the lease is held, and the + * record a release settled after the lease is gone. A runtime that never started a capture has no + * lifecycle to ask, so nothing is loaded for it. + */ + private async listCloudArtifacts( + leaseId: string | undefined, + ): Promise { + if (leaseId === undefined) return undefined; + const captures = this.capturesInstance; + if (captures === undefined) return undefined; + const { networkArtifacts } = await import('./network-har.ts'); + const capture = await captures.list(leaseId, this.leases.get(leaseId)?.capture); + return capture && networkArtifacts(capture); + } + private async attachSession( platform: LimrunPlatform, lease: DeviceLease, @@ -366,39 +726,6 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { ); } - private async createSession( - platform: LimrunPlatform, - lease: DeviceLease, - context?: LeaseLifecycleContext, - ): Promise { - const { - allocateLimrunAndroidSession, - allocateLimrunIosSession, - resolvePreinstalledAppId, - resolveRequestedLimrunAppAsset, - } = await import('./session-allocation.ts'); - const limrun = this.orgClient('Creating an instance'); - const requestedAsset = await resolveRequestedLimrunAppAsset(limrun, platform, context); - const params = this.sessionAllocationParams(limrun, lease, requestedAsset); - const session = - platform === 'ios' - ? await allocateLimrunIosSession(params) - : await allocateLimrunAndroidSession(params); - if (requestedAsset) { - try { - const installedAppId = await resolvePreinstalledAppId(session, requestedAsset); - this.appAliases.set(lease.leaseId, { - assetName: requestedAsset.name, - installedAppId, - }); - } catch (error) { - await this.terminateSession(session); - throw error; - } - } - return session; - } - private sessionAllocationParams(limrun: Limrun, lease: DeviceLease, app?: LimrunAppAsset) { return { limrun, @@ -424,99 +751,51 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { } private async release(lease: DeviceLease): Promise | undefined> { - const session = this.sessions.get(lease.leaseId); - if (!session) return await this.releaseRecoveredSession(lease); - const live = this.networkCaptures.get(lease.leaseId); - const capture = await this.writeNetworkCapture(lease.leaseId); - if (capture) { - // Re-insertion keeps the map in settle order, so the oldest released record goes first. - this.networkCaptures.delete(lease.leaseId); - this.networkCaptures.set(lease.leaseId, capture); - } - for (const leaseId of this.networkCaptures.keys()) { - if (this.networkCaptures.size <= SETTLED_NETWORK_CAPTURE_LIMIT) break; - if (!this.sessions.has(leaseId)) this.networkCaptures.delete(leaseId); - } - if (live?.phase === 'capturing') live.capture.close(); - await this.terminateSession(session); - this.sessions.delete(lease.leaseId); - this.appAliases.delete(lease.leaseId); - return { - limrunInstanceId: session.instanceId, - ...(capture ? { cloudArtifacts: networkArtifacts(capture) } : {}), - }; + // The take happens in this step, before anything is loaded: the release owns the lease from here, and no + // await in between can let the allocation still waiting on its create answer ahead of it. + const owner = this.leases.get(lease.leaseId); + const disposal = owner === undefined ? undefined : this.take(owner, 'release'); + const { releaseLimrunLease } = await import('./lease-release.ts'); + return await releaseLimrunLease(this.limrun, lease, disposal); } - private async releaseRecoveredSession( - lease: DeviceLease, - ): Promise | undefined> { - const platform = platformForLimrunLeaseBackend(lease.backend); - const limrun = this.limrun; - if (!platform || !limrun) return undefined; - const labelSelector = `provider=${LIMRUN_PROVIDER},leaseId=${lease.leaseId}`; - const instances = - platform === 'ios' - ? await limrun.iosInstances.list({ labelSelector }) - : await limrun.androidInstances.list({ labelSelector }); - const instanceIds = instances.getPaginatedItems().map((instance) => instance.metadata.id); - for (const instanceId of instanceIds) { - if (platform === 'ios') { - await limrun.iosInstances.delete(instanceId); - } else { - await limrun.androidInstances.delete(instanceId); - } - } - if (instanceIds.length === 0) return undefined; - return { limrunInstanceId: instanceIds[0], limrunInstanceCount: instanceIds.length }; + private orgClient(operation: string, hint?: string): Limrun { + return requireLimrunOrgClient(this.limrun, operation, hint); } - private startKeepAlive(leaseId: string, session: LimrunRuntimeSession): void { - const timer = setInterval(() => { - try { - session.client.keepAlive(); - } catch { - // A dead connection surfaces on the next real command. - } - }, KEEP_ALIVE_INTERVAL_MS); - timer.unref(); - this.keepAliveTimers.set(leaseId, timer); + /** The live session of one owner: what its create made, and only while the lease is held. */ + private liveSession(owner: LimrunLeaseOwner | undefined): LimrunRuntimeSession | undefined { + return owner?.state.phase === 'live' ? owner.state.session : undefined; } - private async terminateSession(session: LimrunRuntimeSession): Promise { - const leaseId = session.lease.leaseId; - clearInterval(this.keepAliveTimers.get(leaseId)); - this.keepAliveTimers.delete(leaseId); - const capture = this.networkCaptures.get(leaseId); - if (capture?.phase === 'capturing') { - this.networkCaptures.delete(leaseId); - capture.capture.close(); - } - session.client.disconnect(); - if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); - if (session.ownership === 'attached') return; - const limrun = this.orgClient('Deleting an instance'); - if (session.platform === 'ios') { - await limrun.iosInstances.delete(session.instanceId); - } else { - await limrun.androidInstances.delete(session.instanceId); + /** The live session of one owner, for a route that has already refused every other phase. */ + private requireLiveSession(owner: LimrunLeaseOwner): LimrunRuntimeSession { + const session = this.liveSession(owner); + if (session === undefined) { + throw new AppError( + 'COMMAND_FAILED', + `Limrun lease '${owner.lease.leaseId}' has no live session`, + { reason: 'limrun_lease_not_live' }, + ); } + return session; } - private orgClient(operation: string, hint?: string): Limrun { - return requireLimrunOrgClient(this.limrun, operation, hint); + private liveSessionForLease(leaseId: string): LimrunRuntimeSession | undefined { + return this.liveSession(this.leases.get(leaseId)); } private getSessionForDevice(device: DeviceInfo): LimrunRuntimeSession | undefined { const parsed = parseLimrunDeviceId(device.id); if (!parsed) return undefined; - const session = this.sessions.get(parsed.leaseId); + const session = this.liveSessionForLease(parsed.leaseId); return session?.platform === parsed.platform ? session : undefined; } resolveAppReference(device: DeviceInfo, app: string): string { const parsed = parseLimrunDeviceId(device.id); if (!parsed) return app; - const alias = this.appAliases.get(parsed.leaseId); + const alias = this.leases.get(parsed.leaseId)?.appAlias; return alias?.assetName === app ? alias.installedAppId : app; } @@ -549,7 +828,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { } private requireAndroidPortReverseSession(leaseId: string): LimrunAndroidSession | undefined { - const session = this.sessions.get(leaseId); + const session = this.liveSessionForLease(leaseId); if (!session || session.platform === 'android') return session; throw unsupported( 'port reverse', @@ -642,64 +921,75 @@ function unsupported(command: string, message: string): never { throw new AppError('UNSUPPORTED_OPERATION', message, { command }); } -/** Released captures stay listable; the oldest records are forgotten past this count. */ -const SETTLED_NETWORK_CAPTURE_LIMIT = 32; - -type SettledNetworkCapture = - | Readonly<{ phase: 'written'; harPath: string; failures: LimrunNetworkFailures }> - | Readonly<{ - phase: 'unavailable'; - reason: 'capture-dir-unset' | 'tunnel-in-use' | 'tunnel-start-failed' | 'har-write-failed'; - message?: string; - }>; +/** A lease cannot be allocated into a runtime that has been told to end every instance it holds. */ +function allocationRefusedAfterShutdown(lease: DeviceLease): never { + throw new AppError( + 'SESSION_NOT_FOUND', + `Cannot allocate Limrun lease '${lease.leaseId}': this runtime is shut down`, + { + reason: 'limrun_runtime_shut_down', + hint: 'Start the daemon again to allocate a Limrun lease.', + }, + ); +} -type NetworkCapture = - | SettledNetworkCapture - | Readonly<{ phase: 'capturing'; harPath: string; capture: LimrunHarCapture }>; +/** + * The refusal an allocation gets when a teardown took its lease while the create ran. The caller passes the + * disposal's outcome rather than reading the owner, so this answers with what that teardown came to — + * including a delete Limrun did not confirm, which the refused caller is the last route left to hear about. + */ +function allocationRefusedForTake( + lease: DeviceLease, + by: LimrunTeardownRoute, + outcome: LimrunDisposalOutcome, +): never { + const { instanceId } = outcome; + throw new AppError( + 'DEVICE_IN_USE', + `Limrun lease '${lease.leaseId}' was released while its instance was being prepared`, + { + reason: 'limrun_lease_released_mid_allocation', + ...(instanceId === undefined + ? {} + : { + limrunInstanceId: instanceId, + hint: `Instance '${instanceId}' was ended by the ${by} of this lease. Open a new lease.`, + }), + ...(outcome.cleanup === 'uncertain' + ? { retriable: true, ...(outcome.failure === undefined ? {} : { cause: outcome.failure }) } + : {}), + }, + ); +} -const NETWORK_ARTIFACT_NAME = 'Network traffic (HAR on the daemon host)'; +/** What an allocation reports when the create it waited on failed and left an instance running. */ +function createLeftInstanceRunning(lease: DeviceLease, outcome: LimrunDisposalOutcome): never { + throw new AppError( + 'COMMAND_FAILED', + `Limrun lease '${lease.leaseId}' could not be allocated, and its instance could not be deleted`, + { + reason: 'limrun_instance_delete_unconfirmed', + retriable: true, + ...(outcome.instanceId === undefined ? {} : { limrunInstanceId: outcome.instanceId }), + ...(outcome.failure === undefined ? {} : { cause: outcome.failure }), + hint: `Ask Limrun to delete instance '${outcome.instanceId ?? lease.leaseId}' — this runtime could not confirm the delete.`, + }, + ); +} -function networkArtifacts(capture: SettledNetworkCapture): CloudArtifactsResult { - if (capture.phase === 'unavailable') { - return { - provider: LIMRUN_PROVIDER, - status: 'unavailable', - message: `Network capture unavailable (${capture.reason})${capture.message ? `: ${capture.message}` : ''}`, - cloudArtifacts: [ - { - provider: LIMRUN_PROVIDER, - kind: 'network', - name: NETWORK_ARTIFACT_NAME, - availability: 'unavailable', - metadata: { reason: capture.reason }, - }, - ], - }; - } - const { count, last, lastDialFailure, truncated } = capture.failures; - const summary = [ - `Network capture failures: ${count}`, - ...(last - ? [`last ${last.kind}${last.host ? ` ${last.host}${last.port ? `:${last.port}` : ''}` : ''}`] - : []), - ...(lastDialFailure ? [`last dial failure ${lastDialFailure}`] : []), - ...(truncated > 0 ? [`${truncated} oldest requests left the HAR`] : []), - ].join('; '); - return { - provider: LIMRUN_PROVIDER, - status: 'ready', - ...(count > 0 || lastDialFailure || truncated > 0 ? { message: summary } : {}), - cloudArtifacts: [ - { - provider: LIMRUN_PROVIDER, - kind: 'network', - name: NETWORK_ARTIFACT_NAME, - url: capture.harPath, - extension: 'har', - contentType: 'application/json', - availability: 'ready', - metadata: { failures: capture.failures }, - }, - ], - }; +/** What a daemon close reports for every instance it could not stop before the process moved on. */ +function cleanupLeftInstancesRunning(outcomes: readonly LimrunDisposalOutcome[]): never { + const leftRunning = outcomes + .map((outcome) => outcome.instanceId) + .filter((id) => id !== undefined); + throw new AppError( + 'COMMAND_FAILED', + `Limrun cleanup could not delete ${String(leftRunning.length)} instance${leftRunning.length === 1 ? '' : 's'}`, + { + reason: 'limrun_instance_delete_unconfirmed', + ...(leftRunning.length === 0 ? {} : { limrunInstanceIds: leftRunning }), + ...(outcomes[0]?.failure === undefined ? {} : { cause: outcomes[0].failure }), + hint: `Ask Limrun to delete: ${leftRunning.join(', ')}.`, + }, + ); } diff --git a/packages/provider-limrun/src/session-allocation.test.ts b/packages/provider-limrun/src/session-allocation.test.ts new file mode 100644 index 0000000000..94f135e521 --- /dev/null +++ b/packages/provider-limrun/src/session-allocation.test.ts @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { test } from 'vitest'; +import Limrun from '@limrun/api'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { allocateLimrunAndroidSession, allocateLimrunIosSession } from './session-allocation.ts'; +import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; + +/** + * An instance create whose answer was lost is the one request this runtime must not repeat: Limrun may have + * made the instance already, and the SDK's own retries would POST again and rent a second instance no owner + * learns about. These cases put the real SDK in front of a local server, since that retry is decided below the + * request boundary every other Limrun suite replaces. + */ + +test('the Android instance create asks the SDK for one attempt, so a lost answer rents no second instance', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${request.path}`); + // The instance may well have been made; this answer never arrives. + request.drop(); + }); + const rented: string[] = []; + try { + await assert.rejects( + async () => + await allocateLimrunAndroidSession( + allocationParams(server), + (instanceId) => void rented.push(instanceId), + ), + ); + } finally { + await server.close(); + } + // One POST, so at most one instance: the SDK was not left to resolve the ambiguity by asking again. + assert.deepEqual(routes, ['POST /v1/android_instances']); + // Nothing was named, since no answer came back to name it. + assert.deepEqual(rented, []); +}); + +test('the iOS instance create asks the SDK for one attempt, so a lost answer rents no second instance', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${request.path}`); + request.drop(); + }); + const rented: string[] = []; + try { + await assert.rejects( + async () => + await allocateLimrunIosSession( + allocationParams(server), + (instanceId) => void rented.push(instanceId), + ), + ); + } finally { + await server.close(); + } + assert.deepEqual(routes, ['POST /v1/ios_instances']); + assert.deepEqual(rented, []); +}); + +test('an Android instance that answers without its endpoints is reported as rented and left for its owner', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${request.path}`); + await request.respond( + 200, + JSON.stringify({ metadata: { id: 'android-instance-9' }, status: { token: 'lim_token' } }), + ); + }); + const rented: string[] = []; + try { + await assert.rejects( + async () => + await allocateLimrunAndroidSession( + allocationParams(server), + (instanceId) => void rented.push(instanceId), + ), + (error) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /did not expose API and ADB websocket endpoints/.test(error.message), + ); + } finally { + await server.close(); + } + // The create names what it rented before any session is built on it, which is what lets the caller's owner + // end it; the helper does not delete it here, where a refusal could be swallowed. + assert.deepEqual(rented, ['android-instance-9']); + assert.deepEqual(routes, ['POST /v1/android_instances']); +}); + +function allocationParams(server: InstanceServer) { + return { + limrun: new Limrun({ apiKey: server.key, baseURL: server.baseUrl }), + lease: { + leaseId: 'lease-single-attempt', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease, + metadata: { displayName: 'agent-device', labels: { provider: 'limrun' } }, + dependencies: unreachedDependencies, + }; +} + +/** + * The host and platform adapters none of these cases reaches: the create, or the endpoints it names, answer + * first. Any step through one of these means a case lost its premise and fails on the spot. + */ +const unreachedDependencies = { + android: { + createPortReverse: async (): Promise => { + throw new Error('expected no session for an instance whose endpoints never arrived'); + }, + }, +} as unknown as LimrunRuntimeDependencies; + +type InstanceServer = Readonly<{ + baseUrl: string; + key: string; + close(): Promise; +}>; + +type ReceivedRequest = Readonly<{ + method: string | undefined; + path: string; + /** Ends the connection without an answer, the ambiguity a retry must not resolve by POSTing again. */ + drop(): void; + respond(status: number, body: string): Promise; +}>; + +function startInstanceServer( + onRequest: (request: ReceivedRequest) => Promise, +): Promise { + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + void onRequest({ + method: req.method, + // The query is Limrun's own create parameters, not part of the route a retry would repeat. + path: (req.url ?? '').split('?')[0] ?? '', + drop: () => req.socket.destroy(), + respond: async (status, body) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(body); + }, + }); + }); + }); + return new Promise((started, failed) => { + server.once('error', failed); + server.listen(0, '127.0.0.1', () => { + const address = server.address() as AddressInfo; + started({ + baseUrl: `http://127.0.0.1:${String(address.port)}`, + key: 'lim_test_key', + close: async () => + await new Promise((closed, closeFailed) => { + server.close((error) => (error ? closeFailed(error) : closed())); + server.closeAllConnections(); + }), + }); + }); + }); +} diff --git a/packages/provider-limrun/src/session-allocation.ts b/packages/provider-limrun/src/session-allocation.ts index 7c9fda7219..07fe452ccf 100644 --- a/packages/provider-limrun/src/session-allocation.ts +++ b/packages/provider-limrun/src/session-allocation.ts @@ -3,6 +3,7 @@ import type { DeviceLease, LeaseLifecycleContext } from '@agent-device/contracts import { AppError } from '@agent-device/kernel/errors'; import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import { createLimrunAndroidSession, type LimrunAndroidSession } from './android.ts'; +import { LIMRUN_SINGLE_ATTEMPT } from './client-options.ts'; import { buildLimrunDevice, LIMRUN_PROFILE_FIELDS } from './device.ts'; import { createLimrunIosSession, type LimrunIosSession } from './ios.ts'; import { @@ -67,89 +68,99 @@ export async function resolvePreinstalledAppId( export async function allocateLimrunIosSession( params: SessionAllocationParams, + onRent: (instanceId: string) => void, ): Promise { - const instance = (await params.limrun.iosInstances.create({ - wait: true, - metadata: params.metadata, - spec: { - ...(params.region ? { region: params.region } : {}), - ...(params.app - ? { - initialAssets: [ - { - kind: 'App' as const, - source: 'AssetID' as const, - assetId: params.app.id, - launchMode: 'RelaunchIfRunning' as const, - }, - ], - } - : {}), - }, - })) as LimrunInstance; - try { - if (!instance.status.apiUrl) { - throw new AppError('COMMAND_FAILED', 'Limrun iOS instance did not expose apiUrl'); - } - return await createLimrunIosSession( - { - lease: params.lease, - instanceId: instance.metadata.id, - ownership: 'created', - device: buildLimrunDevice('ios', params.lease, instance.metadata.id), - apiUrl: instance.status.apiUrl, - token: instance.status.token, + const instance = (await params.limrun.iosInstances.create( + { + wait: true, + metadata: params.metadata, + spec: { + ...(params.region ? { region: params.region } : {}), + ...(params.app + ? { + initialAssets: [ + { + kind: 'App' as const, + source: 'AssetID' as const, + assetId: params.app.id, + launchMode: 'RelaunchIfRunning' as const, + }, + ], + } + : {}), }, - params.dependencies, - ); - } catch (error) { - await params.limrun.iosInstances.delete(instance.metadata.id).catch(() => {}); - throw error; + }, + // Limrun may have created this instance even though its answer was lost, so the SDK must not POST + // this call again: a second create would rent a second instance nobody is standing beside. + LIMRUN_SINGLE_ATTEMPT, + )) as LimrunInstance; + const instanceId = instance.metadata.id; + // Rented from this step: the caller's owner records the instance here, before any session is built on it, + // so its disposal can always end it — including when the endpoint check or the session below fails. + onRent(instanceId); + if (!instance.status.apiUrl) { + throw new AppError('COMMAND_FAILED', 'Limrun iOS instance did not expose apiUrl'); } + return await createLimrunIosSession( + { + lease: params.lease, + instanceId, + ownership: 'created', + device: buildLimrunDevice('ios', params.lease, instanceId), + apiUrl: instance.status.apiUrl, + token: instance.status.token, + }, + params.dependencies, + ); } export async function allocateLimrunAndroidSession( params: SessionAllocationParams, + onRent: (instanceId: string) => void, ): Promise { - const instance = (await params.limrun.androidInstances.create({ - wait: true, - metadata: params.metadata, - spec: { - ...(params.region ? { region: params.region } : {}), - ...(params.app - ? { - initialAssets: [ - { - kind: 'App' as const, - source: 'AssetIDs' as const, - assetIds: [params.app.id], - }, - ], - } - : {}), - }, - })) as LimrunInstance; - try { - if (!instance.status.apiUrl || !instance.status.adbWebSocketUrl) { - throw new AppError( - 'COMMAND_FAILED', - 'Limrun Android instance did not expose API and ADB websocket endpoints', - ); - } - return await createLimrunAndroidSession( - { - lease: params.lease, - instanceId: instance.metadata.id, - ownership: 'created', - device: buildLimrunDevice('android', params.lease, instance.metadata.id), - apiUrl: instance.status.apiUrl, - token: instance.status.token, - adbUrl: instance.status.adbWebSocketUrl, + const instance = (await params.limrun.androidInstances.create( + { + wait: true, + metadata: params.metadata, + spec: { + ...(params.region ? { region: params.region } : {}), + ...(params.app + ? { + initialAssets: [ + { + kind: 'App' as const, + source: 'AssetIDs' as const, + assetIds: [params.app.id], + }, + ], + } + : {}), }, - params.dependencies, + }, + // Limrun may have created this instance even though its answer was lost, so the SDK must not POST + // this call again: a second create would rent a second instance nobody is standing beside. + LIMRUN_SINGLE_ATTEMPT, + )) as LimrunInstance; + const instanceId = instance.metadata.id; + // Rented from this step: the caller's owner records the instance here, before any session is built on it, + // so its disposal can always end it — including when an endpoint is missing or the session below fails. + onRent(instanceId); + if (!instance.status.apiUrl || !instance.status.adbWebSocketUrl) { + throw new AppError( + 'COMMAND_FAILED', + 'Limrun Android instance did not expose API and ADB websocket endpoints', ); - } catch (error) { - await params.limrun.androidInstances.delete(instance.metadata.id).catch(() => {}); - throw error; } + return await createLimrunAndroidSession( + { + lease: params.lease, + instanceId, + ownership: 'created', + device: buildLimrunDevice('android', params.lease, instanceId), + apiUrl: instance.status.apiUrl, + token: instance.status.token, + adbUrl: instance.status.adbWebSocketUrl, + }, + params.dependencies, + ); } diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index dec506e77a..9bd6304c7a 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -754,7 +754,19 @@ test('Limrun keeps session tracked when release fails so release can be retried' await allocateLease(lease); limrunMockState.androidDelete.mockRejectedValueOnce(new Error('temporary delete failure')); - await assert.rejects(() => releaseLease(lease), /temporary delete failure/); + // An unconfirmed delete is reported as such, with the instance still held, rather than as a release. + await assert.rejects( + () => releaseLease(lease), + (error: unknown) => { + const details = (error as AppError).details; + return ( + error instanceof AppError && + details?.reason === 'limrun_instance_delete_unconfirmed' && + details.limrunInstanceId === 'android-instance-1' && + details.cause === 'temporary delete failure' + ); + }, + ); assert.deepEqual(await releaseLease(lease), { limrunInstanceId: 'android-instance-1' }); } finally { await runtime.shutdown(); From c91a2b9c1a1644a4fe0aed390da31705455cde62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 17:51:10 +0100 Subject: [PATCH 02/11] fix(limrun): bound every lease teardown and keep what a create rented Five ways a lease could outlive its caller's answer are closed here, each behind a test that fails without its fix. The remote delete now has a deadline of its own: the SDK clears a request's timer before it reads the response body, so a DELETE whose body never arrives would otherwise hold the release, the daemon close and every joining allocation open forever. It reports `uncertain`, keeps the lease held, and stays observed, so its late confirmation is recorded. A create that answers after its disposal stopped waiting is no longer unowned: the disposal watches it and ends the instance it rents and the session built on it, rather than leaving both running with no route left to end them. A session is recorded on its owner in the step that answers for it, so an allocation that fails after building one still has its instance connection and host ADB endpoint released by whoever ends the lease. A create that refuses the caller who started it for a profile flag this provider reads no field for answers for that caller alone. The lease rented nothing, so a joiner whose own flags are valid makes that create its own instead of inheriting another caller's refusal. The serial an ADB teardown is ending belongs to that teardown alone: it dispatches through an executor of its own, and retirement is checked before any cached serial, so a retained interactor cannot take commands on a device being disconnected. The disposal itself moved to `lease-release.ts`, beside the refusals that report what it could not stop, and asks only for one instance delete and one capture end. --- packages/provider-limrun/src/android.test.ts | 70 +++- packages/provider-limrun/src/android.ts | 87 +++-- .../provider-limrun/src/lease-ownership.ts | 25 +- packages/provider-limrun/src/lease-release.ts | 367 +++++++++++++++++- packages/provider-limrun/src/runtime.test.ts | 159 +++++++- packages/provider-limrun/src/runtime.ts | 304 +++++---------- 6 files changed, 749 insertions(+), 263 deletions(-) diff --git a/packages/provider-limrun/src/android.test.ts b/packages/provider-limrun/src/android.test.ts index 95ebac68d6..42942ce613 100644 --- a/packages/provider-limrun/src/android.test.ts +++ b/packages/provider-limrun/src/android.test.ts @@ -21,6 +21,43 @@ import type { const ok = { exitCode: 0, stdout: '', stderr: '' }; +/** + * The serial a reverse cleanup is ending is not a route a retained caller may take: an ordinary command + * arriving inside that window has to be refused, or the taken session keeps taking commands on the device. + */ +test('a command that arrives while cleanup holds its serial is refused', async () => { + const runs: string[][] = []; + const session = await createLimrunAndroidSession( + ANDROID_SESSION_OPTIONS, + androidDependencies(runs), + ); + // The first command raises the tunnel this session owns, so a later one has a serial to reach for. + await session.adbProvider.exec(['shell', 'echo', 'ready']); + const dispatched = runs.length; + let reportList!: () => void; + reverse.listGate = new Promise((resolve) => { + reportList = resolve; + }); + try { + // Cleanup has retired this session and is inside the device with the serial it is ending. + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + await vi.waitFor(() => expect(reverse.list).toHaveBeenCalledOnce()); + + await expect(session.adbProvider.exec(['shell', 'echo', 'again'])).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + + reportList(); + await cleanup; + } finally { + reverse.listGate = undefined; + } + // The only commands this session dispatched are its own cleanup's: the listing and the disconnect. + expect(runs.slice(dispatched).filter((argv) => argv.includes('shell'))).toEqual([]); + expect(tunnels.start).toHaveBeenCalledOnce(); +}); + const tunnels = vi.hoisted(() => ({ /** * Held by a case that has to keep a tunnel in flight, which is the window a session's cleanup has to @@ -34,6 +71,19 @@ const tunnels = vi.hoisted(() => ({ close: vi.fn(), })); +/** The reverse manager every session of a case gets, with its listing held by the case that needs a window. */ +const reverse = vi.hoisted(() => ({ + /** Held by a case that has to keep a cleanup inside the window in which it holds its serial. */ + listGate: undefined as Promise | undefined, + list: vi.fn(async () => { + await reverse.listGate; + return []; + }), + ensure: vi.fn(async () => {}), + remove: vi.fn(async () => {}), + removeAllOwned: vi.fn(async () => {}), +})); + vi.mock('@limrun/api/instance-client', () => ({ createInstanceClient: vi.fn(async () => ({ startAdbTunnel: tunnels.start })), })); @@ -42,6 +92,8 @@ beforeEach(() => { tunnels.gate = undefined; tunnels.start.mockClear(); tunnels.close.mockClear(); + reverse.listGate = undefined; + reverse.list.mockClear(); }); test('cleanup asks the platform to address the disconnect as a server-level command and hands it over unchanged', async () => { @@ -182,7 +234,9 @@ const ANDROID_SESSION_OPTIONS = { token: 'instance-token', }; -function androidDependencies(): Pick { +function androidDependencies( + runs?: string[][], +): Pick { return { android: { deviceAdbInvocation: (serial: string, command: readonly string[]) => @@ -190,13 +244,13 @@ function androidDependencies(): Pick androidAdbInvocation(androidAdbHostTarget(), command), adbError: async (message: string) => new Error(message), - createPortReverse: async () => ({ - ensure: async () => {}, - remove: async () => {}, - removeAllOwned: async () => {}, - list: async () => [], - }), + createPortReverse: async () => reverse, } as unknown as LimrunRuntimeDependencies['android'], - host: { runAdb: async () => ok } as unknown as LimrunRuntimeDependencies['host'], + host: { + runAdb: async (invocation: AndroidAdbInvocation) => { + runs?.push([...serializeAndroidAdbInvocation(invocation)]); + return ok; + }, + } as unknown as LimrunRuntimeDependencies['host'], }; } diff --git a/packages/provider-limrun/src/android.ts b/packages/provider-limrun/src/android.ts index e7c221c7f7..9beadc6590 100644 --- a/packages/provider-limrun/src/android.ts +++ b/packages/provider-limrun/src/android.ts @@ -16,6 +16,7 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import type { LimrunAdbCommandOptions, LimrunAdbCommandResult, + LimrunAdbExecutor, LimrunAdbProvider, LimrunPortReverseEndpoint, LimrunRuntimeDependencies, @@ -49,10 +50,11 @@ type LimrunAndroidAdbSession = { */ adbTunnelRetired?: boolean; /** - * The endpoint one teardown is ending, recorded so that teardown keeps its own way to reach the device - * whose reverse mappings it is clearing. No other route reads it. + * Where teardown points this session's ADB route while it ends an endpoint: at that endpoint, whatever the + * session's own public route says about it. Only this session's cleanup sets it, and only for as long as it + * is dispatching, so no caller that kept an interactor can reach a serial that is on its way out. */ - adbTeardownSerial?: string; + adbReverseExecutor?: LimrunAdbExecutor; readonly dependencies: Pick; }; @@ -96,9 +98,13 @@ export async function createLimrunAndroidSession( await createLimrunAndroidTextInjector(client)(request); }, }; - adbProvider.reverse = await dependencies.android.createPortReverse(adbProvider.exec, { - noRebind: options.ownership === 'attached', - }); + // The reverse manager has to outlive this session's own ADB route: its cleanup clears reverse mappings off a + // device the session is disconnecting, after retirement has closed that route to every caller. So it dispatches + // through a route of its own, which is the session's route until cleanup takes it over. + adbProvider.reverse = await dependencies.android.createPortReverse( + async (args, options) => await runLimrunAndroidReverseAdb(session, args, options), + { noRebind: options.ownership === 'attached' }, + ); return Object.assign(session, { adbProvider }); } @@ -166,15 +172,18 @@ export async function cleanupLimrunAndroidAdbTunnel(session: LimrunAndroidSessio session.adbTunnelRetired = true; // Cleanup keeps the serial it finds, or the one the in-flight start publishes while this step waits on it: // a start answering into a retired session records its endpoint here first, because this cleanup is the only - // route left that can reach it. That serial is this session's only access to the host once the endpoint is - // retired, and retirement still lets this step through — it has to clear the reverse mappings off the device - // it is about to disconnect. + // route left that can reach it. const started = await session.adbTunnelPromise?.catch(() => undefined); session.adbTunnelPromise = undefined; const serial = session.adbSerial ?? started; - session.adbTeardownSerial = serial; try { if (serial) { + // Clearing this session's reverse mappings means reaching the device this cleanup is about to disconnect, + // and this session's public ADB route is closed from the step above. So its reverse manager is pointed at + // an executor of this cleanup's own, addressed at the serial being ended — a serial no retained caller can + // ask for. + session.adbReverseExecutor = async (args, options) => + await runLimrunAndroidAdbAtSerial(session, serial, args, options); await cleanupAndroidPortReverse(session); await session.dependencies.host .runAdb(session.dependencies.android.hostAdbInvocation(['disconnect', serial]), { @@ -184,7 +193,7 @@ export async function cleanupLimrunAndroidAdbTunnel(session: LimrunAndroidSessio .catch(() => {}); } } finally { - session.adbTeardownSerial = undefined; + session.adbReverseExecutor = undefined; closeLimrunAndroidAdbTunnel(session); } } @@ -223,25 +232,36 @@ async function runLimrunAndroidAdb( args: readonly string[], options?: LimrunAdbCommandOptions, ): Promise { - const { invocation, result } = await executeLimrunAndroidAdb(session, args, options); - return await requireSuccessfulLimrunAndroidAdb( - invocation, - result, - options?.allowFailure, - session.dependencies, - ); + const serial = await ensurePersistentAndroidAdbSerial(session); + return await runLimrunAndroidAdbAtSerial(session, serial, args, options); } /** - * The tunnel serial is this provider's own addressing decision: it goes on the invocation's - * target, so the command array the Android cluster handed over reaches the host unchanged. + * The reverse manager's own route to the device. It is the session's route while the session is live, and + * cleanup's route of its own once the session is retired: clearing reverse mappings means reaching the device + * that is about to be disconnected, and that has to stay possible after every caller of this session was refused. */ -async function executeLimrunAndroidAdb( +async function runLimrunAndroidReverseAdb( session: LimrunAndroidAdbSession, args: readonly string[], options?: LimrunAdbCommandOptions, -): Promise<{ invocation: AndroidAdbInvocation; result: LimrunAdbCommandResult }> { - const serial = await ensurePersistentAndroidAdbSerial(session); +): Promise { + const atSerial = session.adbReverseExecutor; + if (atSerial !== undefined) return await atSerial(args, options); + return await runLimrunAndroidAdb(session, args, options); +} + +/** + * Dispatches one command at a serial the caller already holds, which is where both of this provider's ADB routes + * end up. The tunnel serial is this provider's own addressing decision: it goes on the invocation's target, so + * the command array the Android cluster handed over reaches the host unchanged. + */ +async function runLimrunAndroidAdbAtSerial( + session: LimrunAndroidAdbSession, + serial: string, + args: readonly string[], + options?: LimrunAdbCommandOptions, +): Promise { const invocation = session.dependencies.android.deviceAdbInvocation(serial, args); const result = await session.dependencies.host.runAdb(invocation, { allowFailure: options?.allowFailure, @@ -250,7 +270,12 @@ async function executeLimrunAndroidAdb( timeoutMs: options?.timeoutMs ?? 30_000, signal: options?.signal, }); - return { invocation, result }; + return await requireSuccessfulLimrunAndroidAdb( + invocation, + result, + options?.allowFailure, + session.dependencies, + ); } async function requireSuccessfulLimrunAndroidAdb( @@ -270,12 +295,10 @@ async function requireSuccessfulLimrunAndroidAdb( } async function ensurePersistentAndroidAdbSerial(session: LimrunAndroidAdbSession): Promise { - // The teardown's own read comes first, and is the only one retirement lets through: clearing an endpoint's - // reverse mappings means asking the device this cleanup is about to disconnect what was mapped on it. - const teardownSerial = session.adbTeardownSerial; - if (teardownSerial !== undefined) return teardownSerial; - // The retired check comes before the serial fast path: a session whose cleanup answered while an - // earlier command was still in flight holds a serial nothing owns any more. + // Retirement is checked before any serial is read, including one this session cached: a caller that kept an + // interactor from a session whose cleanup answered is refused the same as one that arrives first. The serial + // of a session being torn down is that teardown's alone, and its reverse manager reaches it through the + // executor it was handed, not through this route. if (session.adbTunnelRetired) throw alreadyCleanedUp(session); if (session.adbSerial) return session.adbSerial; const pending = session.adbTunnelPromise ?? startAndroidAdbTunnel(session); @@ -307,7 +330,9 @@ function alreadyCleanedUp(session: LimrunAndroidAdbSession): AppError { `Limrun ADB tunnel for instance '${session.instanceId}' was already cleaned up`, { reason: 'limrun_adb_tunnel_cleaned_up', - hint: `Lease '${session.lease.leaseId}' is closed. Reopen the lease instead of driving this interactor.`, + // An unconfirmed delete holds this lease while its release is retried, so the hint cannot promise the + // lease is finished; it says what this route is closed to, and what the caller can still do. + hint: `This interactor's ADB tunnel is closed for lease '${session.lease.leaseId}'. Release the lease again, or open a new lease once its instance is confirmed deleted.`, }, ); } diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts index 388adedc05..e4e51c2b48 100644 --- a/packages/provider-limrun/src/lease-ownership.ts +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -27,15 +27,28 @@ export type LimrunLeaseOwner = { readonly platform: LimrunPlatform; /** * The create that is making this lease's instance; never rejects unobserved — the runtime reads it. It - * carries its own outcome, including a refusal to bill anything for this lease's profile flags. + * carries its own outcome, including a refusal to bill anything for the flags of the caller that started it. + * A joiner whose own flags were answered for replaces an outcome that billed nothing with its own create. */ - readonly creation: Promise; + creation: Promise; state: LimrunLeaseState; /** * The instance this lease's create rented, recorded the moment Limrun answered, before any session was * built on it. A create that could not finish still owes an end for it, and its disposal reads it here. */ rentedInstanceId: string | undefined; + /** + * The session this lease's create built, recorded the moment it existed. It is the handle that holds the + * instance connection and the host ADB endpoint open, so it stays on the owner even when a later step of + * the allocation fails: whoever ends this lease ends those resources too, not only the instance. + */ + session: LimrunRuntimeSession | undefined; + /** + * The one release of this owner's host resources: the instance connection and the Android ADB endpoint. + * Disposal starts it, and the continuation of a create that answered after its disposal refused answers + * to the same promise, so a session that arrived late is never released twice and never skipped. + */ + localRelease: Promise | undefined; /** The keep-alive interval for the live instance; started when it went live, cleared by its disposal. */ keepAlive: NodeJS.Timeout | undefined; /** What this lease has in capture terms, including a start whose tunnel is still being opened. */ @@ -83,7 +96,10 @@ export type LimrunDisposalOutcome = Readonly<{ }>; /** - * What one create amounted to: the session it made, or why it refused or failed. An instance the create + * What one create amounted to: the session it made, why it failed, or that the caller who started it was + * refused before anything was billed. `refused_caller` is that caller's own answer and not the lease's: an + * owner that rented nothing stays allocatable for a caller whose flags this provider does read, and the + * distinction is what keeps one caller's refusal from becoming another caller's outcome. An instance a create * rented before it failed is not carried here — the owner records it as soon as Limrun names it, which is * earlier than any outcome could be reported. */ @@ -93,7 +109,8 @@ export type LimrunCreateOutcome = session: LimrunRuntimeSession; appAlias: LimrunAppAlias | undefined; }> - | Readonly<{ status: 'failed'; error: unknown }>; + | Readonly<{ status: 'failed'; error: unknown }> + | Readonly<{ status: 'refused_caller'; error: unknown }>; type LimrunLeaseState = | Readonly<{ phase: 'allocating' }> diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index ac6fbd4715..6ddc14cea8 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -1,14 +1,273 @@ import type Limrun from '@limrun/api'; import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; -import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend } from './device.ts'; -import type { LimrunDisposalOutcome } from './lease-ownership.ts'; +import { cleanupLimrunAndroidAdbTunnel } from './android.ts'; +import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend, type LimrunPlatform } from './device.ts'; +import type { + LimrunDisposalOutcome, + LimrunLeaseOwner, + LimrunRuntimeSession, + LimrunTeardownRoute, +} from './lease-ownership.ts'; +import type { SettledNetworkCapture } from './network-har.ts'; /** - * The route that ends one lease: the release a caller asked for, or the recovery of a lease this runtime lost - * sight of. Both answer for the same thing — what the lease has running, and whether it stopped — and neither - * reports a lease as released on anything short of Limrun confirming the delete. + * Everything this runtime does once a lease has been taken for good: end what its owner holds, and answer with + * what came of it. One lease's disposal runs here, and every route that ends a lease — release, daemon close, + * expired-lease recovery, and the create that failed — answers from the same outcome, so none of them can + * report a lease as released on anything short of Limrun confirming the delete. + * + * Every wait here is bounded. Work that outlives its deadline keeps running and stays observed: this module + * ends what it eventually makes, and reports what it could not, rather than holding a caller open. + */ + +/** + * How long a disposal waits for the create to say what it rented. The create itself keeps Limrun's own + * timeouts: this bounds only the disposal's wait, so an instance is never held open by a create that stopped + * answering, and its owner disposes what that create eventually names. + */ +const CREATE_ANSWER_BUDGET_MS = 120_000; + +/** + * How long a disposal waits for one delete. The SDK clears a request's timer before it reads the response body, + * so a DELETE whose body never arrives would otherwise hold the release, the daemon close and every joining + * allocation open forever. + */ +const DELETE_ANSWER_BUDGET_MS = 30_000; + +/** + * How long a disposal waits for the local resources it started beside the delete: the instance connection and + * the Android ADB endpoint, whose `adb connect` the SDK runs without a deadline. + */ +const LOCAL_TEARDOWN_BUDGET_MS = 45_000; + +/** How long a disposal waits for a capture's HAR record, which is reporting and never a reason to hold a lease. */ +const CAPTURE_RECORD_BUDGET_MS = 15_000; + +/** + * What a disposal may ask of the runtime that owns the lease, and nothing else: the delete of one instance, and + * the end of one lease's capture. Keeping the disposal to these two is what lets it end an owner without + * reaching back into the routes that allocate and read one, and without deciding when a lease is free again. + */ +export type LimrunDisposalHost = Readonly<{ + /** One instance delete. `undefined` answers that Limrun confirmed the instance is gone. */ + deleteInstance(platform: LimrunPlatform, instanceId: string): Promise; + /** Ends one lease's capture and answers with its record, for a runtime that has captured something. */ + endCapture(owner: LimrunLeaseOwner): Promise; +}>; + +/** + * Ends what one taken owner stands for: the instance its create made or rented, with every resource the owner + * holds beside it. This is the only place an instance is ended, and an owner is taken by exactly one route at a + * time, so one instance is ended exactly once. */ +export async function disposeLimrunLeaseOwner( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + const { awaitLimrunWithinBudget, joinOutcomeFailures } = await import('./lifecycle-budget.ts'); + const create = await awaitLimrunWithinBudget(owner.creation, CREATE_ANSWER_BUDGET_MS); + // The create answered too late for this disposal to wait on it; what it goes on to name is still this + // disposal's to end. + if (!create.ok) return await disposeUnansweredCreate(host, owner); + // The owner records what its create actually made — the instance the moment Limrun named it, the session the + // moment it was built — so a create that failed or refused its caller is disposed from those, exactly like + // one that answered. + const session = owner.session; + const instanceId = session?.instanceId ?? owner.rentedInstanceId; + if (instanceId === undefined) return await disposeUnrented(host, owner); + // The capture's record is optional reporting, and local teardown can outlive its own steps: neither is + // allowed to stand between this disposal and the delete that stops the instance from being billed, so both + // are started here and joined after it. + const capture = host.endCapture(owner); + const local = releaseOwnerResources(owner); + const instance = await endInstance(host, owner, session, instanceId); + const stuck = await awaitLocalTeardown(local); + // The record is optional reporting joined last: a capture that refuses to be written cannot keep a release + // from answering for the instance it ended, and is still listable once it does come to. + const joined = await awaitLimrunWithinBudget(capture, CAPTURE_RECORD_BUDGET_MS); + return { + cleanup: instance.cleanup, + instanceId, + failure: joinOutcomeFailures(instance.failure, stuck), + capture: joined.ok ? joined.value : undefined, + }; +} + +/** + * Ends what a disposal holds when the create it waited for never answered. Nothing is known to be running, so + * the lease is kept held and refused rather than given back — and the create is watched, because whatever it + * eventually names belongs to this disposal and to no other route: a daemon close promises no next pass. + */ +async function disposeUnansweredCreate( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + const { awaitLimrunWithinBudget, joinOutcomeFailures, workOutstandingFailure } = + await import('./lifecycle-budget.ts'); + const capture = host.endCapture(owner); + const outstanding = workOutstandingFailure( + `Instance create for lease '${owner.lease.leaseId}'`, + CREATE_ANSWER_BUDGET_MS, + ); + owner.creation.then(() => { + void endWhatTheCreateNamed(host, owner); + }); + const instanceId = owner.rentedInstanceId; + if (instanceId === undefined) { + return { + cleanup: 'uncertain', + instanceId: undefined, + failure: outstanding, + capture: await capture, + }; + } + // An instance is already named, so this disposal ends the billing it can see and leaves the create's silence + // as what keeps the lease held. + const instance = await endInstance(host, owner, owner.session, instanceId); + const local = await awaitLimrunWithinBudget( + releaseOwnerResources(owner), + LOCAL_TEARDOWN_BUDGET_MS, + ); + return { + cleanup: instance.cleanup, + instanceId, + failure: joinOutcomeFailures( + instance.failure, + local.ok ? outstanding : workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS), + ), + capture: await capture, + }; +} + +/** + * Ends what a create answered for after its disposal had refused its caller: the instance it rented, if that is + * still owed, and every host resource the session it built holds. Nothing else is left to do this — the lease + * was refused to its caller, and no later route is promised. + */ +async function endWhatTheCreateNamed( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + const instanceId = owner.rentedInstanceId; + if (instanceId !== undefined && owner.deletedInstanceId !== instanceId) { + await endInstance(host, owner, owner.session, instanceId); + } + await releaseOwnerResources(owner); +} + +/** + * Ends what a disposal holds when its create answered and no instance is named for the lease: nothing this + * runtime rented is running, so the lease can be given back. + */ +async function disposeUnrented( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + return { + cleanup: 'none', + instanceId: undefined, + failure: undefined, + capture: await host.endCapture(owner), + }; +} + +/** + * Ends one owner's instance within a deadline of its own. An attached instance is the caller's own and is never + * deleted here; a delete Limrun already confirmed answers for itself; and a delete that never answers keeps the + * lease held and stays observed, because what it confirms late is what frees the lease. + */ +async function endInstance( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, + session: LimrunRuntimeSession | undefined, + instanceId: string, +): Promise> { + const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); + if (owner.deletedInstanceId === instanceId) return { cleanup: 'confirmed', failure: undefined }; + if ((session?.ownership ?? 'created') === 'attached') { + return { cleanup: 'attached', failure: undefined }; + } + const requested = host.deleteInstance(owner.platform, instanceId); + const deleted = await awaitLimrunWithinBudget(requested, DELETE_ANSWER_BUDGET_MS); + if (!deleted.ok) { + void requested.then((failure) => { + if (failure === undefined) confirmLateDelete(owner, instanceId); + }); + return { + cleanup: 'uncertain', + failure: workOutstandingFailure( + `Delete of instance '${instanceId}'`, + DELETE_ANSWER_BUDGET_MS, + ), + }; + } + if (deleted.value === undefined) owner.deletedInstanceId = instanceId; + return { + cleanup: deleted.value === undefined ? 'confirmed' : 'uncertain', + failure: deleted.value, + }; +} + +/** + * Records a delete that confirmed after its disposal stopped waiting for it. The lease stays held for its next + * route, which is the one that has to tell its caller the instance is gone; this only makes sure that route + * never asks Limrun to delete the same instance twice. + */ +function confirmLateDelete(owner: LimrunLeaseOwner, instanceId: string): void { + owner.deletedInstanceId = instanceId; +} + +/** + * Ends the resources one owner holds besides its instance: the keep-alive beat that would otherwise ping a + * deleted instance forever, the instance connection, and the Android ADB tunnel with its reverse mappings. One + * release answers for one owner, whichever route asked first, so a session that arrives after this started is + * still released by it rather than by a second pass nobody is promised. + */ +function releaseOwnerResources(owner: LimrunLeaseOwner): Promise { + owner.localRelease ??= releaseLocalResources(owner); + return owner.localRelease; +} + +async function releaseLocalResources(owner: LimrunLeaseOwner): Promise { + if (owner.keepAlive) clearInterval(owner.keepAlive); + owner.keepAlive = undefined; + // The session is what holds a host endpoint open. The owner has kept it from the moment it was built, whether + // or not the allocation that built it answered, so a create this disposal stopped waiting for is awaited here + // rather than passed over: the session it eventually builds has nowhere else to be disconnected. + const session = owner.session ?? (await sessionTheCreateBuilt(owner)); + if (session === undefined) return; + session.client.disconnect(); + // The tunnel's start is an `adb connect` the SDK runs without a deadline. Its cleanup retires the endpoint + // first, so a tunnel that answers late ends itself and no caller can reach it meanwhile. + if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); +} + +/** The session a create answered for, once it answers, for a release that got there first. */ +async function sessionTheCreateBuilt( + owner: LimrunLeaseOwner, +): Promise { + await owner.creation.catch(() => undefined); + return owner.session; +} + +/** + * Reports the local teardown a disposal had to stop waiting for. A step that refused is reported as one that + * never answered: neither is something this runtime can still fix, and the delete that ran beside it has + * already been answered. + */ +async function awaitLocalTeardown(local: Promise): Promise { + const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); + const stuck = workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS); + const ended = await awaitLimrunWithinBudget( + local.then( + (): string | undefined => undefined, + (): string => stuck, + ), + LOCAL_TEARDOWN_BUDGET_MS, + ); + return ended.ok ? ended.value : stuck; +} /** What a release answers for one lease: the instance it ended, with what it left and wrote. */ type LimrunReleaseRecord = Record; @@ -78,17 +337,109 @@ async function releaseLabeledInstances( return { limrunInstanceId: instanceIds[0], limrunInstanceCount: instanceIds.length }; } -/** What a release reports when Limrun did not confirm the delete, so the lease is still held. */ +/** + * What a release reports when Limrun did not confirm the delete, so the lease is still held. An unended lease + * whose create never named an instance is answered for by the lease alone, because the operator has nothing to + * name to Limrun yet. + */ function releaseLeftInstanceRunning(leaseId: string, outcome: LimrunDisposalOutcome): never { + const { instanceId } = outcome; throw new AppError( 'COMMAND_FAILED', `Limrun instance for lease '${leaseId}' was not confirmed deleted`, { reason: 'limrun_instance_delete_unconfirmed', retriable: true, - ...(outcome.instanceId === undefined ? {} : { limrunInstanceId: outcome.instanceId }), + ...(instanceId === undefined ? {} : { limrunInstanceId: instanceId }), + ...(outcome.failure === undefined ? {} : { cause: outcome.failure }), + hint: + instanceId === undefined + ? `Release lease '${leaseId}' again once its instance create names the instance it rented.` + : `Release lease '${leaseId}' again, or ask Limrun to delete instance '${instanceId}'.`, + }, + ); +} + +/** + * The refusal an allocation gets when a teardown took its lease while the create ran. The caller passes the + * disposal's outcome rather than reading the owner, so this answers with what that teardown came to — + * including a delete Limrun did not confirm, which the refused caller is the last route left to hear about. + */ +export function allocationRefusedForTake( + lease: DeviceLease, + by: LimrunTeardownRoute, + outcome: LimrunDisposalOutcome, +): never { + const { instanceId } = outcome; + throw new AppError( + 'DEVICE_IN_USE', + `Limrun lease '${lease.leaseId}' was released while its instance was being prepared`, + { + reason: 'limrun_lease_released_mid_allocation', + ...(instanceId === undefined + ? { hint: `Instance create for lease '${lease.leaseId}' may still rent one. Check Limrun.` } + : { + limrunInstanceId: instanceId, + hint: `Instance '${instanceId}' was ended by the ${by} of this lease. Open a new lease.`, + }), + ...(outcome.cleanup === 'uncertain' + ? { retriable: true, ...(outcome.failure === undefined ? {} : { cause: outcome.failure }) } + : {}), + }, + ); +} + +/** + * What an allocation reports when the create it waited on failed and left an instance running. The hint names + * the instance when the create got as far as renting one, and the lease when it did not. + */ +export function createLeftInstanceRunning( + lease: DeviceLease, + outcome: LimrunDisposalOutcome, +): never { + const { instanceId } = outcome; + throw new AppError( + 'COMMAND_FAILED', + `Limrun lease '${lease.leaseId}' could not be allocated, and its instance could not be deleted`, + { + reason: 'limrun_instance_delete_unconfirmed', + retriable: true, + ...(instanceId === undefined ? {} : { limrunInstanceId: instanceId }), ...(outcome.failure === undefined ? {} : { cause: outcome.failure }), - hint: `Release lease '${leaseId}' again, or ask Limrun to delete instance '${outcome.instanceId ?? leaseId}'.`, + hint: + instanceId === undefined + ? `Ask Limrun what lease '${lease.leaseId}' rented — this runtime could not name the instance.` + : `Ask Limrun to delete instance '${instanceId}' — this runtime could not confirm the delete.`, }, ); } + +/** + * What a daemon close reports for every lease it could not confirm ended. A lease whose create never named an + * instance is counted the same as one whose delete was refused: this process moved on without stopping it, and + * the operator needs both the count and the reason. + */ +export function cleanupLeftInstancesRunning(outcomes: readonly LimrunDisposalOutcome[]): never { + const leftRunning = outcomes.map((outcome) => outcome.instanceId); + const known = leftRunning.filter((id) => id !== undefined); + const plural = outcomes.length === 1 ? '' : 's'; + throw new AppError( + 'COMMAND_FAILED', + `Limrun cleanup left ${String(outcomes.length)} lease${plural} unended, ${String(known.length)} with a known instance`, + { + reason: 'limrun_instance_delete_unconfirmed', + ...(known.length === 0 ? {} : { limrunInstanceIds: known }), + ...(outcomes[0]?.failure === undefined ? {} : { cause: outcomes[0].failure }), + ...(known.length === 0 + ? { hint: unendedLeaseHint(outcomes) } + : { hint: `Ask Limrun to delete: ${known.join(', ')}.` }), + }, + ); +} + +/** What a close says when the work it could not finish has no instance to name yet: the lease and the work. */ +function unendedLeaseHint(outcomes: readonly LimrunDisposalOutcome[]): string { + return `Limrun may still bill work this runtime could not name: ${outcomes + .map((outcome) => outcome.failure ?? 'an unconfirmed delete') + .join('; ')}.`; +} diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index 48347b4acb..e8cb815126 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -45,10 +45,16 @@ const state = vi.hoisted(() => ({ }, })), androidDelete: vi.fn(async (_instanceId: string) => { + await state.deleteGate; if (state.deleteFails) throw new Error('Limrun refused to delete the instance'); }), /** Made to reject by a case whose instance delete is refused, the one failure a lease may not swallow. */ deleteFails: false, + /** + * Held by a case whose instance delete must answer later than the deadline its own route gives it, which is + * the shape a DELETE whose response body never arrives has. + */ + deleteGate: undefined as Promise | undefined, keepAlive: vi.fn(), /** Instance-client connections the provider made, so one lease can be shown to make one. */ connectCount: 0, @@ -64,6 +70,10 @@ const state = vi.hoisted(() => ({ tunnelGate: undefined as Promise | undefined, /** Made to throw by a case whose session connection fails after its instance was rented. */ connectFails: false, + /** Uploaded apps the catalog answers for a case that asks for an app to preinstall. */ + uploadedApps: [] as unknown[], + /** What the device reports installed, for a case whose app identifier cannot be resolved. */ + installedApps: [] as Array<{ id: string; name?: string }>, /** Destination tunnels and ADB tunnels this runtime asked the SDK to open. */ captureStarts: 0, adbTunnelStarts: 0, @@ -88,7 +98,10 @@ vi.mock('@limrun/api', () => ({ list: vi.fn(async () => ({ getPaginatedItems: () => [] })), delete: state.androidDelete, }; - readonly assets = { getOrUpload: vi.fn(), list: vi.fn() }; + readonly assets = { + getOrUpload: vi.fn(), + list: vi.fn(async () => state.uploadedApps), + }; }, })); @@ -153,6 +166,9 @@ test('two allocations of one lease rent one instance and start one keep-alive ti assert.equal(started.mock.calls.length, 1); } finally { started?.mockRestore(); + // Whatever this case asserted, the create is answered before it is torn down, so a failed assertion + // surfaces as a failure rather than as a shutdown that waits out its create budget. + reportCreate(); await runtime.shutdown(); state.createGate = undefined; } @@ -206,6 +222,7 @@ test('a release that arrives before the create answered ends the instance that c assert.equal(state.disconnect.mock.calls.length - disconnects, 1); } finally { started?.mockRestore(); + reportCreate(); await runtime.shutdown(); state.createGate = undefined; } @@ -251,6 +268,7 @@ test('a shutdown while the create runs ends the instance that create made', asyn ); } finally { started?.mockRestore(); + reportCreate(); state.createGate = undefined; } }); @@ -364,6 +382,8 @@ test('two allocations that ask for traffic logs open one destination tunnel', as // refusal would be reported over the capture that is working. assert.equal(state.captureStarts - captures, 1); } finally { + // Opened before teardown for the same reason as the create gate above. + reportTunnel(); state.tunnelGate = undefined; await runtime.shutdown(); } @@ -407,6 +427,7 @@ test('a release that arrives while a tunnel is opening ends the tunnel that answ `expected the capture allocation to be refused for the release, got ${String(error)}`, ); } finally { + reportTunnel(); state.tunnelGate = undefined; await runtime.shutdown(); } @@ -492,6 +513,12 @@ type WarningRecord = Readonly<{ */ const LOCAL_TEARDOWN_WAIT_MS = 60_000; +/** Longer than one delete's deadline, which is what the silent-delete case waits out. */ +const DELETE_ANSWER_WAIT_MS = 40_000; + +/** Longer than a disposal's budget for its create, which is what the late-create case waits out. */ +const CREATE_ANSWER_WAIT_MS = 130_000; + /** * Why one racing call refused, captured from the moment it starts. Two routes can answer in either order, * and the one that answers first must not leave the other's refusal unobserved. @@ -742,6 +769,134 @@ test('a network capture that cannot start is an unavailable artifact with its ty } }); +/** + * A DELETE whose response body never arrives answers later than any route may wait: the SDK clears its fetch + * timer before it reads the body, so the request itself is what has to be bounded. + */ +test('a release refuses a delete that never answers, within that delete’s own deadline', async () => { + const { deletes, disconnects } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-silent-delete'); + let reportDelete!: () => void; + state.deleteGate = new Promise((resolve) => { + reportDelete = resolve; + }); + vi.useFakeTimers(); + try { + await runtime.leaseLifecycle.allocate?.(lease); + const refused = refusalOf(runtime.leaseLifecycle.release?.(lease)); + await vi.advanceTimersByTimeAsync(DELETE_ANSWER_WAIT_MS); + const error = await refused; + // The lease is still held: Limrun never confirmed the instance was gone. + assert.ok( + error instanceof AppError && + error.details?.reason === 'limrun_instance_delete_unconfirmed' && + error.details.limrunInstanceId === 'android-instance-1' && + /delete/i.test(String(error.details.cause)), + `expected the release to refuse within its delete deadline, got ${String(error)}`, + ); + // What that delete goes on to answer is still observed: its late confirmation ends the lease. + reportDelete(); + await vi.advanceTimersByTimeAsync(0); + } finally { + reportDelete(); + state.deleteGate = undefined; + vi.useRealTimers(); + await runtime.shutdown(); + } + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); +}); + +/** A create that answers after its disposal gave up still owes the end of whatever it names. */ +test('a create that answers after its disposal gave up ends the instance it rents', async () => { + const { creates, deletes, connects, disconnects } = limrunActivity(); + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-late-create'); + try { + const refusedAllocation = refusalOf(runtime.leaseLifecycle.allocate?.(lease)); + await waitForCreate(creates + 1); + const refusedRelease = refusalOf(runtime.leaseLifecycle.release?.(lease)); + vi.useFakeTimers(); + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS); + // The release refused rather than waiting longer, and the create has still not named anything. + assert.ok( + (await refusedRelease) instanceof AppError, + 'expected the release to refuse once its create budget expired', + ); + assert.deepEqual(deletedInstances(deletes), []); + // The POST answers after that: the instance it names, and the session built on it, belong to the + // disposal that is already over, so that disposal ends them. + reportCreate(); + await vi.advanceTimersByTimeAsync(0); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + assert.ok((await refusedAllocation) instanceof AppError); + } finally { + vi.useRealTimers(); + reportCreate(); + state.createGate = undefined; + await runtime.shutdown(); + } + assert.equal(state.androidCreate.mock.calls.length - creates, 1); + assert.equal(state.connectCount - connects, 1); +}); + +/** A session built before a later failure is a resource its disposal has to release, not drop. */ +test('a create that fails after its session was built disconnects that session', async () => { + const { deletes, disconnects } = limrunActivity(); + state.uploadedApps = [{ id: 'asset-1', name: 'com.example.app.apk', md5: 'md5', os: 'android' }]; + // The app is installed under something this provider cannot match unambiguously. + state.installedApps = [{ id: 'com.other.app' }, { id: 'com.other.app.fork' }]; + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + try { + const error = await refusalOf( + runtime.leaseLifecycle.allocate?.(androidLease('lease-app-unresolved'), { + initialApp: 'com.example.app.apk', + }), + ); + assert.ok( + error instanceof AppError, + `expected the app resolution to fail, got ${String(error)}`, + ); + // The instance it rented is deleted, and so is the connection the session built beside it. + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + } finally { + state.uploadedApps = []; + state.installedApps = []; + await runtime.shutdown(); + } +}); + +/** A joiner with valid flags is not refused for another caller’s refused flag. */ +test('a joiner whose own flags are valid allocates despite another caller’s refused flag', async () => { + const { creates } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-mixed-flags'); + try { + // The first call carries a flag this provider reads no field for; the second carries none. + const refused = refusalOf( + runtime.leaseLifecycle.allocate?.(lease, { flags: { providerOsVersion: '15' } }), + ); + const joined = await runtime.leaseLifecycle.allocate?.(lease); + const error = await refused; + assert.ok( + error instanceof AppError && error.code === 'INVALID_ARGS', + `expected the refused caller to be refused for its own flag, got ${String(error)}`, + ); + assert.equal(joined?.limrunInstanceId, 'android-instance-1'); + } finally { + await runtime.shutdown(); + } + // Only the valid caller's lease was billed, and only once. + assert.equal(state.androidCreate.mock.calls.length - creates, 1); +}); + function completedRequest(url: string, error?: string): unknown { return { type: 'complete', @@ -811,7 +966,7 @@ function testDependencies(): LimrunRuntimeDependencies { list: async () => [], }), inferAppName: async () => 'Example', - listApps: async (_adb: LimrunAdbExecutor, _filter: AppsFilter) => [], + listApps: async (_adb: LimrunAdbExecutor, _filter: AppsFilter) => state.installedApps, getForegroundApp: async () => (state.foreground ? { ...state.foreground } : undefined), getKeyboardState: async () => ({ visible: false, inputOwner: 'unknown' as const }), dismissKeyboard: async () => ({ diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index 949916f9a7..590f674412 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -15,7 +15,6 @@ import type { import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; import { - cleanupLimrunAndroidAdbTunnel, configureLimrunAndroidPortReverse, createLimrunAndroidInteractor, createLimrunAndroidSession, @@ -59,25 +58,10 @@ import type { LimrunTeardownRoute, } from './lease-ownership.ts'; import type { LimrunCaptureLifecycle } from './capture-lifecycle.ts'; +import type { LimrunDisposalHost } from './lease-release.ts'; const KEEP_ALIVE_INTERVAL_MS = 30_000; -/** - * How long a teardown waits for the create to say what it rented. The create itself keeps Limrun's own - * timeouts: this bounds only the teardown's wait, so an instance is never held open by a create that - * stopped answering, and the owner is kept for the next route to end what that create rents. - */ -const CREATE_ANSWER_BUDGET_MS = 120_000; - -/** - * How long a teardown waits for the local resources it started beside the delete: the instance connection - * and the Android ADB endpoint, whose `adb connect` the SDK runs without a deadline. - */ -const LOCAL_TEARDOWN_BUDGET_MS = 45_000; - -/** How long a teardown waits for a capture's HAR record, which is reporting and never a reason to hold a lease. */ -const CAPTURE_RECORD_BUDGET_MS = 15_000; - export type LimrunRuntimeOptions = { /** Organization API key. It creates and deletes instances for platforms without `instances`. */ apiKey?: string; @@ -304,7 +288,10 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { // plugin's close is where an operator learns what a daemon close left running, so the instance and // the provider's refusal travel on the error rather than being dropped. const unended = outcomes.filter((outcome) => outcome.cleanup === 'uncertain'); - if (unended.length > 0) throw cleanupLeftInstancesRunning(unended); + if (unended.length > 0) { + const { cleanupLeftInstancesRunning } = await import('./lease-release.ts'); + throw cleanupLeftInstancesRunning(unended); + } } private async allocate( @@ -314,12 +301,9 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (lease.leaseProvider !== this.provider) return undefined; const platform = platformForLimrunLeaseBackend(lease.backend); if (!platform) return undefined; - // Joining a lease somebody else allocated, so this call answers for flags of its own, and it settles - // them before it reads any state. Nothing else awaits on this route: the lookup below is the last one. - if (this.leases.has(lease.leaseId)) await this.rejectRefusedProfileFields(context); // Published without awaiting anything, so a lease is never unowned while its allocation is at its first // await: a release or a shutdown arriving there finds this owner and takes it, and the create that owner - // stands on checks this call's own profile flags before it bills anything. + // stands on answers the caller's own profile flags before it bills anything. let owner = this.leases.get(lease.leaseId); if (owner === undefined) { // Nothing between this lookup and the publication below awaits, so two allocations of one lease can @@ -327,18 +311,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (this.shutDown) throw allocationRefusedAfterShutdown(lease); owner = this.beginAllocation(lease, platform, context); } - if (owner.state.phase === 'allocating') { - const create = await owner.creation; - if (create.status === 'failed') { - // A create that failed has no release coming for it, so this call is the only route left to end an - // instance it made before failing. What it came to outranks why the create did not finish: an - // instance Limrun is still billing is the caller's problem first. - const outcome = await this.settleFailedCreate(owner); - if (outcome?.cleanup === 'uncertain') throw createLeftInstanceRunning(lease, outcome); - throw create.error; - } - this.goLive(owner, create.session, create.appAlias); - } + await this.joinCreation(owner, context); // A repeat allocation can ask for the capture, or retry one that could not start. A lease a teardown has // already taken starts nothing: `captures.start` reads the owner's own live session, and what this lease // has is that teardown's to end. @@ -353,12 +326,73 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { const taken = owner.state; if (taken.phase === 'releasing') { const outcome = await taken.disposal; + const { allocationRefusedForTake } = await import('./lease-release.ts'); throw allocationRefusedForTake(lease, taken.by, outcome); } const session = this.requireLiveSession(owner); return { limrunInstanceId: session.instanceId, device: session.device }; } + /** + * Brings one owner to the point its live session answers, and refuses this caller only for what it asked + * for itself. + */ + private async joinCreation( + owner: LimrunLeaseOwner, + context?: LeaseLifecycleContext, + ): Promise { + if (owner.state.phase === 'live') { + // A lease this runtime allocated earlier: this call answers for flags of its own, and settles them + // before it reads any more state. + await this.rejectRefusedProfileFields(context); + return; + } + if (owner.state.phase !== 'allocating') return; + const create = await this.creationFor(owner, context); + if (create === undefined) return; + if (create.status === 'ready') { + this.goLive(owner, create.session, create.appAlias); + return; + } + if (create.status === 'refused_caller') throw create.error; + // A create that failed has no release coming for it, so this call is the only route left to end an + // instance it made before failing. What it came to outranks why the create did not finish: an instance + // Limrun is still billing is the caller's problem first. + const outcome = await this.settleFailedCreate(owner); + if (outcome?.cleanup === 'uncertain') { + const { createLeftInstanceRunning } = await import('./lease-release.ts'); + throw createLeftInstanceRunning(owner.lease, outcome); + } + throw create.error; + } + + /** + * The create one caller is entitled to: the one its lease's owner stands on, unless that create refused the + * caller that started it for a flag this provider reads no field for. That refusal billed nothing, so it is + * the other caller's answer and not this lease's: this caller answers for its own flags first — a refusal of + * its own is its own to throw — and a caller whose flags this provider does read asks for the create the + * refusal foreclosed and makes it the lease's creation, on the owner the lease already has. Any later joiner + * of the same refusal follows that create rather than renting a second instance. + */ + private async creationFor( + owner: LimrunLeaseOwner, + context?: LeaseLifecycleContext, + ): Promise { + for (;;) { + const started = owner.creation; + const create = await started; + if (create.status !== 'refused_caller') return create; + await this.rejectRefusedProfileFields(context); + // Another joiner of the same refusal replaced it while this caller was being answered: join that create. + if (owner.creation !== started) continue; + // A teardown took the lease in the meantime, so what this lease has is that route's to end. + if (owner.state.phase !== 'allocating') return undefined; + const retried = this.createForLease(owner, context); + owner.creation = retried; + return await retried; + } + } + /** The profile flags this provider reads nothing for, refused for a lease this runtime already holds. */ private async rejectRefusedProfileFields(context?: LeaseLifecycleContext): Promise { // Loaded here rather than eagerly so this package entry's declared closure stays unchanged. The create @@ -389,6 +423,8 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { creation, state: { phase: 'allocating' }, rentedInstanceId: undefined, + session: undefined, + localRelease: undefined, keepAlive: undefined, capture: undefined, appAlias: undefined, @@ -421,13 +457,20 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { resolveRequestedLimrunAppAsset, } = await import('./session-allocation.ts'); // The first step of the create, so an allocation that published its owner before it could await - // anything still bills nothing for flags this provider reads no field for. - rejectRefusedLimrunProfileFields(context); + // anything still bills nothing for flags this provider reads no field for. That refusal belongs to the + // caller that carried the flag: it says nothing about the lease, which is why the owner's creation + // carries it as `refused_caller` rather than as a failed create. + try { + rejectRefusedLimrunProfileFields(context); + } catch (error) { + return { status: 'refused_caller', error }; + } const platform = owner.platform; // Attach first, and only ask for the organization client if there is nothing to attach: an attached // instance needs no API key, and is not created with an app asset of ours. const attached = await this.attachSession(platform, owner.lease); if (attached) { + owner.session = attached; return { status: 'ready', session: attached, appAlias: undefined }; } const limrun = this.orgClient('Creating an instance'); @@ -436,10 +479,14 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { const rent = (instanceId: string): void => { owner.rentedInstanceId = instanceId; }; + // The session is recorded in the step that answers for it, so every later step of this allocation + // fails with its handles still owned: the instance client and the host ADB endpoint are released by + // whoever ends this lease, whether or not the allocation ever answered. const session = platform === 'ios' ? await allocateLimrunIosSession(params, rent) : await allocateLimrunAndroidSession(params, rent); + owner.session = session; const appAlias: LimrunAppAlias | undefined = requestedAsset && { assetName: requestedAsset.name, installedAppId: await resolvePreinstalledAppId(session, requestedAsset), @@ -530,125 +577,23 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { } /** - * Ends what one taken owner stands for: the instance its create made or rented, with every resource the - * owner holds beside it. This is the only place an instance is ended, and an owner is taken by exactly one - * route at a time, so one instance is ended exactly once. Every wait here is bounded: work that outlives - * its budget keeps running and cleans up what it makes, and nothing of it stands between this route and the - * delete that stops the instance from being billed. + * Hands one taken owner to the module that ends what it stands for. The disposal may ask only two things of + * this runtime — delete one instance and end one lease's capture — so it never reaches back into the routes + * that allocate and read one, and the lease is still given back by the take that started this disposal. */ private async disposeOwner(owner: LimrunLeaseOwner): Promise { - const { awaitLimrunWithinBudget, joinOutcomeFailures } = await import('./lifecycle-budget.ts'); - // A create that outlives this budget is reported rather than waited on: the owner is kept, so the next - // release, recovery or daemon close ends whatever that create goes on renting. - const create = await awaitLimrunWithinBudget(owner.creation, CREATE_ANSWER_BUDGET_MS); - const session = create.ok && create.value.status === 'ready' ? create.value.session : undefined; - const instanceId = session?.instanceId ?? owner.rentedInstanceId; - if (instanceId === undefined) return await this.disposeUnrented(owner, create.ok); - // The capture's record is optional reporting, and local teardown can outlive its own steps: neither is - // allowed to stand between this route and the delete that stops the instance from being billed, so both - // are started here and joined after it. A lease that never asked for a capture has no lifecycle to end, - // so its disposal never loads one. - const capture = this.capturesInstance?.end(owner); - const local = this.releaseOwnerResources(owner, session); - const instance = await this.endInstance(owner, session, instanceId); - const stuck = await this.awaitLocalTeardown(local); - // The record is optional reporting joined last: a capture that refuses to be written cannot keep a - // release from answering for the instance it ended, and is still listable once it does come to. - const joined = - capture === undefined - ? undefined - : await awaitLimrunWithinBudget(capture, CAPTURE_RECORD_BUDGET_MS); - return { - cleanup: instance.cleanup, - instanceId, - failure: joinOutcomeFailures(instance.failure, stuck), - capture: joined?.ok ? joined.value : undefined, - }; - } - - /** - * Ends what a disposal holds when no instance is named for the lease: its capture, and an outcome that - * keeps the lease held only while a create this route lost sight of may still rent something. - */ - private async disposeUnrented( - owner: LimrunLeaseOwner, - createAnswered: boolean, - ): Promise { - const { workOutstandingFailure } = await import('./lifecycle-budget.ts'); - const capture = await this.capturesInstance?.end(owner); - if (createAnswered) { - return { cleanup: 'none', instanceId: undefined, failure: undefined, capture }; - } - return { - cleanup: 'uncertain', - instanceId: undefined, - failure: workOutstandingFailure( - `Instance create for lease '${owner.lease.leaseId}'`, - CREATE_ANSWER_BUDGET_MS, - ), - capture, - }; - } - - /** - * Ends one owner's instance. A delete Limrun already confirmed answers for itself, an attached instance is - * the caller's own to delete, and a created one is deleted here, with that answer deciding whether the - * lease can be given back. - */ - private async endInstance( - owner: LimrunLeaseOwner, - session: LimrunRuntimeSession | undefined, - instanceId: string, - ): Promise> { - if (owner.deletedInstanceId === instanceId) return { cleanup: 'confirmed', failure: undefined }; - if ((session?.ownership ?? 'created') === 'attached') { - // An attached instance is the caller's own, and this runtime was never given the right to delete it. - return { cleanup: 'attached', failure: undefined }; - } - const failure = await this.deleteInstance(owner.platform, instanceId); - if (failure === undefined) owner.deletedInstanceId = instanceId; - return { cleanup: failure === undefined ? 'confirmed' : 'uncertain', failure }; + const { disposeLimrunLeaseOwner } = await import('./lease-release.ts'); + return await disposeLimrunLeaseOwner(this.disposalHost, owner); } - /** - * Ends the resources one owner holds besides its instance: the keep-alive beat that would otherwise ping a - * deleted instance forever, the instance connection, and the Android ADB tunnel and its reverse mappings. - * Each is released once, whichever route took the lease. - */ - private async releaseOwnerResources( - owner: LimrunLeaseOwner, - session: LimrunRuntimeSession | undefined, - ): Promise { - if (owner.keepAlive) clearInterval(owner.keepAlive); - owner.keepAlive = undefined; - if (session === undefined) return; - session.client.disconnect(); - // The tunnel's start is an `adb connect` the SDK runs without a deadline. Its cleanup retires the - // endpoint first, so a tunnel that answers late ends itself and no caller can reach it meanwhile. - if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); - } - - /** - * Reports the local teardown this route had to stop waiting for. A step that refused is reported as one - * that never answered: neither is something this runtime can still fix, and the delete that ran beside it - * has already been answered. - */ - private async awaitLocalTeardown(local: Promise): Promise { - const { awaitLimrunWithinBudget, workOutstandingFailure } = - await import('./lifecycle-budget.ts'); - const stuck = workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS); - const ended = await awaitLimrunWithinBudget( - local.then( - (): string | undefined => undefined, - (): string => stuck, - ), - LOCAL_TEARDOWN_BUDGET_MS, - ); - return ended.ok ? ended.value : stuck; - } + private readonly disposalHost: LimrunDisposalHost = { + deleteInstance: async (platform, instanceId) => + await this.deleteCreatedInstance(platform, instanceId), + endCapture: async (owner) => await this.capturesInstance?.end(owner), + }; /** One instance delete. `undefined` is Limrun's confirmation that the instance is gone. */ - private async deleteInstance( + private async deleteCreatedInstance( platform: LimrunPlatform, instanceId: string, ): Promise { @@ -932,64 +877,3 @@ function allocationRefusedAfterShutdown(lease: DeviceLease): never { }, ); } - -/** - * The refusal an allocation gets when a teardown took its lease while the create ran. The caller passes the - * disposal's outcome rather than reading the owner, so this answers with what that teardown came to — - * including a delete Limrun did not confirm, which the refused caller is the last route left to hear about. - */ -function allocationRefusedForTake( - lease: DeviceLease, - by: LimrunTeardownRoute, - outcome: LimrunDisposalOutcome, -): never { - const { instanceId } = outcome; - throw new AppError( - 'DEVICE_IN_USE', - `Limrun lease '${lease.leaseId}' was released while its instance was being prepared`, - { - reason: 'limrun_lease_released_mid_allocation', - ...(instanceId === undefined - ? {} - : { - limrunInstanceId: instanceId, - hint: `Instance '${instanceId}' was ended by the ${by} of this lease. Open a new lease.`, - }), - ...(outcome.cleanup === 'uncertain' - ? { retriable: true, ...(outcome.failure === undefined ? {} : { cause: outcome.failure }) } - : {}), - }, - ); -} - -/** What an allocation reports when the create it waited on failed and left an instance running. */ -function createLeftInstanceRunning(lease: DeviceLease, outcome: LimrunDisposalOutcome): never { - throw new AppError( - 'COMMAND_FAILED', - `Limrun lease '${lease.leaseId}' could not be allocated, and its instance could not be deleted`, - { - reason: 'limrun_instance_delete_unconfirmed', - retriable: true, - ...(outcome.instanceId === undefined ? {} : { limrunInstanceId: outcome.instanceId }), - ...(outcome.failure === undefined ? {} : { cause: outcome.failure }), - hint: `Ask Limrun to delete instance '${outcome.instanceId ?? lease.leaseId}' — this runtime could not confirm the delete.`, - }, - ); -} - -/** What a daemon close reports for every instance it could not stop before the process moved on. */ -function cleanupLeftInstancesRunning(outcomes: readonly LimrunDisposalOutcome[]): never { - const leftRunning = outcomes - .map((outcome) => outcome.instanceId) - .filter((id) => id !== undefined); - throw new AppError( - 'COMMAND_FAILED', - `Limrun cleanup could not delete ${String(leftRunning.length)} instance${leftRunning.length === 1 ? '' : 's'}`, - { - reason: 'limrun_instance_delete_unconfirmed', - ...(leftRunning.length === 0 ? {} : { limrunInstanceIds: leftRunning }), - ...(outcomes[0]?.failure === undefined ? {} : { cause: outcomes[0].failure }), - hint: `Ask Limrun to delete: ${leftRunning.join(', ')}.`, - }, - ); -} From e5ebf7682ed545a452a26fdfbf216aadbc016b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 19:08:17 +0100 Subject: [PATCH 03/11] fix(limrun): answer for a joining caller and never end one lease twice Six ways a lease could be torn down wrongly, or a caller handed a session it was never entitled to, each behind a test that fails without its fix. A joiner with profile flags this provider reads no field for is now refused by the create it joins, not handed that create's session: validation runs before the join for every lease this runtime already holds, while a new lease keeps its validation as the first step of its own create so publication stays free of any await. A late end joins the delete already in flight. The owner records the request it has with Limrun, so the continuation of a create that answered after its disposal gave up waits on that request instead of sending a second one for the same instance. The route an ADB teardown ends its serial through is scoped to that teardown. The reverse manager dispatches through it, and a caller command that reaches the same route while the teardown is mid-request carries no marker: it is refused by the session's closed public route, so nothing gets mapped behind the teardown's one listing. A disposal reports every step it stopped waiting for. A rejected local teardown is reported rather than thrown; an expired capture record contributes its own failure; and a daemon close names why each lease it could not end is still held, not only the first. Two disposal routes awaited a capture end with no deadline at all, and a repeat allocation joined a stranger's destination-tunnel handshake unbounded; all three now answer within a budget. Both continuations that outlive every budget carry rejection handlers, so work this runtime deliberately stopped waiting for cannot reach the process as an unhandled rejection. --- packages/provider-limrun/src/android.test.ts | 42 ++++++ packages/provider-limrun/src/android.ts | 23 ++- .../src/capture-lifecycle.test.ts | 68 +++++++++ .../provider-limrun/src/capture-lifecycle.ts | 17 ++- .../provider-limrun/src/client-options.ts | 4 + .../provider-limrun/src/lease-ownership.ts | 8 + .../provider-limrun/src/lease-release.test.ts | 89 +++++++++++ packages/provider-limrun/src/lease-release.ts | 79 +++++++--- packages/provider-limrun/src/runtime.test.ts | 140 +++++++++--------- packages/provider-limrun/src/runtime.ts | 30 ++-- 10 files changed, 392 insertions(+), 108 deletions(-) create mode 100644 packages/provider-limrun/src/capture-lifecycle.test.ts create mode 100644 packages/provider-limrun/src/lease-release.test.ts diff --git a/packages/provider-limrun/src/android.test.ts b/packages/provider-limrun/src/android.test.ts index 42942ce613..85163a9971 100644 --- a/packages/provider-limrun/src/android.test.ts +++ b/packages/provider-limrun/src/android.test.ts @@ -58,6 +58,48 @@ test('a command that arrives while cleanup holds its serial is refused', async ( expect(tunnels.start).toHaveBeenCalledOnce(); }); +/** + * The serial a cleanup is ending is reached only by the cleanup's own dispatches. A reverse command that + * arrives while that cleanup is mid-request goes back through the session's closed public route, so nothing + * gets mapped at a device that is being disconnected, behind the cleanup's one listing. + */ +test('a reverse command that arrives during cleanup is refused, not dispatched at the ending serial', async () => { + const runs: string[][] = []; + let reportList!: () => void; + reverse.listGate = new Promise((resolve) => { + reportList = resolve; + }); + const deps = androidDependencies(runs); + // The route the reverse manager dispatches through, which is what production hands it. + let reverseRoute!: (args: readonly string[]) => Promise; + deps.android.createPortReverse = async (adb) => { + reverseRoute = adb; + return reverse; + }; + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, deps); + // The first command raises the tunnel this session owns, giving cleanup a serial to end. + await session.adbProvider.exec(['shell', 'echo', 'ready']); + const dispatched = runs.length; + try { + // Cleanup holds its listing open: its teardown executor is published at the serial it is ending. + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + await vi.waitFor(() => expect(reverse.list).toHaveBeenCalledOnce()); + + await expect(reverseRoute(['reverse', 'tcp:9001', 'tcp:8001'])).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + + reportList(); + await cleanup; + } finally { + reverse.listGate = undefined; + } + // Nothing mapped an endpoint behind the cleanup's back: no reverse command was dispatched at all, because + // the listing and the removals are stubbed and the caller's command never reached the device. + expect(runs.slice(dispatched).filter((argv) => argv.includes('reverse'))).toEqual([]); +}); + const tunnels = vi.hoisted(() => ({ /** * Held by a case that has to keep a tunnel in flight, which is the window a session's cleanup has to diff --git a/packages/provider-limrun/src/android.ts b/packages/provider-limrun/src/android.ts index 9beadc6590..041f08ab6d 100644 --- a/packages/provider-limrun/src/android.ts +++ b/packages/provider-limrun/src/android.ts @@ -1,4 +1,5 @@ import path from 'node:path'; +import { AsyncLocalStorage } from 'node:async_hooks'; import type Limrun from '@limrun/api'; import { createInstanceClient as createAndroidInstanceClient, @@ -184,7 +185,12 @@ export async function cleanupLimrunAndroidAdbTunnel(session: LimrunAndroidSessio // ask for. session.adbReverseExecutor = async (args, options) => await runLimrunAndroidAdbAtSerial(session, serial, args, options); - await cleanupAndroidPortReverse(session); + // The teardown's marker covers this route and every dispatch this cleanup starts from it, including a + // reverse manager that answers a later request of its own inside the same await chain. A caller command + // that reaches this route through any other path carries no marker. + await limrunAdbTeardownScope.run(true, async () => { + await cleanupAndroidPortReverse(session); + }); await session.dependencies.host .runAdb(session.dependencies.android.hostAdbInvocation(['disconnect', serial]), { allowFailure: true, @@ -239,14 +245,25 @@ async function runLimrunAndroidAdb( /** * The reverse manager's own route to the device. It is the session's route while the session is live, and * cleanup's route of its own once the session is retired: clearing reverse mappings means reaching the device - * that is about to be disconnected, and that has to stay possible after every caller of this session was refused. + * that is about to be disconnected, and that has to stay possible after every caller of this session was + * refused. Only the cleanup's own dispatches carry its teardown scope, so no caller command is ever + * dispatched at the serial the teardown is ending. */ +/** + * Whether one dispatch belongs to a session's teardown: true only for the dispatches that teardown itself + * started. The reverse manager is given this predicate as its route, and a provider that hands a caller + * command to it — a reverse command queued inside an operation of its own — carries no marker, so the + * manager cannot have a caller's mapping bound at a serial that is on its way out. A marker set by one + * session never authorises another session's route. + */ +const limrunAdbTeardownScope = new AsyncLocalStorage(); + async function runLimrunAndroidReverseAdb( session: LimrunAndroidAdbSession, args: readonly string[], options?: LimrunAdbCommandOptions, ): Promise { - const atSerial = session.adbReverseExecutor; + const atSerial = limrunAdbTeardownScope.getStore() ? session.adbReverseExecutor : undefined; if (atSerial !== undefined) return await atSerial(args, options); return await runLimrunAndroidAdb(session, args, options); } diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts new file mode 100644 index 0000000000..aaa26a892f --- /dev/null +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -0,0 +1,68 @@ +import { test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { LimrunCaptureLifecycle, type LimrunCaptureHost } from './capture-lifecycle.ts'; +import type { LimrunLeaseOwner } from './lease-ownership.ts'; +import type { NetworkCapture } from './network-har.ts'; + +/** Longer than the lifecycle's budget for a destination tunnel that is still being opened. */ +const CAPTURE_OPEN_WAIT_MS = 25_000; + +/** + * A repeat allocation joins the start its owner is already running, within the budget a read or a teardown + * gives that same handshake. + */ +test('a repeat allocation stops waiting on a destination tunnel that never answers', async () => { + vi.useFakeTimers(); + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + let joined = false; + const joining = lifecycle.start(owner, undefined).then(() => { + joined = true; + }); + try { + await vi.advanceTimersByTimeAsync(CAPTURE_OPEN_WAIT_MS); + await joining; + } finally { + vi.useRealTimers(); + } + + assert.ok(joined, 'expected the joining allocation to stop waiting on the stalled tunnel'); + // What the stalled start goes on to open stays the owner's to publish, and its teardown's to end: the + // joiner leaves the capture where the allocation that started it put it. + assert.equal(owner.capture?.phase, 'starting'); +}); + +function captureHost(held: readonly LimrunLeaseOwner[] = []): LimrunCaptureHost { + return { + harDir: undefined, + liveSessionOf: () => undefined, + heldOwners: () => held, + }; +} + +/** An owner whose lease has a destination tunnel that has not answered, and never will. */ +function ownerWithCapture(): LimrunLeaseOwner { + const lease = { + leaseId: 'lease-capture-join', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease; + const opening = new Promise(() => {}); + return { + lease, + platform: 'android', + creation: Promise.resolve({ status: 'ready', session: undefined, appAlias: undefined }), + state: { phase: 'live' }, + rentedInstanceId: undefined, + session: undefined, + capture: { phase: 'starting', opening }, + appAlias: undefined, + deletedInstanceId: undefined, + } as unknown as LimrunLeaseOwner; +} diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts index 7e3f99d78c..f6531d4388 100644 --- a/packages/provider-limrun/src/capture-lifecycle.ts +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -50,7 +50,11 @@ export class LimrunCaptureLifecycle { async start(owner: LimrunLeaseOwner, hostList: unknown): Promise { const held = owner.capture; if (held?.phase === 'starting') { - await held.opening; + // Joining another allocation's destination tunnel is waited for within the budget a read or a teardown + // gives that same handshake, and its outcome changes nothing here: the start keeps its own owner to + // publish to, and this lease's teardown is the route that ends whatever tunnel it opens. So a handshake + // that stops answering costs this caller its wait, and nothing else. + await awaitLimrunWithinBudget(held.opening, CAPTURE_OPEN_BUDGET_MS); return; } const harDir = this.host.harDir; @@ -107,9 +111,14 @@ export class LimrunCaptureLifecycle { // written and ended by the continuation below all the same, so it is never left without an owner. const opened = await awaitLimrunWithinBudget(state.opening, CAPTURE_OPEN_BUDGET_MS); if (opened.ok) return await endCapture(leaseId, opened.value, this.record); - void state.opening.then(async (late) => { - await endCapture(leaseId, late, this.record); - }); + void state.opening.then( + async (late) => { + await endCapture(leaseId, late, this.record); + }, + // Observed: this continuation outlives its budget, so a late failure of it must not reach the process + // as an unhandled rejection. The tunnel it could not end stays the host's, reported by nothing else. + () => undefined, + ); return this.settled.get(leaseId); } diff --git a/packages/provider-limrun/src/client-options.ts b/packages/provider-limrun/src/client-options.ts index 1de4d47db1..6b8422f937 100644 --- a/packages/provider-limrun/src/client-options.ts +++ b/packages/provider-limrun/src/client-options.ts @@ -12,6 +12,10 @@ export function limrunUserAgent(clientVersion: string): string { * Asks the SDK for one attempt at a request whose failure this runtime cannot tell apart from Limrun having * done the work: the SDK retries a lost answer by POSTing again, and a repeated instance create rents a * second instance. The SDK defaults to two retries, so a request that must not repeat carries this. + * + * The option is the SDK's request-scoped `maxRetries` (`RequestOptions`, @limrun/api 0.64.1), which every + * resource method takes as its second argument. `session-allocation.test.ts` pins that this is what the + * create honours: it drives the real SDK against a local server that drops the answer and asserts one POST. */ export const LIMRUN_SINGLE_ATTEMPT = Object.freeze({ maxRetries: 0 }); diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts index e4e51c2b48..68f3aa88fc 100644 --- a/packages/provider-limrun/src/lease-ownership.ts +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -57,6 +57,14 @@ export type LimrunLeaseOwner = { appAlias: LimrunAppAlias | undefined; /** The instance Limrun has already confirmed deleted, so a retried disposal never bills a second delete. */ deletedInstanceId: string | undefined; + /** + * The delete this owner has in flight: the instance it names, and the request Limrun has not answered. A + * route that owes the end of that instance waits on this request rather than sending Limrun a second one — + * a delete whose answer this disposal stopped waiting for is still the delete that instance is getting. + */ + pendingDelete: + | Readonly<{ instanceId: string; requested: Promise }> + | undefined; }; /** What one lease has live in this runtime, whatever its platform. */ diff --git a/packages/provider-limrun/src/lease-release.test.ts b/packages/provider-limrun/src/lease-release.test.ts new file mode 100644 index 0000000000..a64d78e7e5 --- /dev/null +++ b/packages/provider-limrun/src/lease-release.test.ts @@ -0,0 +1,89 @@ +import { test, vi, expect } from 'vitest'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { disposeLimrunLeaseOwner, type LimrunDisposalHost } from './lease-release.ts'; +import type { LimrunCreateOutcome, LimrunLeaseOwner } from './lease-ownership.ts'; + +/* + * Each wait is longer than the budgets it steps past: a disposal's create wait, and everything a disposal + * left without a create answer may still wait for — one delete, the local teardown, one capture record. + */ +const CREATE_ANSWER_WAIT_MS = 130_000; +const DISPOSAL_SETTLE_WAIT_MS = 110_000; + +/** + * A create that answers after its disposal stopped waiting owes its end to the disposal already over. If that + * disposal's delete is still with Limrun, the late end joins it: one instance is deleted by one request. + */ +test('a create that answers late joins the delete its disposal already started', async () => { + // Loaded before the clock is replaced, so the disposal's own budget timers are fake timers. + await import('./lifecycle-budget.ts'); + let reportDelete!: () => void; + const deleteHost = vi.fn(async () => { + await new Promise((resolve) => { + reportDelete = resolve; + }); + return undefined; + }); + const host: LimrunDisposalHost = { + deleteInstance: deleteHost, + endCapture: async () => undefined, + }; + let reportCreate!: () => void; + const creation = new Promise((resolve) => { + // A create that rented the instance and then failed, after its disposal had given up on it. + reportCreate = () => resolve({ status: 'failed', error: new Error('instance boot failed') }); + }); + const owner = ownerWithRentedInstance(creation); + + vi.useFakeTimers(); + try { + const disposal = disposeLimrunLeaseOwner(host, owner); + // The disposal stops waiting on the create and asks Limrun once to end the instance it did rent. + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS); + expect(deleteHost).toHaveBeenCalledOnce(); + // Its own delete then runs out its deadline, and the disposal refuses its caller. + await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + await expect(disposal).resolves.toMatchObject({ cleanup: 'uncertain' }); + + // The create answers now. The end it owes is this disposal's, and it joins the request Limrun already + // has for this instance rather than sending a second one. + reportCreate(); + await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + expect(deleteHost).toHaveBeenCalledOnce(); + + // The one request confirming is enough: the joined end records the instance as deleted. + reportDelete(); + await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + } finally { + vi.useRealTimers(); + } + expect(owner.deletedInstanceId).toBe('android-instance-1'); +}); + +/** An owner whose create has rented an instance but has not answered, taken by a disposal. */ +function ownerWithRentedInstance(creation: Promise): LimrunLeaseOwner { + const lease = { + leaseId: 'lease-release-join', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease; + return { + lease, + platform: 'android', + creation, + state: { phase: 'releasing', by: 'release' }, + rentedInstanceId: 'android-instance-1', + session: undefined, + localRelease: undefined, + keepAlive: undefined, + capture: undefined, + appAlias: undefined, + deletedInstanceId: undefined, + pendingDelete: undefined, + } as unknown as LimrunLeaseOwner; +} diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index 6ddc14cea8..b554024210 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -65,7 +65,8 @@ export async function disposeLimrunLeaseOwner( host: LimrunDisposalHost, owner: LimrunLeaseOwner, ): Promise { - const { awaitLimrunWithinBudget, joinOutcomeFailures } = await import('./lifecycle-budget.ts'); + const { awaitLimrunWithinBudget, joinOutcomeFailures, workOutstandingFailure } = + await import('./lifecycle-budget.ts'); const create = await awaitLimrunWithinBudget(owner.creation, CREATE_ANSWER_BUDGET_MS); // The create answered too late for this disposal to wait on it; what it goes on to name is still this // disposal's to end. @@ -89,7 +90,13 @@ export async function disposeLimrunLeaseOwner( return { cleanup: instance.cleanup, instanceId, - failure: joinOutcomeFailures(instance.failure, stuck), + failure: joinOutcomeFailures( + instance.failure, + joinOutcomeFailures( + stuck, + joined.ok ? undefined : workOutstandingFailure('Capture record', CAPTURE_RECORD_BUDGET_MS), + ), + ), capture: joined.ok ? joined.value : undefined, }; } @@ -103,16 +110,20 @@ async function disposeUnansweredCreate( host: LimrunDisposalHost, owner: LimrunLeaseOwner, ): Promise { - const { awaitLimrunWithinBudget, joinOutcomeFailures, workOutstandingFailure } = - await import('./lifecycle-budget.ts'); - const capture = host.endCapture(owner); + const { joinOutcomeFailures, workOutstandingFailure } = await import('./lifecycle-budget.ts'); + const capture = awaitEndedCapture(host, owner); const outstanding = workOutstandingFailure( `Instance create for lease '${owner.lease.leaseId}'`, CREATE_ANSWER_BUDGET_MS, ); - owner.creation.then(() => { - void endWhatTheCreateNamed(host, owner); - }); + // Observed on both branches: this continuation outlives every budget, so a failure of it must not reach + // the process as an unhandled rejection. + owner.creation.then( + () => { + void endWhatTheCreateNamed(host, owner).catch(() => undefined); + }, + () => undefined, + ); const instanceId = owner.rentedInstanceId; if (instanceId === undefined) { return { @@ -125,17 +136,13 @@ async function disposeUnansweredCreate( // An instance is already named, so this disposal ends the billing it can see and leaves the create's silence // as what keeps the lease held. const instance = await endInstance(host, owner, owner.session, instanceId); - const local = await awaitLimrunWithinBudget( - releaseOwnerResources(owner), - LOCAL_TEARDOWN_BUDGET_MS, - ); + // Reported, not thrown: a local step that refuses is one this disposal can no longer fix, and the delete + // it ran beside has already been answered for. + const stuck = await awaitLocalTeardown(releaseOwnerResources(owner)); return { cleanup: instance.cleanup, instanceId, - failure: joinOutcomeFailures( - instance.failure, - local.ok ? outstanding : workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS), - ), + failure: joinOutcomeFailures(instance.failure, stuck ?? outstanding), capture: await capture, }; } @@ -168,10 +175,23 @@ async function disposeUnrented( cleanup: 'none', instanceId: undefined, failure: undefined, - capture: await host.endCapture(owner), + capture: await awaitEndedCapture(host, owner), }; } +/** + * Ends one lease's capture within the budget a record is given everywhere else, so a HAR write that stops + * answering cannot hold a disposal that has already ended, or owes nothing, for longer than reporting is worth. + */ +async function awaitEndedCapture( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + const { awaitLimrunWithinBudget } = await import('./lifecycle-budget.ts'); + const ended = await awaitLimrunWithinBudget(host.endCapture(owner), CAPTURE_RECORD_BUDGET_MS); + return ended.ok ? ended.value : undefined; +} + /** * Ends one owner's instance within a deadline of its own. An attached instance is the caller's own and is never * deleted here; a delete Limrun already confirmed answers for itself; and a delete that never answers keeps the @@ -188,7 +208,22 @@ async function endInstance( if ((session?.ownership ?? 'created') === 'attached') { return { cleanup: 'attached', failure: undefined }; } - const requested = host.deleteInstance(owner.platform, instanceId); + const requested = + owner.pendingDelete?.instanceId === instanceId + ? // This owner already has a delete with Limrun for this instance, however far from answering. One + // instance is deleted by one request: a second route that owes its end waits on that request. + owner.pendingDelete.requested + : host.deleteInstance(owner.platform, instanceId); + if (owner.pendingDelete?.instanceId !== instanceId) { + const inFlight = { instanceId, requested }; + owner.pendingDelete = inFlight; + // A delete that answered leaves nothing for a later route to join: a confirmed one is recorded as + // deleted, and a refused one is that later route's to retry. + const forgotten = (): void => { + if (owner.pendingDelete === inFlight) owner.pendingDelete = undefined; + }; + void requested.then(forgotten, forgotten); + } const deleted = await awaitLimrunWithinBudget(requested, DELETE_ANSWER_BUDGET_MS); if (!deleted.ok) { void requested.then((failure) => { @@ -429,7 +464,13 @@ export function cleanupLeftInstancesRunning(outcomes: readonly LimrunDisposalOut { reason: 'limrun_instance_delete_unconfirmed', ...(known.length === 0 ? {} : { limrunInstanceIds: known }), - ...(outcomes[0]?.failure === undefined ? {} : { cause: outcomes[0].failure }), + // Why every lease this close could not end, not only the first: each failure is a separate thing the + // operator may have to name to Limrun. + ...(outcomes.every((outcome) => outcome.failure === undefined) + ? {} + : { + cause: outcomes.map((outcome) => outcome.failure ?? 'an unconfirmed delete').join('; '), + }), ...(known.length === 0 ? { hint: unendedLeaseHint(outcomes) } : { hint: `Ask Limrun to delete: ${known.join(', ')}.` }), diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index e8cb815126..e2ccca46ee 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -50,23 +50,14 @@ const state = vi.hoisted(() => ({ }), /** Made to reject by a case whose instance delete is refused, the one failure a lease may not swallow. */ deleteFails: false, - /** - * Held by a case whose instance delete must answer later than the deadline its own route gives it, which is - * the shape a DELETE whose response body never arrives has. - */ + /** Held by a case whose instance delete must answer later than its own route's deadline. */ deleteGate: undefined as Promise | undefined, keepAlive: vi.fn(), /** Instance-client connections the provider made, so one lease can be shown to make one. */ connectCount: 0, - /** - * Held by a case that has to keep a create in flight after its instance was rented, which is the window - * in which a release or a daemon close has to find the instance. - */ + /** Held by a case that keeps a create in flight after its instance was rented. */ createGate: undefined as Promise | undefined, - /** - * Held by a case that has to keep a session's destination tunnel or ADB tunnel in flight, which is the - * window a release has to cover for a tunnel that has not been opened yet. - */ + /** Held by a case that keeps a destination tunnel or ADB tunnel in flight. */ tunnelGate: undefined as Promise | undefined, /** Made to throw by a case whose session connection fails after its instance was rented. */ connectFails: false, @@ -77,10 +68,7 @@ const state = vi.hoisted(() => ({ /** Destination tunnels and ADB tunnels this runtime asked the SDK to open. */ captureStarts: 0, adbTunnelStarts: 0, - /** - * The ADB provider last handed to an interactor, which is the handle a caller keeps for a session whose - * lease has since ended. - */ + /** The ADB provider last handed to an interactor: the handle kept for a lease that has since ended. */ heldAdbProvider: undefined as LimrunAdbProvider | undefined, })); @@ -241,13 +229,14 @@ test('a shutdown while the create runs ends the instance that create made', asyn testDependencies(), ); let started: ReturnType | undefined; + let closing: Promise | undefined; try { const allocating = runtime.leaseLifecycle.allocate?.(androidLease('lease-race-shutdown')); const refused = refusalOf(allocating); await waitForCreate(creates + 1); started = vi.spyOn(globalThis, 'setInterval'); - const closing = runtime.shutdown(); + closing = runtime.shutdown(); reportCreate(); await closing; @@ -270,6 +259,8 @@ test('a shutdown while the create runs ends the instance that create made', asyn started?.mockRestore(); reportCreate(); state.createGate = undefined; + // The close this case raced is also its teardown; a failed assertion before it still ends the runtime. + await (closing ?? runtime.shutdown()); } }); @@ -507,20 +498,15 @@ type WarningRecord = Readonly<{ limrunInstanceId?: string; }>; -/** - * Longer than the release's own budget for the local resources it started beside the delete, which is what - * the case waits out. The delete is asked for independently of it. - */ +/* Each wait is longer than the budget its case waits out: the local teardown, one delete, one create. */ const LOCAL_TEARDOWN_WAIT_MS = 60_000; - -/** Longer than one delete's deadline, which is what the silent-delete case waits out. */ const DELETE_ANSWER_WAIT_MS = 40_000; - -/** Longer than a disposal's budget for its create, which is what the late-create case waits out. */ const CREATE_ANSWER_WAIT_MS = 130_000; +/** A profile flag this provider reads no field for, refused for whichever caller carries it. */ +const REFUSED_FLAG = { flags: { providerOsVersion: '15' } }; /** - * Why one racing call refused, captured from the moment it starts. Two routes can answer in either order, + * Why one racing call refused, captured from the moment it starts: two routes can answer in either order, * and the one that answers first must not leave the other's refusal unobserved. */ function refusalOf(call: Promise | undefined): Promise { @@ -531,16 +517,7 @@ function refusalOf(call: Promise | undefined): Promise { } /** Where the shared instance-control mocks stand, so a case can measure only what it caused. */ -function limrunActivity(): { - creates: number; - deletes: number; - disconnects: number; - connects: number; - captures: number; - captureCloses: number; - adbTunnels: number; - tunnelCloses: number; -} { +function limrunActivity() { return { creates: state.androidCreate.mock.calls.length, deletes: state.androidDelete.mock.calls.length, @@ -769,10 +746,7 @@ test('a network capture that cannot start is an unavailable artifact with its ty } }); -/** - * A DELETE whose response body never arrives answers later than any route may wait: the SDK clears its fetch - * timer before it reads the body, so the request itself is what has to be bounded. - */ +// The SDK clears a request's timer before it reads the body, so the DELETE itself is what gets a deadline. test('a release refuses a delete that never answers, within that delete’s own deadline', async () => { const { deletes, disconnects } = limrunActivity(); const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); @@ -786,15 +760,11 @@ test('a release refuses a delete that never answers, within that delete’s own await runtime.leaseLifecycle.allocate?.(lease); const refused = refusalOf(runtime.leaseLifecycle.release?.(lease)); await vi.advanceTimersByTimeAsync(DELETE_ANSWER_WAIT_MS); - const error = await refused; // The lease is still held: Limrun never confirmed the instance was gone. - assert.ok( - error instanceof AppError && - error.details?.reason === 'limrun_instance_delete_unconfirmed' && - error.details.limrunInstanceId === 'android-instance-1' && - /delete/i.test(String(error.details.cause)), - `expected the release to refuse within its delete deadline, got ${String(error)}`, - ); + const refusedDelete = (await refused) as AppError; + assert.equal(refusedDelete.details?.reason, 'limrun_instance_delete_unconfirmed'); + assert.equal(refusedDelete.details.limrunInstanceId, 'android-instance-1'); + assert.match(String(refusedDelete.details.cause), /delete/i); // What that delete goes on to answer is still observed: its late confirmation ends the lease. reportDelete(); await vi.advanceTimersByTimeAsync(0); @@ -808,7 +778,6 @@ test('a release refuses a delete that never answers, within that delete’s own assert.equal(state.disconnect.mock.calls.length - disconnects, 1); }); -/** A create that answers after its disposal gave up still owes the end of whatever it names. */ test('a create that answers after its disposal gave up ends the instance it rents', async () => { const { creates, deletes, connects, disconnects } = limrunActivity(); let reportCreate!: () => void; @@ -824,10 +793,7 @@ test('a create that answers after its disposal gave up ends the instance it rent vi.useFakeTimers(); await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS); // The release refused rather than waiting longer, and the create has still not named anything. - assert.ok( - (await refusedRelease) instanceof AppError, - 'expected the release to refuse once its create budget expired', - ); + assert.ok((await refusedRelease) instanceof AppError, 'expected the expired release to refuse'); assert.deepEqual(deletedInstances(deletes), []); // The POST answers after that: the instance it names, and the session built on it, belong to the // disposal that is already over, so that disposal ends them. @@ -846,7 +812,6 @@ test('a create that answers after its disposal gave up ends the instance it rent assert.equal(state.connectCount - connects, 1); }); -/** A session built before a later failure is a resource its disposal has to release, not drop. */ test('a create that fails after its session was built disconnects that session', async () => { const { deletes, disconnects } = limrunActivity(); state.uploadedApps = [{ id: 'asset-1', name: 'com.example.app.apk', md5: 'md5', os: 'android' }]; @@ -854,15 +819,12 @@ test('a create that fails after its session was built disconnects that session', state.installedApps = [{ id: 'com.other.app' }, { id: 'com.other.app.fork' }]; const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); try { - const error = await refusalOf( + const refused = refusalOf( runtime.leaseLifecycle.allocate?.(androidLease('lease-app-unresolved'), { initialApp: 'com.example.app.apk', }), ); - assert.ok( - error instanceof AppError, - `expected the app resolution to fail, got ${String(error)}`, - ); + assert.ok((await refused) instanceof AppError, 'expected the app resolution to fail'); // The instance it rented is deleted, and so is the connection the session built beside it. assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); assert.equal(state.disconnect.mock.calls.length - disconnects, 1); @@ -873,22 +835,33 @@ test('a create that fails after its session was built disconnects that session', } }); -/** A joiner with valid flags is not refused for another caller’s refused flag. */ +/** The alias a lease was created with answers for the asset name its caller reuses. */ +test('a created app alias resolves the asset name back to the installed identifier', async () => { + state.uploadedApps = [{ id: 'asset-1', name: 'settings.apk', md5: 'md5', os: 'android' }]; + state.installedApps = [{ id: 'com.example.settings' }]; + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const resolve = (device: DeviceInfo, app: string) => runtime.resolveAppReference(device, app); + try { + const device = await allocateAndroidDevice(runtime, 'lease-alias', 'settings.apk'); + assert.equal(resolve(device, 'settings.apk'), 'com.example.settings'); + assert.equal(resolve(device, 'com.other.app'), 'com.other.app'); + assert.equal(resolve({ ...device, id: 'other:android:x' }, 'settings.apk'), 'settings.apk'); + } finally { + state.uploadedApps = []; + state.installedApps = []; + await runtime.shutdown(); + } +}); + test('a joiner whose own flags are valid allocates despite another caller’s refused flag', async () => { const { creates } = limrunActivity(); const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); const lease = androidLease('lease-mixed-flags'); try { // The first call carries a flag this provider reads no field for; the second carries none. - const refused = refusalOf( - runtime.leaseLifecycle.allocate?.(lease, { flags: { providerOsVersion: '15' } }), - ); + const refused = refusalOf(runtime.leaseLifecycle.allocate?.(lease, REFUSED_FLAG)); const joined = await runtime.leaseLifecycle.allocate?.(lease); - const error = await refused; - assert.ok( - error instanceof AppError && error.code === 'INVALID_ARGS', - `expected the refused caller to be refused for its own flag, got ${String(error)}`, - ); + assert.equal(((await refused) as AppError).code, 'INVALID_ARGS'); assert.equal(joined?.limrunInstanceId, 'android-instance-1'); } finally { await runtime.shutdown(); @@ -897,6 +870,31 @@ test('a joiner whose own flags are valid allocates despite another caller’s re assert.equal(state.androidCreate.mock.calls.length - creates, 1); }); +/** A joiner with unsupported flags is refused by a create that is still running, not handed its session. */ +test('a joiner whose own flags are refused is refused while another caller’s create runs', async () => { + const { creates } = limrunActivity(); + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-refused-join'); + try { + const running = runtime.leaseLifecycle.allocate?.(lease); + await waitForCreate(creates + 1); + const joining = refusalOf(runtime.leaseLifecycle.allocate?.(lease, REFUSED_FLAG)); + reportCreate(); + assert.equal(((await joining) as AppError).code, 'INVALID_ARGS'); + assert.equal((await running)?.limrunInstanceId, 'android-instance-1'); + } finally { + reportCreate(); + state.createGate = undefined; + await runtime.shutdown(); + } + // The refused joiner did not start a second create for a lease it was refused on. + assert.equal(state.androidCreate.mock.calls.length - creates, 1); +}); + function completedRequest(url: string, error?: string): unknown { return { type: 'complete', @@ -931,8 +929,12 @@ function requestScope(signal: AbortSignal = new AbortController().signal): Platf async function allocateAndroidDevice( runtime: ReturnType, leaseId = 'lease-appstate', + initialApp?: string, ): Promise { - const allocation = await runtime.leaseLifecycle.allocate?.(androidLease(leaseId)); + const allocation = await runtime.leaseLifecycle.allocate?.( + androidLease(leaseId), + initialApp === undefined ? undefined : { initialApp }, + ); const device = allocation?.device; if (!device || typeof device !== 'object') throw new Error('Expected allocated device'); return device as DeviceInfo; diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index 590f674412..0b713b8fa9 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -77,6 +77,8 @@ export type LimrunRuntimeOptions = { export type LimrunRuntime = ProviderDeviceRuntime & { recoverExpiredLease: ProviderExpiredLeaseRecovery; getDeviceSession(device: DeviceInfo): LimrunDeviceSession | undefined; + /** Maps an uploaded asset name this lease was created with back to its installed identifier. */ + resolveAppReference(device: DeviceInfo, app: string): string; }; export type LimrunRuntimeRegistration = Readonly<{ @@ -122,7 +124,9 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { private readonly leases = new Map(); /** * What each lease's network capture goes through, including the records a release settled. Absent until a - * lease asks for a capture, which is what keeps the capture's module out of a runtime that captures nothing. + * lease asks for a capture, which is what keeps the capture's module out of a runtime that captures nothing: + * the lifecycle is only ever reached through `await import('./capture-lifecycle.ts')` in `captures()`, and + * this declaration stays a type for exactly that reason. */ private capturesInstance?: LimrunCaptureLifecycle; /** Whether this runtime has shut down; it then holds nothing, so it must create nothing. */ @@ -310,6 +314,12 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { // never both pass it, and this runtime can be shut down only before or after this step. if (this.shutDown) throw allocationRefusedAfterShutdown(lease); owner = this.beginAllocation(lease, platform, context); + } else { + // A lease this runtime already holds, so this call answers for flags of its own, and settles them before + // it awaits anything else — including a create that is already running. Joining one is no way to be + // handed a session this caller was never entitled to; a new lease is validated as the first step of its + // own create instead, which is what keeps this publication free of any await. + await this.rejectRefusedProfileFields(context); } await this.joinCreation(owner, context); // A repeat allocation can ask for the capture, or retry one that could not start. A lease a teardown has @@ -341,12 +351,6 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { owner: LimrunLeaseOwner, context?: LeaseLifecycleContext, ): Promise { - if (owner.state.phase === 'live') { - // A lease this runtime allocated earlier: this call answers for flags of its own, and settles them - // before it reads any more state. - await this.rejectRefusedProfileFields(context); - return; - } if (owner.state.phase !== 'allocating') return; const create = await this.creationFor(owner, context); if (create === undefined) return; @@ -369,10 +373,9 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { /** * The create one caller is entitled to: the one its lease's owner stands on, unless that create refused the * caller that started it for a flag this provider reads no field for. That refusal billed nothing, so it is - * the other caller's answer and not this lease's: this caller answers for its own flags first — a refusal of - * its own is its own to throw — and a caller whose flags this provider does read asks for the create the - * refusal foreclosed and makes it the lease's creation, on the owner the lease already has. Any later joiner - * of the same refusal follows that create rather than renting a second instance. + * that caller's answer and not this lease's: a caller whose own flags this provider does read asks for the + * create the refusal foreclosed and makes it the lease's creation, on the owner the lease already has. Any + * later joiner of the same refusal follows that create rather than renting a second instance. */ private async creationFor( owner: LimrunLeaseOwner, @@ -382,8 +385,8 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { const started = owner.creation; const create = await started; if (create.status !== 'refused_caller') return create; - await this.rejectRefusedProfileFields(context); - // Another joiner of the same refusal replaced it while this caller was being answered: join that create. + // This caller's own flags were answered for before it got here, so the refusal below is another + // caller's. Another joiner of it may have replaced it while this caller was being answered: join that. if (owner.creation !== started) continue; // A teardown took the lease in the meantime, so what this lease has is that route's to end. if (owner.state.phase !== 'allocating') return undefined; @@ -429,6 +432,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { capture: undefined, appAlias: undefined, deletedInstanceId: undefined, + pendingDelete: undefined, }; this.leases.set(lease.leaseId, owner); // The create is started after the publication above, so its answer can never arrive without an owner From 4ae736896b3a27644551a8b22a2bda0502434d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 19:59:59 +0100 Subject: [PATCH 04/11] fix(limrun): answer one delete for one instance, with its reason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The instance delete joins the create under the same request contract: one attempt at the SDK, because a lost answer is ambiguous and this runtime retries deletes at its own budgets, not the SDK's. A 404 is Limrun answering that the instance is already gone — exactly what the delete was after — so it confirms the end instead of holding the lease for a retry that can only fail again. The recovery route that ends a lease from its labels keeps the same contract. Every refusal that says an instance is not confirmed deleted now carries the delete's own error as its cause, where a cause renders, beside the text in details; a create that both failed and could not be undone reports both reasons, not only the delete's. The disposal outcome records the delete's cause so each route can raise it without re-deriving anything. A replacement creation is observed the moment it is published, so the owner's promise that a creation cannot reach the process unobserved holds by construction on every path that installs one. Docs and shape: the teardown-scope marker and the reverse route keep their own comments, the capture lifecycle's paragraph sits on its class, shutdown's snapshot names its binding, and the close's report lists every lease it could not end with each of their reasons. --- packages/provider-limrun/src/android.ts | 21 ++- .../src/capture-lifecycle.test.ts | 4 + .../provider-limrun/src/capture-lifecycle.ts | 13 +- .../provider-limrun/src/client-options.ts | 35 ++++- .../provider-limrun/src/lease-ownership.ts | 16 +- .../provider-limrun/src/lease-release.test.ts | 86 +++++++++- packages/provider-limrun/src/lease-release.ts | 147 ++++++++++++------ packages/provider-limrun/src/runtime.ts | 39 ++--- .../src/session-allocation.test.ts | 47 +++++- src/__tests__/limrun-runtime.test.ts | 20 ++- src/provider-device-runtimes.test.ts | 2 +- 11 files changed, 333 insertions(+), 97 deletions(-) diff --git a/packages/provider-limrun/src/android.ts b/packages/provider-limrun/src/android.ts index 041f08ab6d..609593f8fc 100644 --- a/packages/provider-limrun/src/android.ts +++ b/packages/provider-limrun/src/android.ts @@ -242,22 +242,21 @@ async function runLimrunAndroidAdb( return await runLimrunAndroidAdbAtSerial(session, serial, args, options); } -/** - * The reverse manager's own route to the device. It is the session's route while the session is live, and - * cleanup's route of its own once the session is retired: clearing reverse mappings means reaching the device - * that is about to be disconnected, and that has to stay possible after every caller of this session was - * refused. Only the cleanup's own dispatches carry its teardown scope, so no caller command is ever - * dispatched at the serial the teardown is ending. - */ /** * Whether one dispatch belongs to a session's teardown: true only for the dispatches that teardown itself - * started. The reverse manager is given this predicate as its route, and a provider that hands a caller - * command to it — a reverse command queued inside an operation of its own — carries no marker, so the - * manager cannot have a caller's mapping bound at a serial that is on its way out. A marker set by one - * session never authorises another session's route. + * started. A provider that hands a caller command to the reverse route — a reverse command queued inside an + * operation of its own — carries no marker, so the manager cannot have a caller's mapping bound at a serial + * that is on its way out. A marker set by one session never authorises another session's route. */ const limrunAdbTeardownScope = new AsyncLocalStorage(); +/** + * The reverse manager's own route to the device. It is the session's route while the session is live, and + * cleanup's route of its own once the session is retired: clearing reverse mappings means reaching the device + * that is about to be disconnected, and that has to stay possible after every caller of this session was + * refused. Only a dispatch that carries its session's teardown scope may use the teardown's executor, so no + * caller command is ever dispatched at the serial the teardown is ending. + */ async function runLimrunAndroidReverseAdb( session: LimrunAndroidAdbSession, args: readonly string[], diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts index aaa26a892f..0acdb0941a 100644 --- a/packages/provider-limrun/src/capture-lifecycle.test.ts +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -21,6 +21,10 @@ test('a repeat allocation stops waiting on a destination tunnel that never answe joined = true; }); try { + // Below the budget the join is still waiting: only its expiry ends it, not an early return. + await vi.advanceTimersByTimeAsync(19_000); + await Promise.resolve(); + assert.ok(!joined, 'expected the join to keep waiting inside its budget'); await vi.advanceTimersByTimeAsync(CAPTURE_OPEN_WAIT_MS); await joining; } finally { diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts index f6531d4388..873ba45555 100644 --- a/packages/provider-limrun/src/capture-lifecycle.ts +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -7,13 +7,6 @@ import type { NetworkCapture, SettledNetworkCapture } from './network-har.ts'; import { awaitLimrunWithinBudget } from './lifecycle-budget.ts'; import { errorMessage } from '@agent-device/kernel/errors'; -/** - * What one lease's network capture goes through: starting it, ending it, and answering what `artifacts` can - * say about it. The owner is where a capture lives, including a start whose destination tunnel is still being - * opened, so the allocation that started one, a release that took the lease over, and an artifacts read all - * look at the same thing and none of them can end up owning a tunnel nobody can close. - */ - /** * How long a read or a teardown waits for a destination tunnel that is still being opened. The SDK gives this * handshake fifteen seconds of its own. @@ -32,6 +25,12 @@ export type LimrunCaptureHost = Readonly<{ heldOwners(): Iterable; }>; +/** + * What one lease's network capture goes through: starting it, ending it, and answering what `artifacts` can + * say about it. The owner is where a capture lives, including a start whose destination tunnel is still being + * opened, so the allocation that started one, a release that took the lease over, and an artifacts read all + * look at the same thing and none of them can end up owning a tunnel nobody can close. + */ export class LimrunCaptureLifecycle { /** The capture records this lifecycle settled, which outlive their lease so `artifacts` can still name them. */ private readonly settled = new Map(); diff --git a/packages/provider-limrun/src/client-options.ts b/packages/provider-limrun/src/client-options.ts index 6b8422f937..7f0997f335 100644 --- a/packages/provider-limrun/src/client-options.ts +++ b/packages/provider-limrun/src/client-options.ts @@ -1,5 +1,6 @@ import type Limrun from '@limrun/api'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import type { LimrunDeleteResult } from './lease-ownership.ts'; export const LIMRUN_CLIENT_ID = 'agent-device-cli'; @@ -19,6 +20,38 @@ export function limrunUserAgent(clientVersion: string): string { */ export const LIMRUN_SINGLE_ATTEMPT = Object.freeze({ maxRetries: 0 }); +/** + * Ends one instance, answering with no failure once Limrun has it gone and with a failure string and its error + * otherwise. Like the create, the delete is asked for in one attempt: a lost answer is ambiguous, and this + * runtime's retries happen at its own budgets, not the SDK's. A 404 is Limrun answering that the instance is + * already gone, which is exactly what the delete was asked to achieve, so it confirms rather than refuses. + */ +export async function deleteLimrunInstance( + limrun: Limrun, + platform: 'ios' | 'android', + instanceId: string, +): Promise { + try { + if (platform === 'ios') { + await limrun.iosInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); + } else { + await limrun.androidInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); + } + return { failure: undefined, cause: undefined }; + } catch (error) { + // A 404 names the outcome this request was after, however it came about. + if (readHttpStatus(error) === 404) return { failure: undefined, cause: undefined }; + return { failure: errorMessage(error), cause: error }; + } +} + +/** The HTTP status of an SDK failure, for a route that can answer for one particular refusal. */ +function readHttpStatus(error: unknown): number | undefined { + if (!error || typeof error !== 'object') return undefined; + const status = (error as { status?: unknown }).status; + return typeof status === 'number' ? status : undefined; +} + /** The organization client, or a typed refusal for an operation that needs the API key. */ export function requireLimrunOrgClient( limrun: Limrun | undefined, diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts index 68f3aa88fc..f18d7eb374 100644 --- a/packages/provider-limrun/src/lease-ownership.ts +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -63,7 +63,7 @@ export type LimrunLeaseOwner = { * a delete whose answer this disposal stopped waiting for is still the delete that instance is getting. */ pendingDelete: - | Readonly<{ instanceId: string; requested: Promise }> + | Readonly<{ instanceId: string; requested: Promise }> | undefined; }; @@ -99,10 +99,24 @@ export type LimrunDisposalOutcome = Readonly<{ instanceId: string | undefined; /** Why a disposal could not be confirmed; absent once the instance is gone. */ failure: string | undefined; + /** + * The refusal behind that failure, when one exists: the delete's own error. Reported as an error's cause, + * where it renders, rather than only as a string in details. + */ + cause: unknown; /** The released capture's record, so a refusal can still carry the HAR that was written. */ capture: SettledNetworkCapture | undefined; }>; +/** + * What one instance delete came to: whether Limrun has the instance gone, and what refused it otherwise. + */ +export type LimrunDeleteResult = Readonly<{ + failure: string | undefined; + /** The refusal behind that failure, for whoever has to report it. */ + cause: unknown; +}>; + /** * What one create amounted to: the session it made, why it failed, or that the caller who started it was * refused before anything was billed. `refused_caller` is that caller's own answer and not the lease's: an diff --git a/packages/provider-limrun/src/lease-release.test.ts b/packages/provider-limrun/src/lease-release.test.ts index a64d78e7e5..aabbb8a4f4 100644 --- a/packages/provider-limrun/src/lease-release.test.ts +++ b/packages/provider-limrun/src/lease-release.test.ts @@ -1,7 +1,17 @@ import { test, vi, expect } from 'vitest'; import type { DeviceLease } from '@agent-device/contracts/device'; -import { disposeLimrunLeaseOwner, type LimrunDisposalHost } from './lease-release.ts'; -import type { LimrunCreateOutcome, LimrunLeaseOwner } from './lease-ownership.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { + cleanupLeftInstancesRunning, + createLeftInstanceRunning, + disposeLimrunLeaseOwner, + type LimrunDisposalHost, +} from './lease-release.ts'; +import type { + LimrunCreateOutcome, + LimrunDisposalOutcome, + LimrunLeaseOwner, +} from './lease-ownership.ts'; /* * Each wait is longer than the budgets it steps past: a disposal's create wait, and everything a disposal @@ -22,7 +32,7 @@ test('a create that answers late joins the delete its disposal already started', await new Promise((resolve) => { reportDelete = resolve; }); - return undefined; + return { failure: undefined, cause: undefined }; }); const host: LimrunDisposalHost = { deleteInstance: deleteHost, @@ -60,6 +70,76 @@ test('a create that answers late joins the delete its disposal already started', expect(owner.deletedInstanceId).toBe('android-instance-1'); }); +/** + * A delete Limrun refused keeps its own error on the disposal outcome, and the refusals every route raises + * carry it as their cause — where a cause renders — beside the text that says the delete was unconfirmed. + */ +test("a refused delete reports its own error as the refusal's cause", async () => { + await import('./lifecycle-budget.ts'); + const refusal = new Error('Limrun refused to delete the instance'); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ + failure: 'Limrun refused to delete the instance', + cause: refusal, + }), + endCapture: async () => undefined, + }; + const owner = ownerWithRentedInstance( + Promise.resolve({ status: 'failed', error: new Error('boot failed') }), + ); + const outcome = await disposeLimrunLeaseOwner(host, owner); + expect(outcome.cleanup).toBe('uncertain'); + expect(outcome.cause).toBe(refusal); + + // The allocation's refusal names both what it could not settle and why its create failed at all. + const lease = (owner as unknown as { lease: DeviceLease }).lease; + const thrown = captureThrow(() => + createLeftInstanceRunning(lease, outcome, new Error('boot failed')), + ); + expect(thrown.details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect(String(thrown.details?.cause)).toContain('boot failed'); + expect(thrown.cause).toBe(refusal); +}); + +/** + * A daemon close that could not end two leases names both of them and why each is still held, not only the + * first — every one of them may need to be named to Limrun by hand. + */ +test('a close that could not end two leases reports both reasons', async () => { + const refusal = new Error('Limrun refused to delete the instance'); + const outcomes: LimrunDisposalOutcome[] = [ + { + cleanup: 'uncertain', + instanceId: 'android-instance-1', + failure: 'delete one refused', + cause: refusal, + capture: undefined, + }, + { + cleanup: 'uncertain', + instanceId: undefined, + failure: undefined, + cause: undefined, + capture: undefined, + }, + ]; + const thrown = captureThrow(() => cleanupLeftInstancesRunning(outcomes)); + expect(thrown.details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect(thrown.details?.limrunInstanceIds).toEqual(['android-instance-1']); + expect(String(thrown.details?.cause)).toContain('delete one refused'); + expect(String(thrown.details?.cause)).toContain('an unconfirmed delete'); + expect(thrown.cause).toBe(refusal); +}); + +function captureThrow(raise: () => never): AppError { + try { + raise(); + } catch (error) { + return error as AppError; + } + throw new Error('expected the refusal to throw'); +} + /** An owner whose create has rented an instance but has not answered, taken by a disposal. */ function ownerWithRentedInstance(creation: Promise): LimrunLeaseOwner { const lease = { diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index b554024210..c168d6961c 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -1,9 +1,11 @@ import type Limrun from '@limrun/api'; import type { DeviceLease } from '@agent-device/contracts/device'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, errorMessage } from '@agent-device/kernel/errors'; import { cleanupLimrunAndroidAdbTunnel } from './android.ts'; +import { LIMRUN_SINGLE_ATTEMPT } from './client-options.ts'; import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend, type LimrunPlatform } from './device.ts'; import type { + LimrunDeleteResult, LimrunDisposalOutcome, LimrunLeaseOwner, LimrunRuntimeSession, @@ -50,8 +52,8 @@ const CAPTURE_RECORD_BUDGET_MS = 15_000; * reaching back into the routes that allocate and read one, and without deciding when a lease is free again. */ export type LimrunDisposalHost = Readonly<{ - /** One instance delete. `undefined` answers that Limrun confirmed the instance is gone. */ - deleteInstance(platform: LimrunPlatform, instanceId: string): Promise; + /** One instance delete: whether Limrun has the instance gone, and what refused it otherwise. */ + deleteInstance(platform: LimrunPlatform, instanceId: string): Promise; /** Ends one lease's capture and answers with its record, for a runtime that has captured something. */ endCapture(owner: LimrunLeaseOwner): Promise; }>; @@ -97,6 +99,7 @@ export async function disposeLimrunLeaseOwner( joined.ok ? undefined : workOutstandingFailure('Capture record', CAPTURE_RECORD_BUDGET_MS), ), ), + cause: instance.cause, capture: joined.ok ? joined.value : undefined, }; } @@ -130,6 +133,7 @@ async function disposeUnansweredCreate( cleanup: 'uncertain', instanceId: undefined, failure: outstanding, + cause: undefined, capture: await capture, }; } @@ -143,6 +147,7 @@ async function disposeUnansweredCreate( cleanup: instance.cleanup, instanceId, failure: joinOutcomeFailures(instance.failure, stuck ?? outstanding), + cause: instance.cause, capture: await capture, }; } @@ -175,6 +180,7 @@ async function disposeUnrented( cleanup: 'none', instanceId: undefined, failure: undefined, + cause: undefined, capture: await awaitEndedCapture(host, owner), }; } @@ -192,6 +198,13 @@ async function awaitEndedCapture( return ended.ok ? ended.value : undefined; } +/** What ending one instance came to: whether Limrun confirmed it gone, and what it said otherwise. */ +type LimrunInstanceEnd = Readonly<{ + cleanup: LimrunDisposalOutcome['cleanup']; + failure: string | undefined; + cause: unknown; +}>; + /** * Ends one owner's instance within a deadline of its own. An attached instance is the caller's own and is never * deleted here; a delete Limrun already confirmed answers for itself; and a delete that never answers keeps the @@ -202,11 +215,12 @@ async function endInstance( owner: LimrunLeaseOwner, session: LimrunRuntimeSession | undefined, instanceId: string, -): Promise> { +): Promise { const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); - if (owner.deletedInstanceId === instanceId) return { cleanup: 'confirmed', failure: undefined }; + if (owner.deletedInstanceId === instanceId) + return { cleanup: 'confirmed', failure: undefined, cause: undefined }; if ((session?.ownership ?? 'created') === 'attached') { - return { cleanup: 'attached', failure: undefined }; + return { cleanup: 'attached', failure: undefined, cause: undefined }; } const requested = owner.pendingDelete?.instanceId === instanceId @@ -226,8 +240,8 @@ async function endInstance( } const deleted = await awaitLimrunWithinBudget(requested, DELETE_ANSWER_BUDGET_MS); if (!deleted.ok) { - void requested.then((failure) => { - if (failure === undefined) confirmLateDelete(owner, instanceId); + void requested.then((answer) => { + if (answer.failure === undefined) confirmLateDelete(owner, instanceId); }); return { cleanup: 'uncertain', @@ -235,12 +249,14 @@ async function endInstance( `Delete of instance '${instanceId}'`, DELETE_ANSWER_BUDGET_MS, ), + cause: undefined, }; } - if (deleted.value === undefined) owner.deletedInstanceId = instanceId; + if (deleted.value.failure === undefined) owner.deletedInstanceId = instanceId; return { - cleanup: deleted.value === undefined ? 'confirmed' : 'uncertain', - failure: deleted.value, + cleanup: deleted.value.failure === undefined ? 'confirmed' : 'uncertain', + failure: deleted.value.failure, + cause: deleted.value.cause, }; } @@ -362,10 +378,12 @@ async function releaseLabeledInstances( : await limrun.androidInstances.list({ labelSelector }); const instanceIds = instances.getPaginatedItems().map((instance) => instance.metadata.id); for (const instanceId of instanceIds) { + // One attempt each, the same contract every other instance delete keeps: this route's own retries are + // the daemon's expiry sweeps, not the SDK's. if (platform === 'ios') { - await limrun.iosInstances.delete(instanceId); + await limrun.iosInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); } else { - await limrun.androidInstances.delete(instanceId); + await limrun.androidInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); } } if (instanceIds.length === 0) return undefined; @@ -379,19 +397,14 @@ async function releaseLabeledInstances( */ function releaseLeftInstanceRunning(leaseId: string, outcome: LimrunDisposalOutcome): never { const { instanceId } = outcome; - throw new AppError( - 'COMMAND_FAILED', + return throwUnconfirmedDelete( `Limrun instance for lease '${leaseId}' was not confirmed deleted`, - { - reason: 'limrun_instance_delete_unconfirmed', - retriable: true, - ...(instanceId === undefined ? {} : { limrunInstanceId: instanceId }), - ...(outcome.failure === undefined ? {} : { cause: outcome.failure }), - hint: - instanceId === undefined - ? `Release lease '${leaseId}' again once its instance create names the instance it rented.` - : `Release lease '${leaseId}' again, or ask Limrun to delete instance '${instanceId}'.`, - }, + instanceId === undefined ? {} : { limrunInstanceId: instanceId }, + outcome.failure, + instanceId === undefined + ? `Release lease '${leaseId}' again once its instance create names the instance it rented.` + : `Release lease '${leaseId}' again, or ask Limrun to delete instance '${instanceId}'.`, + outcome.cause, ); } @@ -421,6 +434,7 @@ export function allocationRefusedForTake( ? { retriable: true, ...(outcome.failure === undefined ? {} : { cause: outcome.failure }) } : {}), }, + outcome.cause, ); } @@ -431,21 +445,58 @@ export function allocationRefusedForTake( export function createLeftInstanceRunning( lease: DeviceLease, outcome: LimrunDisposalOutcome, + createError: unknown, ): never { const { instanceId } = outcome; + return throwUnconfirmedDelete( + `Limrun lease '${lease.leaseId}' could not be allocated, and its instance could not be deleted`, + instanceId === undefined ? {} : { limrunInstanceId: instanceId }, + // Both refusals belong to this one answer: why the allocation failed, and why its instance is still + // billed. The delete's text leads the report because it is what the operator still has to settle. + joinFailureText( + outcome.failure, + createError === undefined ? undefined : `create failed: ${errorMessage(createError)}`, + ), + instanceId === undefined + ? `Ask Limrun what lease '${lease.leaseId}' rented — this runtime could not name the instance.` + : `Ask Limrun to delete instance '${instanceId}' — this runtime could not confirm the delete.`, + outcome.cause ?? createError, + ); +} + +/** Two reasons one report carries, delete first, when both exist. */ +function joinFailureText( + first: string | undefined, + second: string | undefined, +): string | undefined { + if (first === undefined) return second; + return second === undefined ? first : `${first}; ${second}`; +} + +/** + * The refusal every route that could not confirm an instance delete raises. The instance ids go into details, + * why the delete failed goes into `details.cause` as text, and the delete's own error goes where a cause + * renders — the error's own cause. Each caller names its instances the way its own report reads: a single + * lease its one id, a close the list of what it left. + */ +function throwUnconfirmedDelete( + message: string, + instanceDetails: Readonly>, + failure: string | undefined, + hint: string, + cause: unknown, +): never { throw new AppError( 'COMMAND_FAILED', - `Limrun lease '${lease.leaseId}' could not be allocated, and its instance could not be deleted`, + message, { reason: 'limrun_instance_delete_unconfirmed', retriable: true, - ...(instanceId === undefined ? {} : { limrunInstanceId: instanceId }), - ...(outcome.failure === undefined ? {} : { cause: outcome.failure }), - hint: - instanceId === undefined - ? `Ask Limrun what lease '${lease.leaseId}' rented — this runtime could not name the instance.` - : `Ask Limrun to delete instance '${instanceId}' — this runtime could not confirm the delete.`, + ...instanceDetails, + ...(failure === undefined ? {} : { cause: failure }), + hint, }, + cause, ); } @@ -455,29 +506,25 @@ export function createLeftInstanceRunning( * the operator needs both the count and the reason. */ export function cleanupLeftInstancesRunning(outcomes: readonly LimrunDisposalOutcome[]): never { - const leftRunning = outcomes.map((outcome) => outcome.instanceId); - const known = leftRunning.filter((id) => id !== undefined); + const known = outcomes.map((outcome) => outcome.instanceId).filter((id) => id !== undefined); const plural = outcomes.length === 1 ? '' : 's'; - throw new AppError( - 'COMMAND_FAILED', + // Why every lease this close could not end, not only the first: each failure is a separate thing the + // operator may have to name to Limrun. + return throwUnconfirmedDelete( `Limrun cleanup left ${String(outcomes.length)} lease${plural} unended, ${String(known.length)} with a known instance`, - { - reason: 'limrun_instance_delete_unconfirmed', - ...(known.length === 0 ? {} : { limrunInstanceIds: known }), - // Why every lease this close could not end, not only the first: each failure is a separate thing the - // operator may have to name to Limrun. - ...(outcomes.every((outcome) => outcome.failure === undefined) - ? {} - : { - cause: outcomes.map((outcome) => outcome.failure ?? 'an unconfirmed delete').join('; '), - }), - ...(known.length === 0 - ? { hint: unendedLeaseHint(outcomes) } - : { hint: `Ask Limrun to delete: ${known.join(', ')}.` }), - }, + { limrunInstanceIds: known }, + joinFailureReasons(outcomes), + known.length === 0 ? unendedLeaseHint(outcomes) : `Ask Limrun to delete: ${known.join(', ')}.`, + outcomes.find((outcome) => outcome.cause !== undefined)?.cause, ); } +/** Every lease this close could not end, with why, joined for one report. */ +function joinFailureReasons(outcomes: readonly LimrunDisposalOutcome[]): string | undefined { + if (outcomes.every((outcome) => outcome.failure === undefined)) return undefined; + return outcomes.map((outcome) => outcome.failure ?? 'an unconfirmed delete').join('; '); +} + /** What a close says when the work it could not finish has no instance to name yet: the lease and the work. */ function unendedLeaseHint(outcomes: readonly LimrunDisposalOutcome[]): string { return `Limrun may still bill work this runtime could not name: ${outcomes diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index 0b713b8fa9..a7dca03165 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -40,7 +40,12 @@ import type { import { providerRuntimeOwner } from '@agent-device/contracts/platform-runtime'; import type { LimrunAppLogDescriptor } from './app-log-descriptor.ts'; import type { LimrunAppLogReader } from './app-log-runtime.ts'; -import { LIMRUN_CLIENT_ID, limrunUserAgent, requireLimrunOrgClient } from './client-options.ts'; +import { + deleteLimrunInstance, + LIMRUN_CLIENT_ID, + limrunUserAgent, + requireLimrunOrgClient, +} from './client-options.ts'; import { resolveLimrunRuntimeInstance } from './runtime-instance.ts'; import type { LimrunRequestOperationDrain } from './request-cancellation.ts'; import type { LimrunAppAsset } from './app-catalog.ts'; @@ -52,6 +57,7 @@ import type { import type { LimrunAppAlias, LimrunCreateOutcome, + LimrunDeleteResult, LimrunDisposalOutcome, LimrunLeaseOwner, LimrunRuntimeSession, @@ -282,11 +288,12 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { */ async shutdown(): Promise { this.shutDown = true; - // Snapshot before anything is awaited: `allocate` refuses once this flag is set and publishes an - // owner only where none exists, and nothing between the flag and this loop awaits. So no create can - // start here, and every create already running has an owner for this loop to take. + // Snapshot into a binding before anything is awaited: `allocate` refuses once this flag is set and + // publishes an owner only where none exists, and nothing between the flag and this snapshot awaits. So + // no create can start here, and every create already running has an owner in this snapshot to take. + const held = [...this.leases.values()]; const outcomes = await Promise.all( - [...this.leases.values()].map(async (owner) => await this.take(owner, 'shutdown')), + held.map(async (owner) => await this.take(owner, 'shutdown')), ); // An instance Limrun would not give back is still billed, and this process will not stop it. The // plugin's close is where an operator learns what a daemon close left running, so the instance and @@ -365,7 +372,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { const outcome = await this.settleFailedCreate(owner); if (outcome?.cleanup === 'uncertain') { const { createLeftInstanceRunning } = await import('./lease-release.ts'); - throw createLeftInstanceRunning(owner.lease, outcome); + throw createLeftInstanceRunning(owner.lease, outcome, create.error); } throw create.error; } @@ -392,6 +399,10 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (owner.state.phase !== 'allocating') return undefined; const retried = this.createForLease(owner, context); owner.creation = retried; + // Observed from the moment it is published, like the original creation: the owner's type promises a + // creation that cannot reach the process unobserved, and this keeps that promise by construction, not + // only by the fact that `createForLease` reports its failures as outcomes. + void retried.catch(() => undefined); return await retried; } } @@ -574,6 +585,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { cleanup: 'uncertain', instanceId: owner.rentedInstanceId, failure: errorMessage(error), + cause: error, capture: undefined, }); }); @@ -600,18 +612,9 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { private async deleteCreatedInstance( platform: LimrunPlatform, instanceId: string, - ): Promise { - try { - const limrun = this.orgClient('Deleting an instance'); - if (platform === 'ios') { - await limrun.iosInstances.delete(instanceId); - } else { - await limrun.androidInstances.delete(instanceId); - } - return undefined; - } catch (error) { - return errorMessage(error); - } + ): Promise { + const limrun = this.orgClient('Deleting an instance'); + return await deleteLimrunInstance(limrun, platform, instanceId); } /** Removes an owner once it holds nothing, and only while it is still the map's owner for that lease. */ diff --git a/packages/provider-limrun/src/session-allocation.test.ts b/packages/provider-limrun/src/session-allocation.test.ts index 94f135e521..60096de044 100644 --- a/packages/provider-limrun/src/session-allocation.test.ts +++ b/packages/provider-limrun/src/session-allocation.test.ts @@ -5,6 +5,7 @@ import { test } from 'vitest'; import Limrun from '@limrun/api'; import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; +import { deleteLimrunInstance } from './client-options.ts'; import { allocateLimrunAndroidSession, allocateLimrunIosSession } from './session-allocation.ts'; import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; @@ -93,9 +94,53 @@ test('an Android instance that answers without its endpoints is reported as rent assert.deepEqual(routes, ['POST /v1/android_instances']); }); +/** + * The instance delete carries the same single-attempt contract as the create, and answers for the one refusal + * that names the outcome it was after: an instance Limrun already removed is gone, which is what the delete + * was asked to achieve. + */ + +test('the instance delete asks the SDK for one attempt, so a lost answer is not repeated behind this runtime’s back', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${String(request.path)}`); + // The instance may well be gone; this answer never arrives, and only the daemon's budgets retry it. + request.drop(); + }); + try { + const deleted = await deleteLimrunInstance(clientFor(server), 'android', 'android-instance-1'); + assert.deepEqual(routes, ['DELETE /v1/android_instances/android-instance-1']); + assert.match(String(deleted.failure), /connection|socket|abort|fetch|closed/i); + assert.ok(deleted.cause !== undefined, 'the refusal is reported, not swallowed'); + } finally { + await server.close(); + } +}); + +test('an instance delete that finds the instance already gone confirms it deleted', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${String(request.path)}`); + await request.respond(404, JSON.stringify({ error: { message: 'instance not found' } })); + }); + try { + const deleted = await deleteLimrunInstance(clientFor(server), 'ios', 'ios-instance-1'); + // Gone is gone: the delete was not refused, so no later route re-issues it and no lease is held for it. + assert.equal(deleted.failure, undefined); + assert.equal(deleted.cause, undefined); + assert.deepEqual(routes, ['DELETE /v1/ios_instances/ios-instance-1']); + } finally { + await server.close(); + } +}); + +function clientFor(server: InstanceServer): Limrun { + return new Limrun({ apiKey: server.key, baseURL: server.baseUrl }); +} + function allocationParams(server: InstanceServer) { return { - limrun: new Limrun({ apiKey: server.key, baseURL: server.baseUrl }), + limrun: clientFor(server), lease: { leaseId: 'lease-single-attempt', tenantId: 'team-a', diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index 9bd6304c7a..7a143ca615 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -254,7 +254,10 @@ test('Limrun reclaims a labeled iOS instance without an in-memory session', asyn assert.deepEqual(limrunMockState.iosList.mock.calls, [ [{ labelSelector: 'provider=limrun,leaseId=lease-recovered-ios' }], ]); - assert.deepEqual(limrunMockState.iosDelete.mock.calls, [['ios-instance-recovered']]); + assert.deepEqual( + limrunMockState.iosDelete.mock.calls.map((call) => (call as unknown[])[0]), + ['ios-instance-recovered'], + ); } finally { await runtime.shutdown(); } @@ -692,7 +695,10 @@ test('Limrun deletes iOS instance when post-create validation fails', async () = const allocateLease = runtime.leaseLifecycle.allocate; if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); await assert.rejects(() => allocateLease(lease), /did not expose apiUrl/); - assert.deepEqual(limrunMockState.iosDelete.mock.calls[0], ['ios-instance-missing-api']); + assert.equal( + (limrunMockState.iosDelete.mock.calls[0] as unknown[] | undefined)?.[0], + 'ios-instance-missing-api', + ); } finally { await runtime.shutdown(); } @@ -724,7 +730,10 @@ test('Limrun deletes Android instance when post-create validation fails', async const allocateLease = runtime.leaseLifecycle.allocate; if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); await assert.rejects(() => allocateLease(lease), /did not expose API and ADB/); - assert.deepEqual(limrunMockState.androidDelete.mock.calls[0], ['android-instance-missing-adb']); + assert.equal( + (limrunMockState.androidDelete.mock.calls[0] as unknown[] | undefined)?.[0], + 'android-instance-missing-adb', + ); } finally { await runtime.shutdown(); } @@ -909,7 +918,10 @@ test('Limrun instance access wins over the API key for its platform only', async assert.equal(limrunMockState.iosCreate.mock.calls.length, 0); assert.equal(limrunMockState.iosDelete.mock.calls.length, 0); assert.equal(limrunMockState.androidCreate.mock.calls.length, 1); - assert.deepEqual(limrunMockState.androidDelete.mock.calls, [['android-instance-1']]); + assert.deepEqual( + limrunMockState.androidDelete.mock.calls.map((call) => (call as unknown[])[0]), + ['android-instance-1'], + ); }); test('Limrun without an API key refuses operations that need one', async () => { diff --git a/src/provider-device-runtimes.test.ts b/src/provider-device-runtimes.test.ts index 887ec19236..da13e53ab8 100644 --- a/src/provider-device-runtimes.test.ts +++ b/src/provider-device-runtimes.test.ts @@ -93,7 +93,7 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest assert.deepEqual(limrun.iosList.mock.calls.at(-1), [ { labelSelector: `provider=limrun,leaseId=${lease.leaseId}` }, ]); - assert.deepEqual(limrun.iosDelete.mock.calls.at(-1), ['ios-instance-recovered']); + assert.equal(limrun.iosDelete.mock.calls.at(-1)?.[0], 'ios-instance-recovered'); assert.equal(fs.existsSync(pendingPath), false); } finally { dateNow.mockRestore(); From 385fd27dab2def6acc793cd500372920e4ac7a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 20:46:06 +0100 Subject: [PATCH 05/11] fix(limrun): observe every promise a disposal starts, in the step that starts it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bounded wait now answers for both of a work item's outcomes: a capture record or a delete that refuses is an outcome in its own right, not a failure for this route to re-raise, and it never reaches the process as an unhandled rejection — inside the budget or after it. The two promises a disposal starts beside the instance delete are observed the moment they are started, so a session disconnect or a tunnel close that throws during a gated delete has its observer already beside it. The daemon treats one loose rejection as fatal, and a disposal has other people's sessions. A release whose delete Limrun confirmed still answers for the instance it ended: a refused capture record reports as a teardown warning naming the record, not as the release's failure. The disposal's outcome keeps its shape from the delete alone — no capture or local-teardown result can change what was confirmed gone. Tests: two `createLimrunRuntime` cases with a gated delete and a throwing disconnect, and with a throwing capture close, each with a process-level unhandledRejection spy. The alias case now drives the platform module's binding, through `resolveOpenTarget`, and asserts the name stops answering once the lease is released, so the exported runtime type keeps the surface it had before this PR. --- .../src/dispose-observation.test.ts | 202 ++++++++++++++++++ packages/provider-limrun/src/lease-release.ts | 84 +++++--- .../provider-limrun/src/lifecycle-budget.ts | 32 ++- .../src/runtime-dependencies.fixtures.ts | 61 ++++++ packages/provider-limrun/src/runtime.test.ts | 77 +++---- packages/provider-limrun/src/runtime.ts | 2 - 6 files changed, 373 insertions(+), 85 deletions(-) create mode 100644 packages/provider-limrun/src/dispose-observation.test.ts create mode 100644 packages/provider-limrun/src/runtime-dependencies.fixtures.ts diff --git a/packages/provider-limrun/src/dispose-observation.test.ts b/packages/provider-limrun/src/dispose-observation.test.ts new file mode 100644 index 0000000000..b0920f7706 --- /dev/null +++ b/packages/provider-limrun/src/dispose-observation.test.ts @@ -0,0 +1,202 @@ +import assert from 'node:assert/strict'; +import { test, vi, expect } from 'vitest'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { createLimrunRuntime } from './runtime.ts'; +import { stubLimrunRuntimeDependencies } from './runtime-dependencies.fixtures.ts'; +import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; +import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; + +/* + * A disposal starts its capture end and its local teardown beside the instance delete and joins them only + * after it. A step of either one that refuses inside that window has this disposal as its observer from the + * moment it started: nothing a disposal begins may reach the process as an unhandled rejection, and a + * refused capture or teardown never changes what the delete answered for. + */ + +const state = vi.hoisted(() => ({ + /** Instance deletes this runtime asked Limrun for; each waits for its case to answer it. */ + deletes: [] as Array<{ instanceId: string; report: () => void }>, + disconnects: 0, + /** Made to throw by the case whose session disconnect refuses while its delete is still owed. */ + disconnectThrows: false, + captureCloses: 0, + /** Made to throw by the case whose capture tunnel close refuses as the release writes its record. */ + captureCloseThrows: false, +})); + +vi.mock('@limrun/api', () => ({ + default: class MockLimrun { + readonly iosInstances = { create: vi.fn(), list: vi.fn(), delete: vi.fn() }; + readonly androidInstances = { + create: async () => ({ + metadata: { id: 'android-instance-1' }, + status: { + token: 'instance-token', + apiUrl: 'https://android.example', + adbWebSocketUrl: 'wss://adb.example', + }, + }), + list: vi.fn(async () => ({ getPaginatedItems: () => [] })), + // Every delete waits for its case to answer it, so a release can be held at its delete while the + // local teardown it started beside has long since refused. + delete: vi.fn(async (instanceId: string) => { + let report!: () => void; + const answered = new Promise((answeredNow) => { + report = answeredNow; + }); + state.deletes.push({ instanceId, report }); + await answered; + }), + }; + readonly assets = { getOrUpload: vi.fn(), list: vi.fn(async () => []) }; + }, +})); + +vi.mock('@limrun/api/instance-client', () => ({ + createInstanceClient: vi.fn(async () => ({ + disconnect: () => { + state.disconnects += 1; + if (state.disconnectThrows) throw new Error('instance client refused to disconnect'); + }, + keepAlive: () => {}, + openUrl: vi.fn(), + sendAsset: vi.fn(), + pressKey: vi.fn(), + setText: vi.fn(), + tap: vi.fn(), + startAdbTunnel: vi.fn(async () => { + throw new Error('no ADB tunnel is raised by these cases'); + }), + startTunnel: vi.fn(async () => ({ + tunnelId: 't1', + selectors: [], + close: () => { + state.captureCloses += 1; + if (state.captureCloseThrows) throw new Error('capture tunnel refused to close'); + }, + })), + getTunnelStatus: vi.fn(async () => ({})), + })), +})); + +test('a local teardown that refuses beside a pending delete is reported, not loosed on the process', async () => { + const rejections = watchUnhandledRejections(); + state.deletes.length = 0; + state.disconnectThrows = true; + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-throwing-disconnect'); + let reportDelete: (() => void) | undefined; + try { + await runtime.leaseLifecycle.allocate?.(lease); + // The release holds at its delete while the session disconnect it started beside refuses. + const released = runtime.leaseLifecycle.release?.(lease); + await vi.waitFor(() => expect(state.deletes).toHaveLength(1)); + expect(state.disconnects).toBe(1); + reportDelete = state.deletes[0]?.report; + reportDelete?.(); + + // The delete's confirmation is the release's answer; what the local teardown came to rides beside it + // as the warning it is: a host resource this process could not put down. + const record = (await released) as Record; + expect(record.limrunInstanceId).toBe('android-instance-1'); + const warnings = warningsOf(record); + expect(warnings.map((warning) => warning.code)).toEqual(['LIMRUN_TEARDOWN_INCOMPLETE']); + expect(String(warnings[0]?.message)).toContain('Local teardown refused to finish'); + rejections.assertIsSilent(); + } finally { + rejections.stop(); + reportDelete?.(); + state.deletes.forEach(({ report }) => report()); + state.deletes.length = 0; + state.disconnectThrows = false; + await runtime.shutdown(); + } +}); + +test('a capture close that refuses still leaves the delete it ran beside answered for', async () => { + const rejections = watchUnhandledRejections(); + const harDir = mkdtempForTestSync('limrun-har-observe-'); + state.deletes.length = 0; + state.captureCloseThrows = true; + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', networkHarDir: harDir }, + testDependencies(), + ); + const lease = androidLease('lease-throwing-capture-close'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + await runtime.leaseLifecycle.allocate?.(lease, { flags: { providerNetworkLogs: true } }); + let reportDelete: (() => void) | undefined; + try { + const released = runtime.leaseLifecycle.release?.(lease); + await vi.waitFor(() => expect(state.deletes).toHaveLength(1)); + reportDelete = state.deletes[0]?.report; + reportDelete?.(); + + // The record's write ends with the tunnel's close, and that close refuses inside the disposal's + // join window: the instance the delete confirmed is gone, and the release answers for it. + const record = (await released) as Record; + expect(record.limrunInstanceId).toBe('android-instance-1'); + expect(state.captureCloses).toBe(1); + expect(warningsOf(record).map((warning) => warning.message)).toEqual([ + 'Capture record refused to be written', + ]); + rejections.assertIsSilent(); + } finally { + reportDelete?.(); + } + } finally { + rejections.stop(); + state.deletes.forEach(({ report }) => report()); + state.deletes.length = 0; + state.captureCloseThrows = false; + await runtime.shutdown(); + } +}); + +function warningsOf(record: Record): Array> { + return Array.isArray(record.warnings) ? (record.warnings as Record[]) : []; +} + +/** + * Collects the rejections the process would die on: the daemon's failure mode for one is to take every + * other session down with it, so a case whose disposal looses one fails here rather than in production. + */ +function watchUnhandledRejections(): { + assertIsSilent: () => void; + stop: () => void; +} { + const seen: unknown[] = []; + const watch = (reason: unknown): void => { + seen.push(reason); + }; + process.on('unhandledRejection', watch); + return { + assertIsSilent: () => { + if (seen.length === 0) return; + assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`); + }, + stop: () => process.off('unhandledRejection', watch), + }; +} + +function androidLease(leaseId: string): DeviceLease { + return { + leaseId, + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + }; +} + +/** + * The host and platform adapters these cases run on: every one of them answers a fixed value. The client + * connection and the instance delete are driven from the mock `state` above. + */ +function testDependencies(): LimrunRuntimeDependencies { + return stubLimrunRuntimeDependencies(); +} diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index c168d6961c..f683cf51c1 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -3,6 +3,7 @@ import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; import { cleanupLimrunAndroidAdbTunnel } from './android.ts'; import { LIMRUN_SINGLE_ATTEMPT } from './client-options.ts'; +import type { LimrunBudgetOutcome } from './lifecycle-budget.ts'; import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend, type LimrunPlatform } from './device.ts'; import type { LimrunDeleteResult, @@ -67,8 +68,12 @@ export async function disposeLimrunLeaseOwner( host: LimrunDisposalHost, owner: LimrunLeaseOwner, ): Promise { - const { awaitLimrunWithinBudget, joinOutcomeFailures, workOutstandingFailure } = - await import('./lifecycle-budget.ts'); + const { + awaitLimrunWithinBudget, + joinOutcomeFailures, + keepLimrunObserved, + workOutstandingFailure, + } = await import('./lifecycle-budget.ts'); const create = await awaitLimrunWithinBudget(owner.creation, CREATE_ANSWER_BUDGET_MS); // The create answered too late for this disposal to wait on it; what it goes on to name is still this // disposal's to end. @@ -81,9 +86,11 @@ export async function disposeLimrunLeaseOwner( if (instanceId === undefined) return await disposeUnrented(host, owner); // The capture's record is optional reporting, and local teardown can outlive its own steps: neither is // allowed to stand between this disposal and the delete that stops the instance from being billed, so both - // are started here and joined after it. - const capture = host.endCapture(owner); - const local = releaseOwnerResources(owner); + // are started here and joined after it. Both are observed in this same step: a step of either one that + // refuses before the join below is reached has this disposal as its observer from the moment it started, + // so no refusal of a resource this route already owns can reach the process as an unhandled rejection. + const capture = keepLimrunObserved(host.endCapture(owner)); + const local = keepLimrunObserved(releaseOwnerResources(owner)); const instance = await endInstance(host, owner, session, instanceId); const stuck = await awaitLocalTeardown(local); // The record is optional reporting joined last: a capture that refuses to be written cannot keep a release @@ -94,16 +101,28 @@ export async function disposeLimrunLeaseOwner( instanceId, failure: joinOutcomeFailures( instance.failure, - joinOutcomeFailures( - stuck, - joined.ok ? undefined : workOutstandingFailure('Capture record', CAPTURE_RECORD_BUDGET_MS), - ), + joinOutcomeFailures(stuck, captureRecordOutcome(joined, workOutstandingFailure)), ), cause: instance.cause, capture: joined.ok ? joined.value : undefined, }; } +/** + * What the joined capture record leaves to report: nothing, a refusal of the record itself, or a wait + * that ran out. A refusal is reported as what it is, because a record that will never be written is not + * the same thing as one that may still arrive. + */ +function captureRecordOutcome( + joined: LimrunBudgetOutcome, + outstanding: (work: string, budgetMs: number) => string, +): string | undefined { + if (joined.ok) return undefined; + return 'rejected' in joined + ? 'Capture record refused to be written' + : outstanding('Capture record', CAPTURE_RECORD_BUDGET_MS); +} + /** * Ends what a disposal holds when the create it waited for never answered. Nothing is known to be running, so * the lease is kept held and refused rather than given back — and the create is watched, because whatever it @@ -113,8 +132,11 @@ async function disposeUnansweredCreate( host: LimrunDisposalHost, owner: LimrunLeaseOwner, ): Promise { - const { joinOutcomeFailures, workOutstandingFailure } = await import('./lifecycle-budget.ts'); - const capture = awaitEndedCapture(host, owner); + const { joinOutcomeFailures, keepLimrunObserved, workOutstandingFailure } = + await import('./lifecycle-budget.ts'); + // Observed the step it starts in, like the disposal's other early work: this route answers only after + // the local teardown below, so a capture end that refuses meanwhile has its observer already beside it. + const capture = keepLimrunObserved(awaitEndedCapture(host, owner)); const outstanding = workOutstandingFailure( `Instance create for lease '${owner.lease.leaseId}'`, CREATE_ANSWER_BUDGET_MS, @@ -240,15 +262,21 @@ async function endInstance( } const deleted = await awaitLimrunWithinBudget(requested, DELETE_ANSWER_BUDGET_MS); if (!deleted.ok) { - void requested.then((answer) => { - if (answer.failure === undefined) confirmLateDelete(owner, instanceId); - }); + // Both branches of the request are answered for here: this continuation outlives its budget, and the + // route that started it has stopped waiting, so nothing it reports later may reach the process as an + // unhandled rejection. + void requested.then( + (answer) => { + if (answer.failure === undefined) confirmLateDelete(owner, instanceId); + }, + () => undefined, + ); return { cleanup: 'uncertain', - failure: workOutstandingFailure( - `Delete of instance '${instanceId}'`, - DELETE_ANSWER_BUDGET_MS, - ), + failure: + 'rejected' in deleted + ? `Delete of instance '${instanceId}' could not be attempted` + : workOutstandingFailure(`Delete of instance '${instanceId}'`, DELETE_ANSWER_BUDGET_MS), cause: undefined, }; } @@ -303,21 +331,17 @@ async function sessionTheCreateBuilt( } /** - * Reports the local teardown a disposal had to stop waiting for. A step that refused is reported as one that - * never answered: neither is something this runtime can still fix, and the delete that ran beside it has - * already been answered. + * Reports the local teardown a disposal had to stop waiting for. A step that refused and a step that never + * answered are both reported as what they are: neither is something this runtime can still fix, and the + * delete that ran beside it has already been answered. */ async function awaitLocalTeardown(local: Promise): Promise { const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); - const stuck = workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS); - const ended = await awaitLimrunWithinBudget( - local.then( - (): string | undefined => undefined, - (): string => stuck, - ), - LOCAL_TEARDOWN_BUDGET_MS, - ); - return ended.ok ? ended.value : stuck; + const ended = await awaitLimrunWithinBudget(local, LOCAL_TEARDOWN_BUDGET_MS); + if (ended.ok) return undefined; + return 'rejected' in ended + ? 'Local teardown refused to finish' + : workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS); } /** What a release answers for one lease: the instance it ended, with what it left and wrote. */ diff --git a/packages/provider-limrun/src/lifecycle-budget.ts b/packages/provider-limrun/src/lifecycle-budget.ts index 84b3ea9115..06af3d3185 100644 --- a/packages/provider-limrun/src/lifecycle-budget.ts +++ b/packages/provider-limrun/src/lifecycle-budget.ts @@ -8,25 +8,34 @@ * it runs to its end, and its owner disposes what it eventually makes. */ -/** How the wait ended, kept so a caller states which work was still outstanding rather than guessing. */ +/** + * How the wait ended, kept so a caller states which work was still outstanding rather than guessing. + * A work item that refused is an outcome of its own: every wait this type answers for is reporting, + * cleanup, or a handshake somebody else can still act on, so a refusal is never this route's failure + * to re-raise — it maps to this variant and the caller says what it could not use. + */ export type LimrunBudgetOutcome = | Readonly<{ ok: true; value: T }> - | Readonly<{ ok: false; expired: true }>; + | Readonly<{ ok: false; expired: true }> + | Readonly<{ ok: false; rejected: true }>; /** * Waits for `work` within `budgetMs`, unref'd so a daemon close is never held open by the timer alone. - * A work item that was too slow to be awaited is kept observed, so the outcome it reports later cannot - * surface as an unhandled rejection. + * Both of the work's answers are mapped to outcomes in this step, so a work item that rejects can + * never reach the process as an unhandled rejection, whether it answers inside the budget or after it. */ export async function awaitLimrunWithinBudget( work: Promise, budgetMs: number, ): Promise> { - void Promise.resolve(work).catch(() => undefined); + const settled = Promise.resolve(work).then( + (value): LimrunBudgetOutcome => ({ ok: true, value }), + (): LimrunBudgetOutcome => ({ ok: false, rejected: true }), + ); let timer: ReturnType | undefined; try { return await Promise.race([ - work.then((value): LimrunBudgetOutcome => ({ ok: true, value })), + settled, new Promise>((expired) => { timer = setTimeout(() => expired({ ok: false, expired: true }), budgetMs); timer.unref?.(); @@ -37,6 +46,17 @@ export async function awaitLimrunWithinBudget( } } +/** + * Keeps work that was started without being awaited yet observed for as long as that lasts: the handler + * attached here is what stops a refusal that arrives before the awaiting step from reaching the process as + * an unhandled rejection. The same promise comes back, so the route that starts the work is still the one + * that awaits what it answers with. + */ +export function keepLimrunObserved(work: Promise): Promise { + void work.catch(() => undefined); + return work; +} + /** * Why an outcome is not confirmed: the work this runtime stopped waiting for. It is reported rather than * dropped, because whoever reads it has to say whether the resource may still be running. diff --git a/packages/provider-limrun/src/runtime-dependencies.fixtures.ts b/packages/provider-limrun/src/runtime-dependencies.fixtures.ts new file mode 100644 index 0000000000..54b71674b8 --- /dev/null +++ b/packages/provider-limrun/src/runtime-dependencies.fixtures.ts @@ -0,0 +1,61 @@ +import type { AppsFilter } from '@agent-device/contracts/device'; +import type { Interactor } from '@agent-device/contracts/interactor-types'; +import { AppError } from '@agent-device/kernel/errors'; +import { + androidAdbHostTarget, + androidAdbInvocation, + androidAdbSerialTarget, +} from '@agent-device/platform-android/mechanics'; +import type { LimrunAdbExecutor, LimrunRuntimeDependencies } from './runtime-dependencies.ts'; + +/** What no case here exercises: the keyboard is never visible and nothing owns input. */ +const NO_KEYBOARD = Object.freeze({ + visible: false as const, + inputOwner: 'unknown' as const, +}); + +/** + * The host and platform adapters the runtime tests hand `createLimrunRuntime`: every one of them answers + * a fixed value, so a case that reaches one of them has left the path it means to drive. A case with a + * reading of its own spreads this object and replaces the member it drives. + */ +export function stubLimrunRuntimeDependencies(): LimrunRuntimeDependencies { + return { + clientVersion: 'test-version', + android: { + createInteractor: () => ({}) as Interactor, + createPortReverse: async () => ({ + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + list: async () => [], + }), + inferAppName: async () => 'Example', + listApps: async (_adb: LimrunAdbExecutor, _filter: AppsFilter) => [], + getForegroundApp: async () => undefined, + getKeyboardState: async () => ({ ...NO_KEYBOARD }), + dismissKeyboard: async () => ({ + ...NO_KEYBOARD, + attempts: 0, + wasVisible: false, + dismissed: false, + }), + readLogs: async () => '', + forceStopApp: async () => {}, + deviceAdbInvocation: (serial, command) => + androidAdbInvocation(androidAdbSerialTarget(serial), command), + hostAdbInvocation: (command) => androidAdbInvocation(androidAdbHostTarget(), command), + adbError: async (message) => new AppError('COMMAND_FAILED', message), + }, + host: { + runAdb: async () => ({ stdout: '', stderr: '', exitCode: 0 }), + archiveDirectory: async () => undefined, + downloadFile: async () => undefined, + }, + ios: { + applySimctlSetting: async () => undefined, + resolveAppAlias: async (app) => app, + readBundleAppName: async () => undefined, + }, + }; +} diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index e2ccca46ee..064299aa01 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -7,12 +7,8 @@ import type { Interactor } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformRequestScope } from '@agent-device/contracts/platform-runtime-host'; -import { - androidAdbHostTarget, - androidAdbInvocation, - androidAdbSerialTarget, -} from '@agent-device/platform-android/mechanics'; import { createLimrunRuntime } from './runtime.ts'; +import { stubLimrunRuntimeDependencies } from './runtime-dependencies.fixtures.ts'; import { unusedLimrunRuntimeHost } from './runtime.fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; import type { @@ -835,21 +831,37 @@ test('a create that fails after its session was built disconnects that session', } }); -/** The alias a lease was created with answers for the asset name its caller reuses. */ +/** + * The alias a lease was created with answers for the asset name its caller reuses — through the binding + * the daemon actually resolves open targets with, and only while the lease is held: once it is released, + * the name answers as the plain string it is, mapped by nothing. + */ test('a created app alias resolves the asset name back to the installed identifier', async () => { state.uploadedApps = [{ id: 'asset-1', name: 'settings.apk', md5: 'md5', os: 'android' }]; state.installedApps = [{ id: 'com.example.settings' }]; - const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); - const resolve = (device: DeviceInfo, app: string) => runtime.resolveAppReference(device, app); + const registration = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies(), { + includePlatformModule: true, + }); try { - const device = await allocateAndroidDevice(runtime, 'lease-alias', 'settings.apk'); - assert.equal(resolve(device, 'settings.apk'), 'com.example.settings'); - assert.equal(resolve(device, 'com.other.app'), 'com.other.app'); - assert.equal(resolve({ ...device, id: 'other:android:x' }, 'settings.apk'), 'settings.apk'); + const device = await allocateAndroidDevice(registration.runtime, 'lease-alias', 'settings.apk'); + const owner = await registration.platformModule.loadRuntime(unusedLimrunRuntimeHost()); + const binding = await owner.bind({ + device, + intent: { kind: 'ordinary' }, + scope: requestScope(), + }); + const resolve = async (app: string) => + await binding.operations.resolveOpenTarget?.({ target: app, surface: 'app' }); + assert.equal((await resolve('settings.apk'))?.appBundleId, 'com.example.settings'); + assert.equal((await resolve('com.other.app'))?.appBundleId, 'com.other.app'); + + // The alias belonged to the lease, and the lease is gone: nothing maps the asset name any more. + await registration.runtime.shutdown(); + assert.equal((await resolve('settings.apk'))?.appBundleId, 'settings.apk'); } finally { state.uploadedApps = []; state.installedApps = []; - await runtime.shutdown(); + await registration.runtime.shutdown(); } }); @@ -954,47 +966,18 @@ function androidLease(leaseId: string): DeviceLease { } function testDependencies(): LimrunRuntimeDependencies { + const stub = stubLimrunRuntimeDependencies(); return { - clientVersion: 'test-version', + ...stub, android: { + ...stub.android, + // The ADB provider an interactor was handed is kept: a case drives it after its lease is gone. createInteractor: (_device, adb) => { state.heldAdbProvider = adb; return {} as Interactor; }, - createPortReverse: async () => ({ - ensure: async () => {}, - remove: async () => {}, - removeAllOwned: async () => {}, - list: async () => [], - }), - inferAppName: async () => 'Example', listApps: async (_adb: LimrunAdbExecutor, _filter: AppsFilter) => state.installedApps, getForegroundApp: async () => (state.foreground ? { ...state.foreground } : undefined), - getKeyboardState: async () => ({ visible: false, inputOwner: 'unknown' as const }), - dismissKeyboard: async () => ({ - visible: false, - inputOwner: 'unknown' as const, - attempts: 0, - wasVisible: false, - dismissed: false, - }), - readLogs: async () => '', - forceStopApp: async () => {}, - deviceAdbInvocation: (serial: string, command: readonly string[]) => - androidAdbInvocation(androidAdbSerialTarget(serial), command), - hostAdbInvocation: (command: readonly string[]) => - androidAdbInvocation(androidAdbHostTarget(), command), - adbError: async (message: string) => new AppError('COMMAND_FAILED', message), - }, - host: { - runAdb: async () => ({ stdout: '', stderr: '', exitCode: 0 }), - archiveDirectory: async () => undefined, - downloadFile: async () => undefined, }, - ios: { - applySimctlSetting: async () => undefined, - resolveAppAlias: async (app: string) => app, - readBundleAppName: async () => undefined, - }, - } satisfies LimrunRuntimeDependencies; + }; } diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index a7dca03165..3e8d94ffbc 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -83,8 +83,6 @@ export type LimrunRuntimeOptions = { export type LimrunRuntime = ProviderDeviceRuntime & { recoverExpiredLease: ProviderExpiredLeaseRecovery; getDeviceSession(device: DeviceInfo): LimrunDeviceSession | undefined; - /** Maps an uploaded asset name this lease was created with back to its installed identifier. */ - resolveAppReference(device: DeviceInfo, app: string): string; }; export type LimrunRuntimeRegistration = Readonly<{ From 90a3cd146ac38a9f99a34c8ec157b95e7bc6f540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 21:31:24 +0100 Subject: [PATCH 06/11] =?UTF-8?q?fix(limrun):=20answer=20every=20bot=20fin?= =?UTF-8?q?ding=20at=20385fd27=20=E2=80=94=20recovery=20deletes,=20budgets?= =?UTF-8?q?,=20read=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The label-based recovery route ends instances through the same delete helper every other route uses, so a 404 confirms there exactly as it does everywhere else, and a delete the helper could not confirm refuses with the typed unconfirmed-delete error — naming the instance left in question beside every id the labels carried, and which of them are already gone. The daemon's expiry sweep is that refusal's retry. A disposal's join for a capture record now outlasts the destination tunnel's own open budget it contains, so a handshake that answers at the end of its stated budget is no longer reported as a refused record. An artifacts read answers at once for a capture still opening its tunnel: the read is not the route that owns that handshake, and waiting it bought an immediate answer nothing to wait twenty seconds for. A take-refusal names what the disposal actually confirmed: an uncertain delete is said as unconfirmed, not as an instance that was ended. The release-lease hint keeps the same honesty. Tests: recovery route cases (confirm, 404, refusal with both ids), the budget ordering, the read path's immediate answer, and the take-refusal's two hints; typed delete mocks retire the cast assertions, and one pins the single-attempt options argument. --- .../src/capture-lifecycle.test.ts | 18 ++ .../provider-limrun/src/capture-lifecycle.ts | 17 +- .../provider-limrun/src/lease-ownership.ts | 5 +- .../provider-limrun/src/lease-release.test.ts | 165 ++++++++++++++++++ packages/provider-limrun/src/lease-release.ts | 69 ++++++-- src/__tests__/limrun-runtime.test.ts | 20 +-- src/provider-device-runtimes.test.ts | 7 +- 7 files changed, 262 insertions(+), 39 deletions(-) diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts index 0acdb0941a..0039ebd80d 100644 --- a/packages/provider-limrun/src/capture-lifecycle.test.ts +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -37,6 +37,24 @@ test('a repeat allocation stops waiting on a destination tunnel that never answe assert.equal(owner.capture?.phase, 'starting'); }); +/** + * An `artifacts` read for a capture whose destination tunnel is still being opened answers on the spot: + * the read has nothing to wait for — the tunnel belongs to the lease's release, and a stalled handshake + * would otherwise hold the request for the whole open budget to say nothing new. + */ +test('an artifacts read answers at once for a capture whose tunnel is still opening', async () => { + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + const listed = await Promise.race([ + lifecycle.list(owner.lease.leaseId, owner.capture), + new Promise<'stalled'>((resolve) => { + const timer = setTimeout(() => resolve('stalled'), 250); + timer.unref?.(); + }), + ] as const); + assert.equal(listed, undefined); +}); + function captureHost(held: readonly LimrunLeaseOwner[] = []): LimrunCaptureHost { return { harDir: undefined, diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts index 873ba45555..0d4da09ac3 100644 --- a/packages/provider-limrun/src/capture-lifecycle.ts +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -9,9 +9,11 @@ import { errorMessage } from '@agent-device/kernel/errors'; /** * How long a read or a teardown waits for a destination tunnel that is still being opened. The SDK gives this - * handshake fifteen seconds of its own. + * handshake fifteen seconds of its own. Exported because the disposal that joins a capture end must give it + * more than this: `end` waits the handshake inside its own call, and a join shorter than it would report a + * stalled-but-in-budget handshake as a refused record. */ -const CAPTURE_OPEN_BUDGET_MS = 20_000; +export const CAPTURE_OPEN_BUDGET_MS = 20_000; /** Released captures stay listable; the oldest settled records are forgotten past this count. */ const SETTLED_NETWORK_CAPTURE_LIMIT = 32; @@ -134,12 +136,11 @@ export class LimrunCaptureLifecycle { const { writeCaptureRecord } = await import('./network-har.ts'); return await writeCaptureRecord(held); } - // A capture whose tunnel is still being opened has no traffic to report yet; that tunnel stays this - // lease's own to end, which is its release's route, not this read. - const opened = await awaitLimrunWithinBudget(held.opening, CAPTURE_OPEN_BUDGET_MS); - if (!opened.ok) return undefined; - const { writeCaptureRecord } = await import('./network-har.ts'); - return await writeCaptureRecord(opened.value); + // A capture whose tunnel is still being opened has no traffic to report yet, and this read is not the + // route that owns that tunnel: its release is. Waiting the open budget here would hold an `artifacts` + // request for a handshake whose outcome this read cannot act on, so the read answers now with whatever + // record this lease already has and lets the release write the rest. + return this.settled.get(leaseId); } /** Keeps one released capture's record, forgetting the oldest once the record limit is passed. */ diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts index f18d7eb374..07b7b99e04 100644 --- a/packages/provider-limrun/src/lease-ownership.ts +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -100,8 +100,9 @@ export type LimrunDisposalOutcome = Readonly<{ /** Why a disposal could not be confirmed; absent once the instance is gone. */ failure: string | undefined; /** - * The refusal behind that failure, when one exists: the delete's own error. Reported as an error's cause, - * where it renders, rather than only as a string in details. + * The refusal behind that failure, when one exists: the delete's own error, or the crash of the disposal + * itself when it could not report anything. Reported as an error's cause, where it renders, rather than + * only as a string in details. */ cause: unknown; /** The released capture's record, so a refusal can still carry the HAR that was written. */ diff --git a/packages/provider-limrun/src/lease-release.test.ts b/packages/provider-limrun/src/lease-release.test.ts index aabbb8a4f4..99aa92c020 100644 --- a/packages/provider-limrun/src/lease-release.test.ts +++ b/packages/provider-limrun/src/lease-release.test.ts @@ -1,10 +1,13 @@ import { test, vi, expect } from 'vitest'; import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; +import type Limrun from '@limrun/api'; import { + allocationRefusedForTake, cleanupLeftInstancesRunning, createLeftInstanceRunning, disposeLimrunLeaseOwner, + releaseLimrunLease, type LimrunDisposalHost, } from './lease-release.ts'; import type { @@ -131,6 +134,168 @@ test('a close that could not end two leases reports both reasons', async () => { expect(thrown.cause).toBe(refusal); }); +/** + * The label-based recovery route deletes through the same helper every other route deletes with: a 404 is + * Limrun answering that the instance is gone, and a delete it could not confirm refuses with the typed error + * naming every instance the labels carried, because the sweep that retries this route has nothing else to go on. + */ +test('recovery by labels confirms a gone instance and reports every id it ended', async () => { + const deletes: string[] = []; + const limrun = labeledInstancesClient( + ['android-instance-a', 'android-instance-b'], + async (id) => { + deletes.push(id); + }, + ); + const record = await releaseLimrunLease(limrun, leaseForRecovery(), undefined); + expect(deletes).toEqual(['android-instance-a', 'android-instance-b']); + expect(record).toMatchObject({ + limrunInstanceId: 'android-instance-a', + limrunInstanceIds: ['android-instance-a', 'android-instance-b'], + limrunInstanceCount: 2, + }); +}); + +test('recovery by labels treats a 404 as confirmed, the way every other delete route does', async () => { + const limrun = labeledInstancesClient(['android-instance-a'], async () => { + throw Object.assign(new Error('not found'), { status: 404 }); + }); + const record = await releaseLimrunLease(limrun, leaseForRecovery(), undefined); + expect(record).toMatchObject({ limrunInstanceId: 'android-instance-a' }); +}); + +/** + * A label delete Limrun refused is a lease the daemon's sweep must retry: the refusal is the typed one, it + * names the instance still in question beside every id the labels had, and the confirmed ones are said apart. + */ +test('recovery by labels refuses with every id when a delete is not confirmed', async () => { + const limrun = labeledInstancesClient( + ['android-instance-a', 'android-instance-b'], + async (id) => { + if (id === 'android-instance-b') throw new Error('Limrun refused to delete the instance'); + return undefined; + }, + ); + const thrown = await captureReject(() => + releaseLimrunLease(limrun, leaseForRecovery(), undefined), + ); + expect(thrown.details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect(thrown.details?.limrunInstanceId).toBe('android-instance-b'); + expect(thrown.details?.limrunInstanceIds).toEqual(['android-instance-a', 'android-instance-b']); + expect(String(thrown.details?.hint)).toContain( + "Ask Limrun to delete instance 'android-instance-b'", + ); + expect(String(thrown.details?.hint)).toContain('android-instance-a was confirmed deleted'); + expect(String(thrown.cause)).toContain('refused'); +}); + +/** + * What a close says when the work it could not finish has no instance to name yet: the lease and the work. + * A take-refusal for an instance whose delete was not confirmed says so rather than calling it ended: + * the refused caller is the last route that will hear the instance may still be billing. + */ +test('a take-refusal for an unconfirmed delete does not call the instance ended', () => { + const lease = leaseForRecovery(); + const confirmed = captureThrow(() => + allocationRefusedForTake(lease, 'release', { + cleanup: 'confirmed', + instanceId: 'android-instance-1', + failure: undefined, + cause: undefined, + capture: undefined, + }), + ); + expect(String(confirmed.details?.hint)).toContain('was ended by the release'); + + const uncertain = captureThrow(() => + allocationRefusedForTake(lease, 'shutdown', { + cleanup: 'uncertain', + instanceId: 'android-instance-1', + failure: 'delete refused', + cause: new Error('refused'), + capture: undefined, + }), + ); + expect(String(uncertain.details?.hint)).toContain('could not confirm'); + expect(String(uncertain.details?.hint)).toContain('android-instance-1'); + expect(uncertain.details?.retriable).toBe(true); +}); + +/** + * The disposal's join for a capture record contains the destination tunnel's own open budget: a capture end + * that spends almost all of that handshake and still comes to in time is answered for without a warning, + * because the join is longer than the work inside it. + */ +test('a capture end that spends its tunnel budget is still joined in time', async () => { + await import('./lifecycle-budget.ts'); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ failure: undefined, cause: undefined }), + // The tunnel answers at the very end of its own 20 s budget, then the record is written. + endCapture: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 20_000); + }); + return { phase: 'written', harPath: '/tmp/late.har', failures: { count: 0, truncated: 0 } }; + }, + }; + const owner = ownerWithRentedInstance( + // A create that failed but named the instance (rentedInstanceId is set by the fixture): the disposal + // ends that instance and joins the capture record it started beside the delete. + Promise.resolve({ + status: 'failed', + error: new Error('create answered after its session was gone'), + }), + ); + vi.useFakeTimers(); + let settled: LimrunDisposalOutcome | undefined; + try { + const disposal = disposeLimrunLeaseOwner(host, owner).then((outcome) => { + settled = outcome; + }); + await vi.advanceTimersByTimeAsync(31_000); + await disposal; + } finally { + vi.useRealTimers(); + } + expect(settled?.cleanup).toBe('confirmed'); + expect(settled?.failure).toBeUndefined(); +}); + +function captureReject(raise: () => Promise): Promise { + return raise().then( + () => { + throw new Error('expected the refusal to reject'); + }, + (error: unknown) => error as AppError, + ); +} + +/** A Limrun whose labels name these instances, answering every delete the way a case says. */ +function labeledInstancesClient( + instanceIds: readonly string[], + remove: (instanceId: string) => Promise, +): Limrun { + const page = { getPaginatedItems: () => instanceIds.map((id) => ({ metadata: { id } })) }; + const resource = { + list: async () => page, + delete: async (instanceId: string) => await remove(instanceId), + }; + return { iosInstances: resource, androidInstances: resource } as unknown as Limrun; +} + +function leaseForRecovery(): DeviceLease { + return { + leaseId: 'lease-labeled-recovery', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + }; +} + function captureThrow(raise: () => never): AppError { try { raise(); diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index f683cf51c1..1c212ed32a 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -2,7 +2,8 @@ import type Limrun from '@limrun/api'; import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; import { cleanupLimrunAndroidAdbTunnel } from './android.ts'; -import { LIMRUN_SINGLE_ATTEMPT } from './client-options.ts'; +import { CAPTURE_OPEN_BUDGET_MS } from './capture-lifecycle.ts'; +import { deleteLimrunInstance } from './client-options.ts'; import type { LimrunBudgetOutcome } from './lifecycle-budget.ts'; import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend, type LimrunPlatform } from './device.ts'; import type { @@ -44,8 +45,13 @@ const DELETE_ANSWER_BUDGET_MS = 30_000; */ const LOCAL_TEARDOWN_BUDGET_MS = 45_000; -/** How long a disposal waits for a capture's HAR record, which is reporting and never a reason to hold a lease. */ -const CAPTURE_RECORD_BUDGET_MS = 15_000; +/** + * How long a disposal waits for a capture's HAR record, which is reporting and never a reason to hold a lease. + * The disposal joins a capture end that contains the destination tunnel's own open budget, so this is that + * handshake plus a window of its own for writing the HAR: a join shorter than the work inside it would report + * a handshake that is still inside its own stated budget as a record that refused. + */ +const CAPTURE_RECORD_BUDGET_MS = CAPTURE_OPEN_BUDGET_MS + 10_000; /** * What a disposal may ask of the runtime that owns the lease, and nothing else: the delete of one instance, and @@ -388,7 +394,13 @@ export async function releaseLimrunLease( }; } -/** Ends every instance a lease's own labels name, for a lease this runtime no longer holds an owner for. */ +/** + * Ends every instance a lease's own labels name, for a lease this runtime no longer holds an owner for. + * Every delete goes through the same helper every other route deletes with, so a 404 confirms here exactly + * as it does there; a delete the helper could not confirm refuses with the typed unconfirmed-delete error, + * naming every instance the labels carried and which of them are already confirmed gone. The refusal keeps + * the lease held: the daemon's expiry sweep is the retry, and nothing else knows these ids. + */ async function releaseLabeledInstances( limrun: Limrun | undefined, lease: DeviceLease, @@ -401,17 +413,37 @@ async function releaseLabeledInstances( ? await limrun.iosInstances.list({ labelSelector }) : await limrun.androidInstances.list({ labelSelector }); const instanceIds = instances.getPaginatedItems().map((instance) => instance.metadata.id); + const confirmed: string[] = []; for (const instanceId of instanceIds) { - // One attempt each, the same contract every other instance delete keeps: this route's own retries are - // the daemon's expiry sweeps, not the SDK's. - if (platform === 'ios') { - await limrun.iosInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); - } else { - await limrun.androidInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); + const deleted = await deleteLimrunInstance(limrun, platform, instanceId); + if (deleted.failure !== undefined) { + throwUnconfirmedDelete( + `Recovery could not confirm instance '${instanceId}' of lease '${lease.leaseId}' deleted`, + { limrunInstanceId: instanceId, limrunInstanceIds: instanceIds }, + deleted.failure, + recoveryHint(instanceId, instanceIds, confirmed), + deleted.cause, + ); } + confirmed.push(instanceId); } if (instanceIds.length === 0) return undefined; - return { limrunInstanceId: instanceIds[0], limrunInstanceCount: instanceIds.length }; + return { + limrunInstanceId: instanceIds[0], + limrunInstanceIds: instanceIds, + limrunInstanceCount: instanceIds.length, + }; +} + +/** What the recovery refusal tells the operator to name to Limrun: the instance left, and those gone. */ +function recoveryHint( + instanceId: string, + instanceIds: readonly string[], + confirmed: string[], +): string { + if (confirmed.length === 0) return `Ask Limrun to delete: ${instanceIds.join(', ')}.`; + const gone = `${confirmed.length > 1 ? 'Instances' : 'Instance'} ${confirmed.join(', ')} ${confirmed.length > 1 ? 'were' : 'was'} confirmed deleted.`; + return `Ask Limrun to delete instance '${instanceId}'. ${gone}`; } /** @@ -443,6 +475,16 @@ export function allocationRefusedForTake( outcome: LimrunDisposalOutcome, ): never { const { instanceId } = outcome; + // The hint says what the disposal actually confirmed: an unconfirmed delete leaves the instance possibly + // still billing, and that caller is the last route left that will hear about it. + const ended = + instanceId === undefined + ? undefined + : outcome.cleanup === 'uncertain' + ? `The ${by} of this lease could not confirm instance '${instanceId}' deleted. Release the lease again, or ask Limrun to delete it.` + : outcome.cleanup === 'attached' + ? `Instance '${instanceId}' is the caller's own attached instance and was left running. Open a new lease.` + : `Instance '${instanceId}' was ended by the ${by} of this lease. Open a new lease.`; throw new AppError( 'DEVICE_IN_USE', `Limrun lease '${lease.leaseId}' was released while its instance was being prepared`, @@ -450,10 +492,7 @@ export function allocationRefusedForTake( reason: 'limrun_lease_released_mid_allocation', ...(instanceId === undefined ? { hint: `Instance create for lease '${lease.leaseId}' may still rent one. Check Limrun.` } - : { - limrunInstanceId: instanceId, - hint: `Instance '${instanceId}' was ended by the ${by} of this lease. Open a new lease.`, - }), + : { limrunInstanceId: instanceId, ...(ended === undefined ? {} : { hint: ended }) }), ...(outcome.cleanup === 'uncertain' ? { retriable: true, ...(outcome.failure === undefined ? {} : { cause: outcome.failure }) } : {}), diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index 7a143ca615..c1a7569da1 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -52,7 +52,7 @@ const limrunMockState = vi.hoisted(() => { iosList: vi.fn<() => Promise>(async () => ({ getPaginatedItems: () => [], })), - iosDelete: vi.fn(async () => undefined), + iosDelete: vi.fn(async (_instanceId: string, _options?: { maxRetries: number }) => undefined), androidCreate: vi.fn(async () => ({ metadata: { id: 'android-instance-1' }, status: { @@ -64,7 +64,9 @@ const limrunMockState = vi.hoisted(() => { androidList: vi.fn<() => Promise>(async () => ({ getPaginatedItems: () => [], })), - androidDelete: vi.fn(async () => undefined), + androidDelete: vi.fn( + async (_instanceId: string, _options?: { maxRetries: number }) => undefined, + ), }; }); @@ -255,7 +257,7 @@ test('Limrun reclaims a labeled iOS instance without an in-memory session', asyn [{ labelSelector: 'provider=limrun,leaseId=lease-recovered-ios' }], ]); assert.deepEqual( - limrunMockState.iosDelete.mock.calls.map((call) => (call as unknown[])[0]), + limrunMockState.iosDelete.mock.calls.map(([instanceId]) => instanceId), ['ios-instance-recovered'], ); } finally { @@ -695,10 +697,7 @@ test('Limrun deletes iOS instance when post-create validation fails', async () = const allocateLease = runtime.leaseLifecycle.allocate; if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); await assert.rejects(() => allocateLease(lease), /did not expose apiUrl/); - assert.equal( - (limrunMockState.iosDelete.mock.calls[0] as unknown[] | undefined)?.[0], - 'ios-instance-missing-api', - ); + assert.equal(limrunMockState.iosDelete.mock.calls[0]?.[0], 'ios-instance-missing-api'); } finally { await runtime.shutdown(); } @@ -730,10 +729,7 @@ test('Limrun deletes Android instance when post-create validation fails', async const allocateLease = runtime.leaseLifecycle.allocate; if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); await assert.rejects(() => allocateLease(lease), /did not expose API and ADB/); - assert.equal( - (limrunMockState.androidDelete.mock.calls[0] as unknown[] | undefined)?.[0], - 'android-instance-missing-adb', - ); + assert.equal(limrunMockState.androidDelete.mock.calls[0]?.[0], 'android-instance-missing-adb'); } finally { await runtime.shutdown(); } @@ -919,7 +915,7 @@ test('Limrun instance access wins over the API key for its platform only', async assert.equal(limrunMockState.iosDelete.mock.calls.length, 0); assert.equal(limrunMockState.androidCreate.mock.calls.length, 1); assert.deepEqual( - limrunMockState.androidDelete.mock.calls.map((call) => (call as unknown[])[0]), + limrunMockState.androidDelete.mock.calls.map(([instanceId]) => instanceId), ['android-instance-1'], ); }); diff --git a/src/provider-device-runtimes.test.ts b/src/provider-device-runtimes.test.ts index da13e53ab8..a023aba458 100644 --- a/src/provider-device-runtimes.test.ts +++ b/src/provider-device-runtimes.test.ts @@ -13,7 +13,7 @@ const limrun = vi.hoisted(() => ({ status: { token: 'instance-token', apiUrl: 'https://ios.example' }, })), iosList: vi.fn(async () => ({ getPaginatedItems: () => [] as { metadata: { id: string } }[] })), - iosDelete: vi.fn(async (_instanceId: string) => undefined), + iosDelete: vi.fn(async (_instanceId: string, _options?: { maxRetries: number }) => undefined), iosDisconnect: vi.fn(), })); @@ -93,7 +93,10 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest assert.deepEqual(limrun.iosList.mock.calls.at(-1), [ { labelSelector: `provider=limrun,leaseId=${lease.leaseId}` }, ]); - assert.equal(limrun.iosDelete.mock.calls.at(-1)?.[0], 'ios-instance-recovered'); + assert.deepEqual(limrun.iosDelete.mock.calls.at(-1), [ + 'ios-instance-recovered', + { maxRetries: 0 }, + ]); assert.equal(fs.existsSync(pendingPath), false); } finally { dateNow.mockRestore(); From ed0d64ebfcf1a886e97d29ae1027fb0c529e87de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 22:12:03 +0100 Subject: [PATCH 07/11] fix(limrun): finish bounded release and capture cleanup --- packages/provider-limrun/src/android.ts | 11 +- .../src/capture-lifecycle.test.ts | 114 +++++++++++++++++- .../provider-limrun/src/capture-lifecycle.ts | 39 +++--- ...t.ts => lease-release-observation.test.ts} | 20 ++- .../provider-limrun/src/lease-release.test.ts | 85 ++++++++++++- packages/provider-limrun/src/lease-release.ts | 59 ++++++--- .../src/runtime-dependencies.fixtures.ts | 8 +- packages/provider-limrun/src/runtime.test.ts | 31 +++-- packages/provider-limrun/src/runtime.ts | 8 ++ 9 files changed, 312 insertions(+), 63 deletions(-) rename packages/provider-limrun/src/{dispose-observation.test.ts => lease-release-observation.test.ts} (91%) diff --git a/packages/provider-limrun/src/android.ts b/packages/provider-limrun/src/android.ts index 609593f8fc..e532f5e512 100644 --- a/packages/provider-limrun/src/android.ts +++ b/packages/provider-limrun/src/android.ts @@ -247,6 +247,11 @@ async function runLimrunAndroidAdb( * started. A provider that hands a caller command to the reverse route — a reverse command queued inside an * operation of its own — carries no marker, so the manager cannot have a caller's mapping bound at a serial * that is on its way out. A marker set by one session never authorises another session's route. + * + * Requirement this rests on: the reverse manager must dispatch its caller's commands within the caller's + * async context. A manager that answered from work it scheduled outside this call (an internal queue drained + * by a timer, or a driver loop made at construction) would run the teardown's removals without this marker, + * and the mappings would survive the session they were made for. */ const limrunAdbTeardownScope = new AsyncLocalStorage(); @@ -346,9 +351,9 @@ function alreadyCleanedUp(session: LimrunAndroidAdbSession): AppError { `Limrun ADB tunnel for instance '${session.instanceId}' was already cleaned up`, { reason: 'limrun_adb_tunnel_cleaned_up', - // An unconfirmed delete holds this lease while its release is retried, so the hint cannot promise the - // lease is finished; it says what this route is closed to, and what the caller can still do. - hint: `This interactor's ADB tunnel is closed for lease '${session.lease.leaseId}'. Release the lease again, or open a new lease once its instance is confirmed deleted.`, + // The tunnel is gone whether the instance's delete confirmed or is still held for retry, so the hint + // sends the caller to a new lease, and to Limrun only for the state where a release would refuse again. + hint: `This interactor's ADB tunnel is closed for lease '${session.lease.leaseId}'. Open a new lease. If this lease's release was refused as an unconfirmed delete, ask Limrun to delete the instance first.`, }, ); } diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts index 0039ebd80d..e7f567d84d 100644 --- a/packages/provider-limrun/src/capture-lifecycle.test.ts +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -1,4 +1,4 @@ -import { test, vi } from 'vitest'; +import { test, vi, expect } from 'vitest'; import assert from 'node:assert/strict'; import type { DeviceLease } from '@agent-device/contracts/device'; import { LimrunCaptureLifecycle, type LimrunCaptureHost } from './capture-lifecycle.ts'; @@ -55,6 +55,83 @@ test('an artifacts read answers at once for a capture whose tunnel is still open assert.equal(listed, undefined); }); +/** + * A destination tunnel that answers after the release stopped waiting is written and ended all the same. When + * that late end refuses — a capture whose traffic will not be read, a tunnel that will not close — the refusal + * is observed where it lands: the continuation outlives every budget and every caller, so nothing above it is + * left to catch it. + */ +test('a capture that answers late and then refuses is not loosed on the process', async () => { + // Loaded before the clock is replaced: the late end writes its HAR through this module, and a dynamic + // import that had to resolve under fake timers would never answer at all. + await import('./network-har.ts'); + const rejections = watchUnhandledRejections(); + let reportOpened!: (capture: NetworkCapture) => void; + const opening = new Promise((opened) => { + reportOpened = opened; + }); + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + owner.capture = { phase: 'starting', opening }; + vi.useFakeTimers(); + try { + const ended = lifecycle.end(owner); + // The release gives up on the handshake and answers for its lease; the tunnel is still its own to end. + await vi.advanceTimersByTimeAsync(CAPTURE_OPEN_WAIT_MS); + expect(await ended).toBeUndefined(); + // The tunnel arrives late and refuses to be read: the end of it fails with nobody left above it. + reportOpened({ + phase: 'capturing', + harPath: '/tmp/sm4-late-capture.har', + capture: { + entries: () => [], + failures: async () => { + throw new Error('the destination tunnel refused to answer'); + }, + close: () => {}, + }, + }); + await vi.advanceTimersByTimeAsync(1_000); + // Asserted while the listener is still attached, and after a yield: that is the only window in which a + // rejection this continuation loosed can be seen at all. + vi.useRealTimers(); + await rejections.assertIsSilent(); + } finally { + vi.useRealTimers(); + rejections.stop(); + } +}); + +/** + * The record limit counts a lease whose owner still holds a capture, so the very same test spares that + * record and no other. A lease the runtime still holds for an unconfirmed delete, whose capture is already + * ended, keeps nothing the limit should spare: its record competes for the limit like any released one, and + * the map cannot grow past it. + */ +test("a released lease's record competes for the limit even while its lease is still held", async () => { + // The runtime still holds this lease; its owner no longer holds a capture. + const stillHeld = ownerWithCapture('lease-held-no-capture'); + stillHeld.capture = undefined; + const lifecycle = new LimrunCaptureLifecycle(captureHost([stillHeld])); + await lifecycle.end( + Object.assign(ownerWithCapture(stillHeld.lease.leaseId), { + capture: { phase: 'unavailable', reason: 'capture-dir-unset' }, + }), + ); + // Past the limit, with the held lease's record the oldest of them. + for (let index = 0; index < 33; index += 1) { + const released = ownerWithCapture(`lease-released-${String(index)}`); + released.capture = { phase: 'unavailable', reason: 'capture-dir-unset' }; + await lifecycle.end(released); + } + + // The newest record still answers, and the record of the lease that holds no capture is the one that went. + expect(await lifecycle.list('lease-released-32', undefined)).toMatchObject({ + reason: 'capture-dir-unset', + }); + expect(await lifecycle.list(stillHeld.lease.leaseId, undefined)).toBeUndefined(); +}); + function captureHost(held: readonly LimrunLeaseOwner[] = []): LimrunCaptureHost { return { harDir: undefined, @@ -63,10 +140,39 @@ function captureHost(held: readonly LimrunLeaseOwner[] = []): LimrunCaptureHost }; } -/** An owner whose lease has a destination tunnel that has not answered, and never will. */ -function ownerWithCapture(): LimrunLeaseOwner { +/** + * Collects the rejections the process would die on: the daemon's failure mode for one is to take every other + * session down with it. The check yields to a macrotask first, because Node reports a rejection only after the + * current microtask drain — asserting before that yield, or removing the listener without one, could pass a + * leaked rejection unseen. + */ +function watchUnhandledRejections(): { + assertIsSilent: () => Promise; + stop: () => void; +} { + const seen: unknown[] = []; + const watch = (reason: unknown): void => { + seen.push(reason); + }; + process.on('unhandledRejection', watch); + return { + assertIsSilent: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + if (seen.length > 0) assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`); + }, + stop: () => process.off('unhandledRejection', watch), + }; +} + +/** + * An owner whose lease has a destination tunnel that has not answered, and never will. A case that drives a + * different lease overrides the id it needs. + */ +function ownerWithCapture(leaseId = 'lease-capture-join'): LimrunLeaseOwner { const lease = { - leaseId: 'lease-capture-join', + leaseId, tenantId: 'team-a', runId: 'run-a', backend: 'android-instance', diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts index 0d4da09ac3..5b728e9359 100644 --- a/packages/provider-limrun/src/capture-lifecycle.ts +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -9,9 +9,9 @@ import { errorMessage } from '@agent-device/kernel/errors'; /** * How long a read or a teardown waits for a destination tunnel that is still being opened. The SDK gives this - * handshake fifteen seconds of its own. Exported because the disposal that joins a capture end must give it - * more than this: `end` waits the handshake inside its own call, and a join shorter than it would report a - * stalled-but-in-budget handshake as a refused record. + * handshake fifteen seconds of its own; the extra five seconds is this runtime's margin over it. Exported + * because the disposal that joins a capture end must give it more than this: `end` waits the handshake inside + * its own call, and a join shorter than it would report a stalled-but-in-budget handshake as a refused record. */ export const CAPTURE_OPEN_BUDGET_MS = 20_000; @@ -112,14 +112,14 @@ export class LimrunCaptureLifecycle { // written and ended by the continuation below all the same, so it is never left without an owner. const opened = await awaitLimrunWithinBudget(state.opening, CAPTURE_OPEN_BUDGET_MS); if (opened.ok) return await endCapture(leaseId, opened.value, this.record); - void state.opening.then( - async (late) => { + // Observed end to end: this continuation outlives its budget, so neither a late tunnel that refuses to be + // written nor one whose close refuses may reach the process as an unhandled rejection. The tunnel it could + // not end stays the host's, reported by nothing else. + void state.opening + .then(async (late) => { await endCapture(leaseId, late, this.record); - }, - // Observed: this continuation outlives its budget, so a late failure of it must not reach the process - // as an unhandled rejection. The tunnel it could not end stays the host's, reported by nothing else. - () => undefined, - ); + }) + .catch(() => undefined); return this.settled.get(leaseId); } @@ -157,22 +157,25 @@ export class LimrunCaptureLifecycle { /** * Forgets the oldest released capture once every record this lifecycle can answer for passes the limit. A - * lease that is still held keeps its own record: what is still being captured is not the oldest thing to - * throw away. + * lease whose owner still holds a capture keeps its own record: the limit counts those captures, so the very + * same test spares them, and a record of any other lease is the oldest thing here to throw away. */ private forgetRecordsOverLimit(): void { - while (this.settled.size + this.heldCaptureCount() > SETTLED_NETWORK_CAPTURE_LIMIT) { - const held = new Set([...this.host.heldOwners()].map((owner) => owner.lease.leaseId)); + const held = this.heldCaptures(); + while (this.settled.size + held.size > SETTLED_NETWORK_CAPTURE_LIMIT) { const released = [...this.settled.keys()].find((leaseId) => !held.has(leaseId)); if (released === undefined) return; this.settled.delete(released); } } - private heldCaptureCount(): number { - let count = 0; - for (const owner of this.host.heldOwners()) count += owner.capture === undefined ? 0 : 1; - return count; + /** The leases whose owners still hold a capture: counted by the record limit and spared by it, one test both. */ + private heldCaptures(): Set { + const leaseIds = new Set(); + for (const owner of this.host.heldOwners()) { + if (owner.capture !== undefined) leaseIds.add(owner.lease.leaseId); + } + return leaseIds; } } diff --git a/packages/provider-limrun/src/dispose-observation.test.ts b/packages/provider-limrun/src/lease-release-observation.test.ts similarity index 91% rename from packages/provider-limrun/src/dispose-observation.test.ts rename to packages/provider-limrun/src/lease-release-observation.test.ts index b0920f7706..308b5c8f96 100644 --- a/packages/provider-limrun/src/dispose-observation.test.ts +++ b/packages/provider-limrun/src/lease-release-observation.test.ts @@ -82,6 +82,7 @@ vi.mock('@limrun/api/instance-client', () => ({ test('a local teardown that refuses beside a pending delete is reported, not loosed on the process', async () => { const rejections = watchUnhandledRejections(); state.deletes.length = 0; + const disconnectsBefore = state.disconnects; state.disconnectThrows = true; const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); const lease = androidLease('lease-throwing-disconnect'); @@ -91,7 +92,7 @@ test('a local teardown that refuses beside a pending delete is reported, not loo // The release holds at its delete while the session disconnect it started beside refuses. const released = runtime.leaseLifecycle.release?.(lease); await vi.waitFor(() => expect(state.deletes).toHaveLength(1)); - expect(state.disconnects).toBe(1); + expect(state.disconnects - disconnectsBefore).toBe(1); reportDelete = state.deletes[0]?.report; reportDelete?.(); @@ -102,7 +103,7 @@ test('a local teardown that refuses beside a pending delete is reported, not loo const warnings = warningsOf(record); expect(warnings.map((warning) => warning.code)).toEqual(['LIMRUN_TEARDOWN_INCOMPLETE']); expect(String(warnings[0]?.message)).toContain('Local teardown refused to finish'); - rejections.assertIsSilent(); + await rejections.assertIsSilent(); } finally { rejections.stop(); reportDelete?.(); @@ -117,6 +118,7 @@ test('a capture close that refuses still leaves the delete it ran beside answere const rejections = watchUnhandledRejections(); const harDir = mkdtempForTestSync('limrun-har-observe-'); state.deletes.length = 0; + const captureClosesBefore = state.captureCloses; state.captureCloseThrows = true; const runtime = createLimrunRuntime( { apiKey: 'lim_test_key', networkHarDir: harDir }, @@ -137,11 +139,11 @@ test('a capture close that refuses still leaves the delete it ran beside answere // join window: the instance the delete confirmed is gone, and the release answers for it. const record = (await released) as Record; expect(record.limrunInstanceId).toBe('android-instance-1'); - expect(state.captureCloses).toBe(1); + expect(state.captureCloses - captureClosesBefore).toBe(1); expect(warningsOf(record).map((warning) => warning.message)).toEqual([ 'Capture record refused to be written', ]); - rejections.assertIsSilent(); + await rejections.assertIsSilent(); } finally { reportDelete?.(); } @@ -161,9 +163,12 @@ function warningsOf(record: Record): Array void; + assertIsSilent: () => Promise; stop: () => void; } { const seen: unknown[] = []; @@ -172,7 +177,10 @@ function watchUnhandledRejections(): { }; process.on('unhandledRejection', watch); return { - assertIsSilent: () => { + assertIsSilent: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); if (seen.length === 0) return; assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`); }, diff --git a/packages/provider-limrun/src/lease-release.test.ts b/packages/provider-limrun/src/lease-release.test.ts index 99aa92c020..aabceea803 100644 --- a/packages/provider-limrun/src/lease-release.test.ts +++ b/packages/provider-limrun/src/lease-release.test.ts @@ -37,9 +37,11 @@ test('a create that answers late joins the delete its disposal already started', }); return { failure: undefined, cause: undefined }; }); + const freed: string[] = []; const host: LimrunDisposalHost = { deleteInstance: deleteHost, endCapture: async () => undefined, + ownerHoldsNothing: (owner) => freed.push(owner.lease.leaseId), }; let reportCreate!: () => void; const creation = new Promise((resolve) => { @@ -61,7 +63,7 @@ test('a create that answers late joins the delete its disposal already started', // The create answers now. The end it owes is this disposal's, and it joins the request Limrun already // has for this instance rather than sending a second one. reportCreate(); - await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + await vi.advanceTimersByTimeAsync(0); expect(deleteHost).toHaveBeenCalledOnce(); // The one request confirming is enough: the joined end records the instance as deleted. @@ -71,6 +73,9 @@ test('a create that answers late joins the delete its disposal already started', vi.useRealTimers(); } expect(owner.deletedInstanceId).toBe('android-instance-1'); + // The instance is gone and the host resources are down, so this owner's lease is given back: the route + // that ends this instance had already refused its caller, and that caller is not coming back to be told. + expect(freed).toEqual(['lease-release-join']); }); /** @@ -86,6 +91,7 @@ test("a refused delete reports its own error as the refusal's cause", async () = cause: refusal, }), endCapture: async () => undefined, + ownerHoldsNothing: () => {}, }; const owner = ownerWithRentedInstance( Promise.resolve({ status: 'failed', error: new Error('boot failed') }), @@ -156,6 +162,36 @@ test('recovery by labels confirms a gone instance and reports every id it ended' }); }); +test('recovery stops waiting for a delete whose response never finishes', async () => { + await import('./lifecycle-budget.ts'); + let reportDelete!: () => void; + const pending = new Promise((resolve) => { + reportDelete = resolve; + }); + const limrun = labeledInstancesClient(['android-instance-a'], async () => await pending); + let outcome: unknown; + vi.useFakeTimers(); + try { + const released = releaseLimrunLease(limrun, leaseForRecovery(), undefined).then( + (value) => { + outcome = value; + }, + (error: unknown) => { + outcome = error; + }, + ); + await vi.advanceTimersByTimeAsync(30_001); + expect(outcome).toBeInstanceOf(AppError); + expect((outcome as AppError).details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect((outcome as AppError).details?.hint).toContain('android-instance-a'); + reportDelete(); + await released; + } finally { + reportDelete(); + vi.useRealTimers(); + } +}); + test('recovery by labels treats a 404 as confirmed, the way every other delete route does', async () => { const limrun = labeledInstancesClient(['android-instance-a'], async () => { throw Object.assign(new Error('not found'), { status: 404 }); @@ -170,7 +206,7 @@ test('recovery by labels treats a 404 as confirmed, the way every other delete r */ test('recovery by labels refuses with every id when a delete is not confirmed', async () => { const limrun = labeledInstancesClient( - ['android-instance-a', 'android-instance-b'], + ['android-instance-a', 'android-instance-b', 'android-instance-c'], async (id) => { if (id === 'android-instance-b') throw new Error('Limrun refused to delete the instance'); return undefined; @@ -181,9 +217,14 @@ test('recovery by labels refuses with every id when a delete is not confirmed', ); expect(thrown.details?.reason).toBe('limrun_instance_delete_unconfirmed'); expect(thrown.details?.limrunInstanceId).toBe('android-instance-b'); - expect(thrown.details?.limrunInstanceIds).toEqual(['android-instance-a', 'android-instance-b']); + expect(thrown.details?.limrunInstanceIds).toEqual([ + 'android-instance-a', + 'android-instance-b', + 'android-instance-c', + ]); + expect(String(thrown.details?.hint)).toContain('android-instance-c'); expect(String(thrown.details?.hint)).toContain( - "Ask Limrun to delete instance 'android-instance-b'", + 'Ask Limrun to delete: android-instance-b, android-instance-c.', ); expect(String(thrown.details?.hint)).toContain('android-instance-a was confirmed deleted'); expect(String(thrown.cause)).toContain('refused'); @@ -226,6 +267,41 @@ test('a take-refusal for an unconfirmed delete does not call the instance ended' * that spends almost all of that handshake and still comes to in time is answered for without a warning, * because the join is longer than the work inside it. */ +/** + * A late route that could not give the instance back must keep the lease held: the next release, recovery or + * shutdown is the route that retries the delete, and only it can tell its caller the instance is gone. + */ +test('a late end that Limrun refused does not give the lease back', async () => { + await import('./lifecycle-budget.ts'); + const freed: string[] = []; + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ + failure: 'Limrun refused to delete the instance', + cause: undefined, + }), + endCapture: async () => undefined, + ownerHoldsNothing: (owner) => freed.push(owner.lease.leaseId), + }; + let reportCreate!: () => void; + const creation = new Promise((resolve) => { + reportCreate = () => resolve({ status: 'failed', error: new Error('instance boot failed') }); + }); + const owner = ownerWithRentedInstance(creation); + vi.useFakeTimers(); + try { + const disposal = disposeLimrunLeaseOwner(host, owner); + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS + DISPOSAL_SETTLE_WAIT_MS); + await expect(disposal).resolves.toMatchObject({ cleanup: 'uncertain' }); + // The create answers after that, and the end it owes is refused by Limrun as well: the lease stays held. + reportCreate(); + await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + } finally { + vi.useRealTimers(); + } + expect(owner.deletedInstanceId).toBeUndefined(); + expect(freed).toEqual([]); +}); + test('a capture end that spends its tunnel budget is still joined in time', async () => { await import('./lifecycle-budget.ts'); const host: LimrunDisposalHost = { @@ -237,6 +313,7 @@ test('a capture end that spends its tunnel budget is still joined in time', asyn }); return { phase: 'written', harPath: '/tmp/late.har', failures: { count: 0, truncated: 0 } }; }, + ownerHoldsNothing: () => {}, }; const owner = ownerWithRentedInstance( // A create that failed but named the instance (rentedInstanceId is set by the fixture): the disposal diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index 1c212ed32a..37d08280a0 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -63,6 +63,13 @@ export type LimrunDisposalHost = Readonly<{ deleteInstance(platform: LimrunPlatform, instanceId: string): Promise; /** Ends one lease's capture and answers with its record, for a runtime that has captured something. */ endCapture(owner: LimrunLeaseOwner): Promise; + /** + * Tells the runtime this owner stands for nothing any more, so its lease may be allocated again. Only a + * route that finished ending the instance and put down the host resources may say this, and only the + * runtime's own take answers for a lease that is over; this is for the route whose disposal had already + * refused its caller, and whose caller will never come back to be told the instance is gone. + */ + ownerHoldsNothing(owner: LimrunLeaseOwner): void; }>; /** @@ -97,7 +104,7 @@ export async function disposeLimrunLeaseOwner( // so no refusal of a resource this route already owns can reach the process as an unhandled rejection. const capture = keepLimrunObserved(host.endCapture(owner)); const local = keepLimrunObserved(releaseOwnerResources(owner)); - const instance = await endInstance(host, owner, session, instanceId); + const instance = await endInstance(host, owner, instanceId); const stuck = await awaitLocalTeardown(local); // The record is optional reporting joined last: a capture that refuses to be written cannot keep a release // from answering for the instance it ended, and is still listable once it does come to. @@ -167,7 +174,7 @@ async function disposeUnansweredCreate( } // An instance is already named, so this disposal ends the billing it can see and leaves the create's silence // as what keeps the lease held. - const instance = await endInstance(host, owner, owner.session, instanceId); + const instance = await endInstance(host, owner, instanceId); // Reported, not thrown: a local step that refuses is one this disposal can no longer fix, and the delete // it ran beside has already been answered for. const stuck = await awaitLocalTeardown(releaseOwnerResources(owner)); @@ -190,10 +197,18 @@ async function endWhatTheCreateNamed( owner: LimrunLeaseOwner, ): Promise { const instanceId = owner.rentedInstanceId; + let owed = false; if (instanceId !== undefined && owner.deletedInstanceId !== instanceId) { - await endInstance(host, owner, owner.session, instanceId); + const ended = await endInstance(host, owner, instanceId); + // Limrun has not given this instance back: the lease stays held for the next route, which is the one + // that will retry the delete and tell its caller what came of it. + owed = ended.cleanup === 'uncertain'; } - await releaseOwnerResources(owner); + const stuck = await awaitLocalTeardown(releaseOwnerResources(owner)); + // The instance is gone, or was never this runtime's to delete, and nothing local refuses: this owner + // stands for nothing, so the lease may be allocated again. No take will give it back the usual way — the + // disposal that took it had already refused its caller, and that caller is not coming back. + if (!owed && stuck === undefined) host.ownerHoldsNothing(owner); } /** @@ -236,18 +251,19 @@ type LimrunInstanceEnd = Readonly<{ /** * Ends one owner's instance within a deadline of its own. An attached instance is the caller's own and is never * deleted here; a delete Limrun already confirmed answers for itself; and a delete that never answers keeps the - * lease held and stays observed, because what it confirms late is what frees the lease. + * lease held and stays observed, because what it confirms late is what frees the lease. Ownership is read from + * the owner and not from an argument, so no route that reaches an instance can default an attached one to + * being ours to delete. */ async function endInstance( host: LimrunDisposalHost, owner: LimrunLeaseOwner, - session: LimrunRuntimeSession | undefined, instanceId: string, ): Promise { const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); if (owner.deletedInstanceId === instanceId) return { cleanup: 'confirmed', failure: undefined, cause: undefined }; - if ((session?.ownership ?? 'created') === 'attached') { + if ((owner.session?.ownership ?? 'created') === 'attached') { return { cleanup: 'attached', failure: undefined, cause: undefined }; } const requested = @@ -413,15 +429,28 @@ async function releaseLabeledInstances( ? await limrun.iosInstances.list({ labelSelector }) : await limrun.androidInstances.list({ labelSelector }); const instanceIds = instances.getPaginatedItems().map((instance) => instance.metadata.id); + const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); const confirmed: string[] = []; for (const instanceId of instanceIds) { - const deleted = await deleteLimrunInstance(limrun, platform, instanceId); + const answer = await awaitLimrunWithinBudget( + deleteLimrunInstance(limrun, platform, instanceId), + DELETE_ANSWER_BUDGET_MS, + ); + const deleted = answer.ok + ? answer.value + : { + failure: workOutstandingFailure( + `Delete of instance '${instanceId}'`, + DELETE_ANSWER_BUDGET_MS, + ), + cause: undefined, + }; if (deleted.failure !== undefined) { throwUnconfirmedDelete( `Recovery could not confirm instance '${instanceId}' of lease '${lease.leaseId}' deleted`, { limrunInstanceId: instanceId, limrunInstanceIds: instanceIds }, deleted.failure, - recoveryHint(instanceId, instanceIds, confirmed), + recoveryHint(instanceIds, confirmed), deleted.cause, ); } @@ -436,14 +465,12 @@ async function releaseLabeledInstances( } /** What the recovery refusal tells the operator to name to Limrun: the instance left, and those gone. */ -function recoveryHint( - instanceId: string, - instanceIds: readonly string[], - confirmed: string[], -): string { - if (confirmed.length === 0) return `Ask Limrun to delete: ${instanceIds.join(', ')}.`; +function recoveryHint(instanceIds: readonly string[], confirmed: readonly string[]): string { + const outstanding = instanceIds.filter((id) => !confirmed.includes(id)); + const request = `Ask Limrun to delete: ${outstanding.join(', ')}.`; + if (confirmed.length === 0) return request; const gone = `${confirmed.length > 1 ? 'Instances' : 'Instance'} ${confirmed.join(', ')} ${confirmed.length > 1 ? 'were' : 'was'} confirmed deleted.`; - return `Ask Limrun to delete instance '${instanceId}'. ${gone}`; + return `${request} ${gone}`; } /** diff --git a/packages/provider-limrun/src/runtime-dependencies.fixtures.ts b/packages/provider-limrun/src/runtime-dependencies.fixtures.ts index 54b71674b8..82323635f6 100644 --- a/packages/provider-limrun/src/runtime-dependencies.fixtures.ts +++ b/packages/provider-limrun/src/runtime-dependencies.fixtures.ts @@ -15,9 +15,11 @@ const NO_KEYBOARD = Object.freeze({ }); /** - * The host and platform adapters the runtime tests hand `createLimrunRuntime`: every one of them answers - * a fixed value, so a case that reaches one of them has left the path it means to drive. A case with a - * reading of its own spreads this object and replaces the member it drives. + * The host and platform adapters the runtime tests hand `createLimrunRuntime`: every one of them answers a + * fixed or a pass-through value, so a case that reaches one of them has left the path it means to drive. The + * pass-throughs are the ones that shape what they were handed into an invocation or an error, and they shape + * it the same way every time. A case with a reading of its own spreads this object and replaces the member it + * drives. */ export function stubLimrunRuntimeDependencies(): LimrunRuntimeDependencies { return { diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index 064299aa01..f21d0ac9d7 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -355,15 +355,22 @@ test('two allocations that ask for traffic logs open one destination tunnel', as try { await runtime.leaseLifecycle.allocate?.(lease); // Both repeat allocations ask for the capture while the first one's tunnel is still being opened. - const first = runtime.leaseLifecycle.allocate?.(lease, { - flags: { providerNetworkLogs: true }, - }); - const second = runtime.leaseLifecycle.allocate?.(lease, { - flags: { providerNetworkLogs: true }, - }); + const first = refusalOf( + runtime.leaseLifecycle.allocate?.(lease, { + flags: { providerNetworkLogs: true }, + }), + ); + const second = refusalOf( + runtime.leaseLifecycle.allocate?.(lease, { + flags: { providerNetworkLogs: true }, + }), + ); await waitForNewOpen(captures, () => state.captureStarts); + const pending = await runtime.cloudArtifacts?.listCloudArtifacts?.({ leaseId: lease.leaseId }); + assert.equal(pending?.status, 'pending'); + assert.equal(pending?.provider, 'limrun'); reportTunnel(); - await Promise.all([first, second]); + assert.deepEqual(await Promise.all([first, second]), [undefined, undefined]); // The second allocation joined the start in flight rather than opening a tunnel whose `already_active` // refusal would be reported over the capture that is working. @@ -798,14 +805,20 @@ test('a create that answers after its disposal gave up ends the instance it rent assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); assert.equal(state.disconnect.mock.calls.length - disconnects, 1); assert.ok((await refusedAllocation) instanceof AppError); + assert.equal(state.connectCount - connects, 1); + // The instance is gone and the session is down, so the lease is allocatable again: what the disposal + // that had already refused its caller owed is paid, and nothing is left owed to the next allocation. + state.createGate = undefined; + await runtime.leaseLifecycle.allocate?.(lease); + assert.equal(state.androidCreate.mock.calls.length - creates, 2); } finally { vi.useRealTimers(); reportCreate(); state.createGate = undefined; await runtime.shutdown(); } - assert.equal(state.androidCreate.mock.calls.length - creates, 1); - assert.equal(state.connectCount - connects, 1); + assert.equal(state.androidCreate.mock.calls.length - creates, 2); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1', 'android-instance-1']); }); test('a create that fails after its session was built disconnects that session', async () => { diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index 3e8d94ffbc..4b52d6a67b 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -604,6 +604,11 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { deleteInstance: async (platform, instanceId) => await this.deleteCreatedInstance(platform, instanceId), endCapture: async (owner) => await this.capturesInstance?.end(owner), + // The one route that may free a lease without a take is the disposal that had already refused its + // caller: what it ends leaves nothing for a later allocation of that lease to be owed. + ownerHoldsNothing: (owner) => { + this.closeOwner(owner); + }, }; /** One instance delete. `undefined` is Limrun's confirmation that the instance is gone. */ @@ -647,6 +652,9 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (leaseId === undefined) return undefined; const captures = this.capturesInstance; if (captures === undefined) return undefined; + if (this.leases.get(leaseId)?.capture?.phase === 'starting') { + return { provider: LIMRUN_PROVIDER, status: 'pending', cloudArtifacts: [] }; + } const { networkArtifacts } = await import('./network-har.ts'); const capture = await captures.list(leaseId, this.leases.get(leaseId)?.capture); return capture && networkArtifacts(capture); From 079711dc92a0a29d1d67bc88b523b6bfaa9fa008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 22:20:27 +0100 Subject: [PATCH 08/11] fix(limrun): clear reverse mappings before closing instance client --- .../src/capture-lifecycle.test.ts | 4 +- .../provider-limrun/src/capture-lifecycle.ts | 8 ++-- .../src/lease-release-observation.test.ts | 46 +++++++++++++++++-- packages/provider-limrun/src/lease-release.ts | 10 ++-- packages/provider-limrun/src/runtime.test.ts | 3 +- src/__tests__/limrun-runtime.test.ts | 7 ++- 6 files changed, 62 insertions(+), 16 deletions(-) diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts index e7f567d84d..da854328da 100644 --- a/packages/provider-limrun/src/capture-lifecycle.test.ts +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -66,6 +66,7 @@ test('a capture that answers late and then refuses is not loosed on the process' // import that had to resolve under fake timers would never answer at all. await import('./network-har.ts'); const rejections = watchUnhandledRejections(); + const close = vi.fn(); let reportOpened!: (capture: NetworkCapture) => void; const opening = new Promise((opened) => { reportOpened = opened; @@ -88,7 +89,7 @@ test('a capture that answers late and then refuses is not loosed on the process' failures: async () => { throw new Error('the destination tunnel refused to answer'); }, - close: () => {}, + close, }, }); await vi.advanceTimersByTimeAsync(1_000); @@ -96,6 +97,7 @@ test('a capture that answers late and then refuses is not loosed on the process' // rejection this continuation loosed can be seen at all. vi.useRealTimers(); await rejections.assertIsSilent(); + expect(close).toHaveBeenCalledOnce(); } finally { vi.useRealTimers(); rejections.stop(); diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts index 5b728e9359..fe1ae9c41a 100644 --- a/packages/provider-limrun/src/capture-lifecycle.ts +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -203,9 +203,11 @@ async function endCapture( record: (leaseId: string, capture: SettledNetworkCapture) => SettledNetworkCapture, ): Promise { const { writeCaptureRecord } = await import('./network-har.ts'); - const settled = await writeCaptureRecord(capture); - if (capture.phase === 'capturing') capture.capture.close(); - return record(leaseId, settled); + try { + return record(leaseId, await writeCaptureRecord(capture)); + } finally { + if (capture.phase === 'capturing') capture.capture.close(); + } } function hostsToCapture(hostList: unknown): string[] { diff --git a/packages/provider-limrun/src/lease-release-observation.test.ts b/packages/provider-limrun/src/lease-release-observation.test.ts index 308b5c8f96..cb6324ca83 100644 --- a/packages/provider-limrun/src/lease-release-observation.test.ts +++ b/packages/provider-limrun/src/lease-release-observation.test.ts @@ -64,9 +64,10 @@ vi.mock('@limrun/api/instance-client', () => ({ pressKey: vi.fn(), setText: vi.fn(), tap: vi.fn(), - startAdbTunnel: vi.fn(async () => { - throw new Error('no ADB tunnel is raised by these cases'); - }), + startAdbTunnel: vi.fn(async () => ({ + address: { address: '127.0.0.1', port: 62001 }, + close: () => {}, + })), startTunnel: vi.fn(async () => ({ tunnelId: 't1', selectors: [], @@ -114,6 +115,45 @@ test('a local teardown that refuses beside a pending delete is reported, not loo } }); +test('release clears reverse mappings before disconnect closes their ADB endpoint', async () => { + const disconnectsBefore = state.disconnects; + const removed = vi.fn(async () => {}); + const dependencies = testDependencies(); + dependencies.android.createPortReverse = async (adb) => ({ + ensure: async () => { + await adb(['reverse', 'tcp:18441', 'tcp:18441']); + }, + list: async () => { + if (state.disconnects !== disconnectsBefore) throw new Error('ADB endpoint is disconnected'); + return [{ local: 'tcp:18441', remote: 'tcp:18441' }]; + }, + remove: removed, + removeAllOwned: async () => {}, + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, dependencies); + const lease = androidLease('lease-reverse-before-disconnect'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + await runtime.configurePortReverse?.({ + leaseId: lease.leaseId, + provider: 'limrun', + devicePort: 18441, + hostPort: 18441, + name: 'live-regression', + }); + const released = runtime.leaseLifecycle.release?.(lease); + await vi.waitFor(() => expect(state.deletes).toHaveLength(1)); + state.deletes[0]?.report(); + expect(await released).toMatchObject({ limrunInstanceId: 'android-instance-1' }); + expect(removed).toHaveBeenCalledWith('tcp:18441'); + expect(state.disconnects - disconnectsBefore).toBe(1); + } finally { + state.deletes.forEach(({ report }) => report()); + state.deletes.length = 0; + await runtime.shutdown(); + } +}); + test('a capture close that refuses still leaves the delete it ran beside answered for', async () => { const rejections = watchUnhandledRejections(); const harDir = mkdtempForTestSync('limrun-har-observe-'); diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index 37d08280a0..4dfd17bbe6 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -338,10 +338,12 @@ async function releaseLocalResources(owner: LimrunLeaseOwner): Promise { // rather than passed over: the session it eventually builds has nowhere else to be disconnected. const session = owner.session ?? (await sessionTheCreateBuilt(owner)); if (session === undefined) return; - session.client.disconnect(); - // The tunnel's start is an `adb connect` the SDK runs without a deadline. Its cleanup retires the endpoint - // first, so a tunnel that answers late ends itself and no caller can reach it meanwhile. - if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); + try { + // Disconnect closes the SDK's ADB endpoint, so reverse cleanup must finish while that endpoint is live. + if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); + } finally { + session.client.disconnect(); + } } /** The session a create answered for, once it answers, for a release that got there first. */ diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index f21d0ac9d7..db963633cf 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -453,7 +453,7 @@ test('a release stops waiting on a stuck ADB tunnel, deletes the instance and sa await vi.advanceTimersByTimeAsync(0); // The delete does not queue behind the stuck tunnel: it was asked for while that tunnel was still open. assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); - assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + assert.equal(state.disconnect.mock.calls.length - disconnects, 0); // The release's own budget for the local teardown elapses; the delete it ran beside is already answered. await vi.advanceTimersByTimeAsync(LOCAL_TEARDOWN_WAIT_MS); @@ -468,6 +468,7 @@ test('a release stops waiting on a stuck ADB tunnel, deletes the instance and sa reportTunnel(); await vi.advanceTimersByTimeAsync(0); const refused = await refusedCommand; + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); assert.ok( refused instanceof AppError && refused.details?.reason === 'limrun_adb_tunnel_cleaned_up', `expected the retained command to be refused, got ${String(refused)}`, diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index c1a7569da1..ed9f85793f 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -256,10 +256,9 @@ test('Limrun reclaims a labeled iOS instance without an in-memory session', asyn assert.deepEqual(limrunMockState.iosList.mock.calls, [ [{ labelSelector: 'provider=limrun,leaseId=lease-recovered-ios' }], ]); - assert.deepEqual( - limrunMockState.iosDelete.mock.calls.map(([instanceId]) => instanceId), - ['ios-instance-recovered'], - ); + assert.deepEqual(limrunMockState.iosDelete.mock.calls, [ + ['ios-instance-recovered', { maxRetries: 0 }], + ]); } finally { await runtime.shutdown(); } From f051db5814e716239e08faf8c3e611f7ba5d859f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 22:35:40 +0100 Subject: [PATCH 09/11] fix(limrun): give client disconnect an independent deadline --- .../src/capture-lifecycle.test.ts | 27 +---------------- .../src/lease-release-observation.test.ts | 30 +------------------ packages/provider-limrun/src/lease-release.ts | 15 ++++++---- .../provider-limrun/src/lifecycle.fixtures.ts | 27 +++++++++++++++++ packages/provider-limrun/src/runtime.test.ts | 3 +- 5 files changed, 41 insertions(+), 61 deletions(-) create mode 100644 packages/provider-limrun/src/lifecycle.fixtures.ts diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts index da854328da..2672fd36a7 100644 --- a/packages/provider-limrun/src/capture-lifecycle.test.ts +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -4,6 +4,7 @@ import type { DeviceLease } from '@agent-device/contracts/device'; import { LimrunCaptureLifecycle, type LimrunCaptureHost } from './capture-lifecycle.ts'; import type { LimrunLeaseOwner } from './lease-ownership.ts'; import type { NetworkCapture } from './network-har.ts'; +import { watchUnhandledRejections } from './lifecycle.fixtures.ts'; /** Longer than the lifecycle's budget for a destination tunnel that is still being opened. */ const CAPTURE_OPEN_WAIT_MS = 25_000; @@ -142,32 +143,6 @@ function captureHost(held: readonly LimrunLeaseOwner[] = []): LimrunCaptureHost }; } -/** - * Collects the rejections the process would die on: the daemon's failure mode for one is to take every other - * session down with it. The check yields to a macrotask first, because Node reports a rejection only after the - * current microtask drain — asserting before that yield, or removing the listener without one, could pass a - * leaked rejection unseen. - */ -function watchUnhandledRejections(): { - assertIsSilent: () => Promise; - stop: () => void; -} { - const seen: unknown[] = []; - const watch = (reason: unknown): void => { - seen.push(reason); - }; - process.on('unhandledRejection', watch); - return { - assertIsSilent: async () => { - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - if (seen.length > 0) assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`); - }, - stop: () => process.off('unhandledRejection', watch), - }; -} - /** * An owner whose lease has a destination tunnel that has not answered, and never will. A case that drives a * different lease overrides the id it needs. diff --git a/packages/provider-limrun/src/lease-release-observation.test.ts b/packages/provider-limrun/src/lease-release-observation.test.ts index cb6324ca83..882490f692 100644 --- a/packages/provider-limrun/src/lease-release-observation.test.ts +++ b/packages/provider-limrun/src/lease-release-observation.test.ts @@ -1,10 +1,10 @@ -import assert from 'node:assert/strict'; import { test, vi, expect } from 'vitest'; import type { DeviceLease } from '@agent-device/contracts/device'; import { createLimrunRuntime } from './runtime.ts'; import { stubLimrunRuntimeDependencies } from './runtime-dependencies.fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; +import { watchUnhandledRejections } from './lifecycle.fixtures.ts'; /* * A disposal starts its capture end and its local teardown beside the instance delete and joins them only @@ -200,34 +200,6 @@ function warningsOf(record: Record): Array[]) : []; } -/** - * Collects the rejections the process would die on: the daemon's failure mode for one is to take every - * other session down with it, so a case whose disposal looses one fails here rather than in production. - * The check yields to a macrotask first: Node reports a rejection only after the current microtask drain, - * so asserting before that yield, or removing the listener without one, could pass a leaked rejection - * unseen. - */ -function watchUnhandledRejections(): { - assertIsSilent: () => Promise; - stop: () => void; -} { - const seen: unknown[] = []; - const watch = (reason: unknown): void => { - seen.push(reason); - }; - process.on('unhandledRejection', watch); - return { - assertIsSilent: async () => { - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - if (seen.length === 0) return; - assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`); - }, - stop: () => process.off('unhandledRejection', watch), - }; -} - function androidLease(leaseId: string): DeviceLease { return { leaseId, diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index 4dfd17bbe6..556c5f59fb 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -338,12 +338,17 @@ async function releaseLocalResources(owner: LimrunLeaseOwner): Promise { // rather than passed over: the session it eventually builds has nowhere else to be disconnected. const session = owner.session ?? (await sessionTheCreateBuilt(owner)); if (session === undefined) return; - try { - // Disconnect closes the SDK's ADB endpoint, so reverse cleanup must finish while that endpoint is live. - if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); - } finally { + const { awaitLimrunWithinBudget } = await import('./lifecycle-budget.ts'); + const tunnel = + session.platform === 'android' ? cleanupLimrunAndroidAdbTunnel(session) : Promise.resolve(); + // Disconnect closes the ADB endpoint: reverse cleanup gets its window, but cannot postpone disconnect + // past that window. Both promises are observed before either can settle. + const disconnect = awaitLimrunWithinBudget(tunnel, LOCAL_TEARDOWN_BUDGET_MS).then(() => { session.client.disconnect(); - } + }); + const results = await Promise.allSettled([tunnel, disconnect]); + const refused = results.find((result) => result.status === 'rejected'); + if (refused?.status === 'rejected') throw refused.reason; } /** The session a create answered for, once it answers, for a release that got there first. */ diff --git a/packages/provider-limrun/src/lifecycle.fixtures.ts b/packages/provider-limrun/src/lifecycle.fixtures.ts new file mode 100644 index 0000000000..5279bf5c49 --- /dev/null +++ b/packages/provider-limrun/src/lifecycle.fixtures.ts @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; + +/** + * Collects the rejections the process would die on: the daemon's failure mode for one is to take every other + * session down with it. The check yields to a macrotask first, because Node reports a rejection only after the + * current microtask drain — asserting before that yield, or removing the listener without one, could pass a + * leaked rejection unseen. + */ +export function watchUnhandledRejections(): { + assertIsSilent: () => Promise; + stop: () => void; +} { + const seen: unknown[] = []; + const watch = (reason: unknown): void => { + seen.push(reason); + }; + process.on('unhandledRejection', watch); + return { + assertIsSilent: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + if (seen.length > 0) assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`); + }, + stop: () => process.off('unhandledRejection', watch), + }; +} diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index db963633cf..021c06f341 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -430,6 +430,7 @@ test('a release that arrives while a tunnel is opening ends the tunnel that answ }); test('a release stops waiting on a stuck ADB tunnel, deletes the instance and says what it left', async () => { + await import('./lease-release.ts'); const { deletes, disconnects, adbTunnels, tunnelCloses } = limrunActivity(); let reportTunnel!: () => void; state.tunnelGate = new Promise((resolve) => { @@ -458,6 +459,7 @@ test('a release stops waiting on a stuck ADB tunnel, deletes the instance and sa // The release's own budget for the local teardown elapses; the delete it ran beside is already answered. await vi.advanceTimersByTimeAsync(LOCAL_TEARDOWN_WAIT_MS); const [warning] = warningsOf(await released); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); assert.equal(warning?.code, 'LIMRUN_TEARDOWN_INCOMPLETE'); // The report names the work this route stopped waiting for, and the instance it left a host endpoint on. assert.match(String(warning?.message), /Local teardown/); @@ -468,7 +470,6 @@ test('a release stops waiting on a stuck ADB tunnel, deletes the instance and sa reportTunnel(); await vi.advanceTimersByTimeAsync(0); const refused = await refusedCommand; - assert.equal(state.disconnect.mock.calls.length - disconnects, 1); assert.ok( refused instanceof AppError && refused.details?.reason === 'limrun_adb_tunnel_cleaned_up', `expected the retained command to be refused, got ${String(refused)}`, From 9043233d01a3b1bb28797124c615bb50ecc40176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 07:53:48 +0100 Subject: [PATCH 10/11] fix(limrun): free late leases through normal disposal --- .../provider-limrun/src/capture-lifecycle.ts | 13 +- .../provider-limrun/src/lease-ownership.ts | 2 + .../src/lease-release-observation.test.ts | 112 ++++++++++++++++-- .../provider-limrun/src/lease-release.test.ts | 71 ++++++++--- packages/provider-limrun/src/lease-release.ts | 57 ++------- .../provider-limrun/src/lifecycle-budget.ts | 4 +- packages/provider-limrun/src/network-har.ts | 17 ++- packages/provider-limrun/src/runtime.ts | 8 +- 8 files changed, 202 insertions(+), 82 deletions(-) diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts index fe1ae9c41a..e1c2e6b567 100644 --- a/packages/provider-limrun/src/capture-lifecycle.ts +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -203,11 +203,20 @@ async function endCapture( record: (leaseId: string, capture: SettledNetworkCapture) => SettledNetworkCapture, ): Promise { const { writeCaptureRecord } = await import('./network-har.ts'); + let settled: SettledNetworkCapture; + let closeFailure: string | undefined; try { - return record(leaseId, await writeCaptureRecord(capture)); + settled = await writeCaptureRecord(capture); } finally { - if (capture.phase === 'capturing') capture.capture.close(); + if (capture.phase === 'capturing') { + try { + capture.capture.close(); + } catch (error) { + closeFailure = errorMessage(error); + } + } } + return record(leaseId, closeFailure === undefined ? settled : { ...settled, closeFailure }); } function hostsToCapture(hostList: unknown): string[] { diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts index 07b7b99e04..86211e63da 100644 --- a/packages/provider-limrun/src/lease-ownership.ts +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -95,6 +95,8 @@ export type LimrunOwnerCapture = */ export type LimrunDisposalOutcome = Readonly<{ cleanup: 'confirmed' | 'attached' | 'uncertain' | 'none'; + /** Creation outlived this disposal's wait; the runtime re-takes the owner after creation settles. */ + creationPending?: true; /** The instance the lease held, known as soon as the create named it. */ instanceId: string | undefined; /** Why a disposal could not be confirmed; absent once the instance is gone. */ diff --git a/packages/provider-limrun/src/lease-release-observation.test.ts b/packages/provider-limrun/src/lease-release-observation.test.ts index 882490f692..cc4fb086c0 100644 --- a/packages/provider-limrun/src/lease-release-observation.test.ts +++ b/packages/provider-limrun/src/lease-release-observation.test.ts @@ -1,4 +1,5 @@ import { test, vi, expect } from 'vitest'; +import fs from 'node:fs'; import type { DeviceLease } from '@agent-device/contracts/device'; import { createLimrunRuntime } from './runtime.ts'; import { stubLimrunRuntimeDependencies } from './runtime-dependencies.fixtures.ts'; @@ -16,6 +17,9 @@ import { watchUnhandledRejections } from './lifecycle.fixtures.ts'; const state = vi.hoisted(() => ({ /** Instance deletes this runtime asked Limrun for; each waits for its case to answer it. */ deletes: [] as Array<{ instanceId: string; report: () => void }>, + creates: 0, + createGate: undefined as Promise | undefined, + autoDelete: false, disconnects: 0, /** Made to throw by the case whose session disconnect refuses while its delete is still owed. */ disconnectThrows: false, @@ -28,18 +32,23 @@ vi.mock('@limrun/api', () => ({ default: class MockLimrun { readonly iosInstances = { create: vi.fn(), list: vi.fn(), delete: vi.fn() }; readonly androidInstances = { - create: async () => ({ - metadata: { id: 'android-instance-1' }, - status: { - token: 'instance-token', - apiUrl: 'https://android.example', - adbWebSocketUrl: 'wss://adb.example', - }, - }), + create: async () => { + state.creates += 1; + await state.createGate; + return { + metadata: { id: 'android-instance-1' }, + status: { + token: 'instance-token', + apiUrl: 'https://android.example', + adbWebSocketUrl: 'wss://adb.example', + }, + }; + }, list: vi.fn(async () => ({ getPaginatedItems: () => [] })), // Every delete waits for its case to answer it, so a release can be held at its delete while the // local teardown it started beside has long since refused. delete: vi.fn(async (instanceId: string) => { + if (state.autoDelete) return; let report!: () => void; const answered = new Promise((answeredNow) => { report = answeredNow; @@ -103,7 +112,9 @@ test('a local teardown that refuses beside a pending delete is reported, not loo expect(record.limrunInstanceId).toBe('android-instance-1'); const warnings = warningsOf(record); expect(warnings.map((warning) => warning.code)).toEqual(['LIMRUN_TEARDOWN_INCOMPLETE']); - expect(String(warnings[0]?.message)).toContain('Local teardown refused to finish'); + expect(String(warnings[0]?.message)).toBe( + 'Local teardown refused to finish: instance client refused to disconnect', + ); await rejections.assertIsSilent(); } finally { rejections.stop(); @@ -181,8 +192,18 @@ test('a capture close that refuses still leaves the delete it ran beside answere expect(record.limrunInstanceId).toBe('android-instance-1'); expect(state.captureCloses - captureClosesBefore).toBe(1); expect(warningsOf(record).map((warning) => warning.message)).toEqual([ - 'Capture record refused to be written', + 'Capture close failed: capture tunnel refused to close', ]); + const harPath = `${harDir}/${lease.leaseId}.har`; + expect(record).toMatchObject({ + cloudArtifacts: { status: 'ready', cloudArtifacts: [{ url: harPath }] }, + }); + expect(fs.existsSync(harPath)).toBe(true); + const listed = await runtime.cloudArtifacts?.listCloudArtifacts?.({ leaseId: lease.leaseId }); + expect(listed).toMatchObject({ + status: 'ready', + message: expect.stringContaining('capture tunnel refused to close'), + }); await rejections.assertIsSilent(); } finally { reportDelete?.(); @@ -196,6 +217,77 @@ test('a capture close that refuses still leaves the delete it ran beside answere } }); +test('a late create with failed local cleanup is freed by the normal take with confirmed deletion', async () => { + await import('./lease-release.ts'); + const createsBefore = state.creates; + state.deletes.length = 0; + state.disconnectThrows = true; + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-late-failed-local'); + const allocating = Promise.resolve(runtime.leaseLifecycle.allocate?.(lease)).catch( + (error: unknown) => error, + ); + try { + await vi.waitFor(() => expect(state.creates - createsBefore).toBe(1)); + const expired = Promise.resolve(runtime.leaseLifecycle.release?.(lease)).catch( + (error: unknown) => error, + ); + vi.useFakeTimers(); + await vi.advanceTimersByTimeAsync(130_000); + expect(await expired).toMatchObject({ + details: { reason: 'limrun_instance_delete_unconfirmed' }, + }); + reportCreate(); + await vi.advanceTimersByTimeAsync(0); + expect(state.deletes).toHaveLength(1); + let answered = false; + const joining = Promise.resolve(runtime.leaseLifecycle.allocate?.(lease)) + .catch((error: unknown) => error) + .then((answer) => { + answered = true; + return answer; + }); + await vi.advanceTimersByTimeAsync(0); + expect(answered).toBe(false); + const release = runtime.leaseLifecycle.release?.(lease); + state.deletes[0]?.report(); + await vi.advanceTimersByTimeAsync(0); + const record = (await release) as Record; + expect(record.limrunInstanceId).toBe('android-instance-1'); + expect(warningsOf(record)).toMatchObject([ + { + code: 'LIMRUN_TEARDOWN_INCOMPLETE', + message: 'Local teardown refused to finish: instance client refused to disconnect', + }, + ]); + expect(await joining).toMatchObject({ + details: { + reason: 'limrun_lease_released_mid_allocation', + hint: expect.stringContaining('was ended by the release'), + }, + }); + await allocating; + state.disconnectThrows = false; + state.createGate = undefined; + await runtime.leaseLifecycle.allocate?.(lease); + expect(state.creates - createsBefore).toBe(2); + } finally { + reportCreate(); + state.createGate = undefined; + state.disconnectThrows = false; + state.autoDelete = true; + state.deletes.forEach(({ report }) => report()); + vi.useRealTimers(); + await runtime.shutdown(); + state.autoDelete = false; + state.deletes.length = 0; + } +}); + function warningsOf(record: Record): Array> { return Array.isArray(record.warnings) ? (record.warnings as Record[]) : []; } diff --git a/packages/provider-limrun/src/lease-release.test.ts b/packages/provider-limrun/src/lease-release.test.ts index aabceea803..00f2a90058 100644 --- a/packages/provider-limrun/src/lease-release.test.ts +++ b/packages/provider-limrun/src/lease-release.test.ts @@ -37,11 +37,9 @@ test('a create that answers late joins the delete its disposal already started', }); return { failure: undefined, cause: undefined }; }); - const freed: string[] = []; const host: LimrunDisposalHost = { deleteInstance: deleteHost, endCapture: async () => undefined, - ownerHoldsNothing: (owner) => freed.push(owner.lease.leaseId), }; let reportCreate!: () => void; const creation = new Promise((resolve) => { @@ -63,19 +61,18 @@ test('a create that answers late joins the delete its disposal already started', // The create answers now. The end it owes is this disposal's, and it joins the request Limrun already // has for this instance rather than sending a second one. reportCreate(); + const retaken = disposeLimrunLeaseOwner(host, owner); await vi.advanceTimersByTimeAsync(0); expect(deleteHost).toHaveBeenCalledOnce(); // The one request confirming is enough: the joined end records the instance as deleted. reportDelete(); await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + await expect(retaken).resolves.toMatchObject({ cleanup: 'confirmed' }); } finally { vi.useRealTimers(); } expect(owner.deletedInstanceId).toBe('android-instance-1'); - // The instance is gone and the host resources are down, so this owner's lease is given back: the route - // that ends this instance had already refused its caller, and that caller is not coming back to be told. - expect(freed).toEqual(['lease-release-join']); }); /** @@ -91,7 +88,6 @@ test("a refused delete reports its own error as the refusal's cause", async () = cause: refusal, }), endCapture: async () => undefined, - ownerHoldsNothing: () => {}, }; const owner = ownerWithRentedInstance( Promise.resolve({ status: 'failed', error: new Error('boot failed') }), @@ -262,25 +258,18 @@ test('a take-refusal for an unconfirmed delete does not call the instance ended' expect(uncertain.details?.retriable).toBe(true); }); -/** - * The disposal's join for a capture record contains the destination tunnel's own open budget: a capture end - * that spends almost all of that handshake and still comes to in time is answered for without a warning, - * because the join is longer than the work inside it. - */ /** * A late route that could not give the instance back must keep the lease held: the next release, recovery or * shutdown is the route that retries the delete, and only it can tell its caller the instance is gone. */ -test('a late end that Limrun refused does not give the lease back', async () => { +test('a late create whose delete Limrun refused keeps disposal uncertain', async () => { await import('./lifecycle-budget.ts'); - const freed: string[] = []; const host: LimrunDisposalHost = { deleteInstance: async () => ({ failure: 'Limrun refused to delete the instance', cause: undefined, }), endCapture: async () => undefined, - ownerHoldsNothing: (owner) => freed.push(owner.lease.leaseId), }; let reportCreate!: () => void; const creation = new Promise((resolve) => { @@ -294,14 +283,23 @@ test('a late end that Limrun refused does not give the lease back', async () => await expect(disposal).resolves.toMatchObject({ cleanup: 'uncertain' }); // The create answers after that, and the end it owes is refused by Limrun as well: the lease stays held. reportCreate(); + const retaken = disposeLimrunLeaseOwner(host, owner); await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + await expect(retaken).resolves.toMatchObject({ + cleanup: 'uncertain', + failure: 'Limrun refused to delete the instance', + }); } finally { vi.useRealTimers(); } expect(owner.deletedInstanceId).toBeUndefined(); - expect(freed).toEqual([]); }); +/** + * The disposal's join for a capture record contains the destination tunnel's own open budget: a capture end + * that spends almost all of that handshake and still comes to in time is answered for without a warning, + * because the join is longer than the work inside it. + */ test('a capture end that spends its tunnel budget is still joined in time', async () => { await import('./lifecycle-budget.ts'); const host: LimrunDisposalHost = { @@ -313,7 +311,6 @@ test('a capture end that spends its tunnel budget is still joined in time', asyn }); return { phase: 'written', harPath: '/tmp/late.har', failures: { count: 0, truncated: 0 } }; }, - ownerHoldsNothing: () => {}, }; const owner = ownerWithRentedInstance( // A create that failed but named the instance (rentedInstanceId is set by the fixture): the disposal @@ -338,6 +335,48 @@ test('a capture end that spends its tunnel budget is still joined in time', asyn expect(settled?.failure).toBeUndefined(); }); +test('a rejected capture report preserves its error without changing confirmed deletion', async () => { + const refusal = new Error('HAR inspection refused'); + const owner = ownerWithRentedInstance( + Promise.resolve({ status: 'failed', error: new Error('boot failed') }), + ); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ failure: undefined, cause: undefined }), + endCapture: async () => { + throw refusal; + }, + }; + const outcome = await disposeLimrunLeaseOwner(host, owner); + expect(outcome.cleanup).toBe('confirmed'); + expect(outcome.failure).toBe('Capture record refused to be written: HAR inspection refused'); +}); + +test('an unanswered create does not make a confirmed instance delete uncertain', async () => { + await import('./lifecycle-budget.ts'); + let reportCreate!: () => void; + const creation = new Promise((resolve) => { + reportCreate = () => resolve({ status: 'failed', error: new Error('boot failed') }); + }); + const owner = ownerWithRentedInstance(creation); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ failure: undefined, cause: undefined }), + endCapture: async () => undefined, + }; + vi.useFakeTimers(); + try { + const disposal = disposeLimrunLeaseOwner(host, owner); + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS + DISPOSAL_SETTLE_WAIT_MS); + await expect(disposal).resolves.toMatchObject({ + cleanup: 'confirmed', + instanceId: 'android-instance-1', + }); + } finally { + reportCreate(); + await vi.advanceTimersByTimeAsync(0); + vi.useRealTimers(); + } +}); + function captureReject(raise: () => Promise): Promise { return raise().then( () => { diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index 556c5f59fb..cc05500ddd 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -63,13 +63,6 @@ export type LimrunDisposalHost = Readonly<{ deleteInstance(platform: LimrunPlatform, instanceId: string): Promise; /** Ends one lease's capture and answers with its record, for a runtime that has captured something. */ endCapture(owner: LimrunLeaseOwner): Promise; - /** - * Tells the runtime this owner stands for nothing any more, so its lease may be allocated again. Only a - * route that finished ending the instance and put down the host resources may say this, and only the - * runtime's own take answers for a lease that is over; this is for the route whose disposal had already - * refused its caller, and whose caller will never come back to be told the instance is gone. - */ - ownerHoldsNothing(owner: LimrunLeaseOwner): void; }>; /** @@ -130,16 +123,20 @@ function captureRecordOutcome( joined: LimrunBudgetOutcome, outstanding: (work: string, budgetMs: number) => string, ): string | undefined { - if (joined.ok) return undefined; + if (joined.ok) { + return joined.value?.closeFailure === undefined + ? undefined + : `Capture close failed: ${joined.value.closeFailure}`; + } return 'rejected' in joined - ? 'Capture record refused to be written' + ? `Capture record refused to be written: ${errorMessage(joined.error)}` : outstanding('Capture record', CAPTURE_RECORD_BUDGET_MS); } /** - * Ends what a disposal holds when the create it waited for never answered. Nothing is known to be running, so - * the lease is kept held and refused rather than given back — and the create is watched, because whatever it - * eventually names belongs to this disposal and to no other route: a daemon close promises no next pass. + * Ends what a disposal can name after creation outlives its wait. The lease stays held until the runtime + * re-takes it after creation settles. A known instance is deleted now; the re-take joins that delete and the + * memoized local teardown before the runtime decides whether the lease is free. */ async function disposeUnansweredCreate( host: LimrunDisposalHost, @@ -154,18 +151,11 @@ async function disposeUnansweredCreate( `Instance create for lease '${owner.lease.leaseId}'`, CREATE_ANSWER_BUDGET_MS, ); - // Observed on both branches: this continuation outlives every budget, so a failure of it must not reach - // the process as an unhandled rejection. - owner.creation.then( - () => { - void endWhatTheCreateNamed(host, owner).catch(() => undefined); - }, - () => undefined, - ); const instanceId = owner.rentedInstanceId; if (instanceId === undefined) { return { cleanup: 'uncertain', + creationPending: true, instanceId: undefined, failure: outstanding, cause: undefined, @@ -180,6 +170,7 @@ async function disposeUnansweredCreate( const stuck = await awaitLocalTeardown(releaseOwnerResources(owner)); return { cleanup: instance.cleanup, + creationPending: true, instanceId, failure: joinOutcomeFailures(instance.failure, stuck ?? outstanding), cause: instance.cause, @@ -187,30 +178,6 @@ async function disposeUnansweredCreate( }; } -/** - * Ends what a create answered for after its disposal had refused its caller: the instance it rented, if that is - * still owed, and every host resource the session it built holds. Nothing else is left to do this — the lease - * was refused to its caller, and no later route is promised. - */ -async function endWhatTheCreateNamed( - host: LimrunDisposalHost, - owner: LimrunLeaseOwner, -): Promise { - const instanceId = owner.rentedInstanceId; - let owed = false; - if (instanceId !== undefined && owner.deletedInstanceId !== instanceId) { - const ended = await endInstance(host, owner, instanceId); - // Limrun has not given this instance back: the lease stays held for the next route, which is the one - // that will retry the delete and tell its caller what came of it. - owed = ended.cleanup === 'uncertain'; - } - const stuck = await awaitLocalTeardown(releaseOwnerResources(owner)); - // The instance is gone, or was never this runtime's to delete, and nothing local refuses: this owner - // stands for nothing, so the lease may be allocated again. No take will give it back the usual way — the - // disposal that took it had already refused its caller, and that caller is not coming back. - if (!owed && stuck === undefined) host.ownerHoldsNothing(owner); -} - /** * Ends what a disposal holds when its create answered and no instance is named for the lease: nothing this * runtime rented is running, so the lease can be given back. @@ -369,7 +336,7 @@ async function awaitLocalTeardown(local: Promise): Promise = | Readonly<{ ok: true; value: T }> | Readonly<{ ok: false; expired: true }> - | Readonly<{ ok: false; rejected: true }>; + | Readonly<{ ok: false; rejected: true; error: unknown }>; /** * Waits for `work` within `budgetMs`, unref'd so a daemon close is never held open by the timer alone. @@ -30,7 +30,7 @@ export async function awaitLimrunWithinBudget( ): Promise> { const settled = Promise.resolve(work).then( (value): LimrunBudgetOutcome => ({ ok: true, value }), - (): LimrunBudgetOutcome => ({ ok: false, rejected: true }), + (error: unknown): LimrunBudgetOutcome => ({ ok: false, rejected: true, error }), ); let timer: ReturnType | undefined; try { diff --git a/packages/provider-limrun/src/network-har.ts b/packages/provider-limrun/src/network-har.ts index dbe4d9b5a6..d470025aef 100644 --- a/packages/provider-limrun/src/network-har.ts +++ b/packages/provider-limrun/src/network-har.ts @@ -186,13 +186,15 @@ function defaultPort(url: URL | null): number | undefined { } /** What one lease's capture came to once it was written or refused. */ -export type SettledNetworkCapture = +export type SettledNetworkCapture = ( | Readonly<{ phase: 'written'; harPath: string; failures: LimrunNetworkFailures }> | Readonly<{ phase: 'unavailable'; reason: 'capture-dir-unset' | 'tunnel-in-use' | 'tunnel-start-failed' | 'har-write-failed'; message?: string; - }>; + }> +) & + Readonly<{ closeFailure?: string }>; /** What one lease has in capture terms: what it is capturing now, or what its capture came to. */ export type NetworkCapture = @@ -228,6 +230,17 @@ const NETWORK_ARTIFACT_NAME = 'Network traffic (HAR on the daemon host)'; /** What `artifacts` answers for one lease's capture record. */ export function networkArtifacts(capture: SettledNetworkCapture): CloudArtifactsResult { + const artifacts = captureRecordArtifacts(capture); + if (capture.closeFailure === undefined) return artifacts; + return { + ...artifacts, + message: [artifacts.message, `Capture close failed: ${capture.closeFailure}`] + .filter((message) => message !== undefined) + .join('; '), + }; +} + +function captureRecordArtifacts(capture: SettledNetworkCapture): CloudArtifactsResult { if (capture.phase === 'unavailable') { return { provider: LIMRUN_PROVIDER, diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index 4b52d6a67b..096157529b 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -573,6 +573,9 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { // Whatever the create made is now ended, or was never made, so this owner holds nothing and the // lease is free again. An unconfirmed delete keeps the owner so its outcome can be retried. if (outcome.cleanup !== 'uncertain') this.closeOwner(owner); + else if (outcome.creationPending) { + void owner.creation.then(() => this.take(owner, by)).catch(() => undefined); + } } reportDisposal(outcome); }; @@ -604,11 +607,6 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { deleteInstance: async (platform, instanceId) => await this.deleteCreatedInstance(platform, instanceId), endCapture: async (owner) => await this.capturesInstance?.end(owner), - // The one route that may free a lease without a take is the disposal that had already refused its - // caller: what it ends leaves nothing for a later allocation of that lease to be owed. - ownerHoldsNothing: (owner) => { - this.closeOwner(owner); - }, }; /** One instance delete. `undefined` is Limrun's confirmation that the instance is gone. */ From acd4c1026c6a623afaef3f50fb90be618a48879b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 09:18:50 +0100 Subject: [PATCH 11/11] fix(limrun): retain capture close failures beside write errors --- .../src/capture-lifecycle.test.ts | 31 +++++ .../provider-limrun/src/capture-lifecycle.ts | 26 +++-- .../provider-limrun/src/lease-ownership.ts | 5 +- .../src/lease-release-observation.test.ts | 110 ++++++++++++++---- packages/provider-limrun/src/lease-release.ts | 10 +- 5 files changed, 142 insertions(+), 40 deletions(-) diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts index 2672fd36a7..7710144cf5 100644 --- a/packages/provider-limrun/src/capture-lifecycle.test.ts +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -105,6 +105,37 @@ test('a capture that answers late and then refuses is not loosed on the process' } }); +test('a refused capture record carries the close failure beside the write error', async () => { + const rejections = watchUnhandledRejections(); + const writeError = new Error('HAR inspection refused'); + const close = vi.fn(() => { + throw new Error('capture tunnel refused to close'); + }); + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + owner.capture = { + phase: 'capturing', + harPath: '/tmp/sm4-refused-capture.har', + capture: { + entries: () => [], + failures: async () => { + throw writeError; + }, + close, + }, + }; + try { + await expect(lifecycle.end(owner)).rejects.toMatchObject({ + message: 'HAR inspection refused; Capture close failed: capture tunnel refused to close', + cause: writeError, + }); + expect(close).toHaveBeenCalledOnce(); + await rejections.assertIsSilent(); + } finally { + rejections.stop(); + } +}); + /** * The record limit counts a lease whose owner still holds a capture, so the very same test spares that * record and no other. A lease the runtime still holds for an unconfirmed delete, whose capture is already diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts index e1c2e6b567..99ab1a3252 100644 --- a/packages/provider-limrun/src/capture-lifecycle.ts +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -203,19 +203,25 @@ async function endCapture( record: (leaseId: string, capture: SettledNetworkCapture) => SettledNetworkCapture, ): Promise { const { writeCaptureRecord } = await import('./network-har.ts'); - let settled: SettledNetworkCapture; + const written = await writeCaptureRecord(capture).then( + (value) => ({ ok: true, value }) as const, + (error: unknown) => ({ ok: false, error }) as const, + ); let closeFailure: string | undefined; - try { - settled = await writeCaptureRecord(capture); - } finally { - if (capture.phase === 'capturing') { - try { - capture.capture.close(); - } catch (error) { - closeFailure = errorMessage(error); - } + if (capture.phase === 'capturing') { + try { + capture.capture.close(); + } catch (error) { + closeFailure = errorMessage(error); } } + if (!written.ok) { + if (closeFailure === undefined) throw written.error; + throw new Error(`${errorMessage(written.error)}; Capture close failed: ${closeFailure}`, { + cause: written.error, + }); + } + const settled = written.value; return record(leaseId, closeFailure === undefined ? settled : { ...settled, closeFailure }); } diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts index 86211e63da..699e4aaf8d 100644 --- a/packages/provider-limrun/src/lease-ownership.ts +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -95,7 +95,10 @@ export type LimrunOwnerCapture = */ export type LimrunDisposalOutcome = Readonly<{ cleanup: 'confirmed' | 'attached' | 'uncertain' | 'none'; - /** Creation outlived this disposal's wait; the runtime re-takes the owner after creation settles. */ + /** + * Creation outlived this disposal's wait. An uncertain outcome re-takes the owner after creation settles; + * a confirmed delete already ended the rented instance, and its local teardown awaits the late session. + */ creationPending?: true; /** The instance the lease held, known as soon as the create named it. */ instanceId: string | undefined; diff --git a/packages/provider-limrun/src/lease-release-observation.test.ts b/packages/provider-limrun/src/lease-release-observation.test.ts index cc4fb086c0..f1beb07e1c 100644 --- a/packages/provider-limrun/src/lease-release-observation.test.ts +++ b/packages/provider-limrun/src/lease-release-observation.test.ts @@ -18,6 +18,8 @@ const state = vi.hoisted(() => ({ /** Instance deletes this runtime asked Limrun for; each waits for its case to answer it. */ deletes: [] as Array<{ instanceId: string; report: () => void }>, creates: 0, + clientStarts: 0, + clientGate: undefined as Promise | undefined, createGate: undefined as Promise | undefined, autoDelete: false, disconnects: 0, @@ -62,31 +64,35 @@ vi.mock('@limrun/api', () => ({ })); vi.mock('@limrun/api/instance-client', () => ({ - createInstanceClient: vi.fn(async () => ({ - disconnect: () => { - state.disconnects += 1; - if (state.disconnectThrows) throw new Error('instance client refused to disconnect'); - }, - keepAlive: () => {}, - openUrl: vi.fn(), - sendAsset: vi.fn(), - pressKey: vi.fn(), - setText: vi.fn(), - tap: vi.fn(), - startAdbTunnel: vi.fn(async () => ({ - address: { address: '127.0.0.1', port: 62001 }, - close: () => {}, - })), - startTunnel: vi.fn(async () => ({ - tunnelId: 't1', - selectors: [], - close: () => { - state.captureCloses += 1; - if (state.captureCloseThrows) throw new Error('capture tunnel refused to close'); + createInstanceClient: vi.fn(async () => { + state.clientStarts += 1; + await state.clientGate; + return { + disconnect: () => { + state.disconnects += 1; + if (state.disconnectThrows) throw new Error('instance client refused to disconnect'); }, - })), - getTunnelStatus: vi.fn(async () => ({})), - })), + keepAlive: () => {}, + openUrl: vi.fn(), + sendAsset: vi.fn(), + pressKey: vi.fn(), + setText: vi.fn(), + tap: vi.fn(), + startAdbTunnel: vi.fn(async () => ({ + address: { address: '127.0.0.1', port: 62001 }, + close: () => {}, + })), + startTunnel: vi.fn(async () => ({ + tunnelId: 't1', + selectors: [], + close: () => { + state.captureCloses += 1; + if (state.captureCloseThrows) throw new Error('capture tunnel refused to close'); + }, + })), + getTunnelStatus: vi.fn(async () => ({})), + }; + }), })); test('a local teardown that refuses beside a pending delete is reported, not loosed on the process', async () => { @@ -219,6 +225,7 @@ test('a capture close that refuses still leaves the delete it ran beside answere test('a late create with failed local cleanup is freed by the normal take with confirmed deletion', async () => { await import('./lease-release.ts'); + const rejections = watchUnhandledRejections(); const createsBefore = state.creates; state.deletes.length = 0; state.disconnectThrows = true; @@ -275,6 +282,8 @@ test('a late create with failed local cleanup is freed by the normal take with c state.createGate = undefined; await runtime.leaseLifecycle.allocate?.(lease); expect(state.creates - createsBefore).toBe(2); + vi.useRealTimers(); + await rejections.assertIsSilent(); } finally { reportCreate(); state.createGate = undefined; @@ -285,6 +294,59 @@ test('a late create with failed local cleanup is freed by the normal take with c await runtime.shutdown(); state.autoDelete = false; state.deletes.length = 0; + rejections.stop(); + } +}); + +test('a confirmed delete during creation still disconnects the late session without another rent', async () => { + await import('./lease-release.ts'); + const rejections = watchUnhandledRejections(); + const createsBefore = state.creates; + const clientsBefore = state.clientStarts; + const disconnectsBefore = state.disconnects; + state.deletes.length = 0; + let reportClient!: () => void; + state.clientGate = new Promise((resolve) => { + reportClient = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-confirmed-before-session'); + const allocating = Promise.resolve(runtime.leaseLifecycle.allocate?.(lease)).catch( + (error: unknown) => error, + ); + try { + // Reaching the instance client means the one POST has already named the rented instance. + await vi.waitFor(() => expect(state.clientStarts - clientsBefore).toBe(1)); + const released = runtime.leaseLifecycle.release?.(lease); + vi.useFakeTimers(); + await vi.advanceTimersByTimeAsync(125_000); + expect(state.deletes).toHaveLength(1); + state.deletes[0]?.report(); + await vi.advanceTimersByTimeAsync(50_000); + expect(await released).toMatchObject({ + limrunInstanceId: 'android-instance-1', + warnings: [{ code: 'LIMRUN_TEARDOWN_INCOMPLETE' }], + }); + expect(state.disconnects - disconnectsBefore).toBe(0); + // The owner has been closed, but its observed, memoized local release still owns this session. + reportClient(); + await vi.advanceTimersByTimeAsync(0); + expect(await allocating).toMatchObject({ + details: { reason: 'limrun_lease_released_mid_allocation' }, + }); + expect(state.creates - createsBefore).toBe(1); + expect(state.deletes).toHaveLength(1); + expect(state.disconnects - disconnectsBefore).toBe(1); + vi.useRealTimers(); + await rejections.assertIsSilent(); + } finally { + reportClient(); + state.clientGate = undefined; + state.deletes.forEach(({ report }) => report()); + vi.useRealTimers(); + await runtime.shutdown(); + state.deletes.length = 0; + rejections.stop(); } }); diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts index cc05500ddd..bb621d5468 100644 --- a/packages/provider-limrun/src/lease-release.ts +++ b/packages/provider-limrun/src/lease-release.ts @@ -134,9 +134,9 @@ function captureRecordOutcome( } /** - * Ends what a disposal can name after creation outlives its wait. The lease stays held until the runtime - * re-takes it after creation settles. A known instance is deleted now; the re-take joins that delete and the - * memoized local teardown before the runtime decides whether the lease is free. + * Ends what a disposal can name after creation outlives its wait. An uncertain outcome keeps the lease for + * a re-take after creation settles. A confirmed delete frees it: that create already named its only rented + * instance, and the memoized local teardown still awaits any session the create goes on to build. */ async function disposeUnansweredCreate( host: LimrunDisposalHost, @@ -162,8 +162,8 @@ async function disposeUnansweredCreate( capture: await capture, }; } - // An instance is already named, so this disposal ends the billing it can see and leaves the create's silence - // as what keeps the lease held. + // The create has named its only rented instance, so confirming this delete ends its billing even while + // session construction is still pending. const instance = await endInstance(host, owner, instanceId); // Reported, not thrown: a local step that refuses is one this disposal can no longer fix, and the delete // it ran beside has already been answered for.