diff --git a/packages/provider-limrun/src/android.test.ts b/packages/provider-limrun/src/android.test.ts index 08983d7ab3..85163a9971 100644 --- a/packages/provider-limrun/src/android.test.ts +++ b/packages/provider-limrun/src/android.test.ts @@ -1,15 +1,143 @@ -import { expect, test, vi } from 'vitest'; +import { beforeEach, expect, test, vi } from 'vitest'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import type { DeviceInfo } from '@agent-device/kernel/device'; import { androidAdbHostTarget, androidAdbInvocation, + androidAdbSerialTarget, serializeAndroidAdbInvocation, type AndroidAdbInvocation, } from '@agent-device/platform-android/mechanics'; -import { cleanupLimrunAndroidAdbTunnel, type LimrunAndroidSession } from './android.ts'; -import type { LimrunAdbCommandOptions, LimrunAdbProvider } from './runtime-dependencies.ts'; +import { + cleanupLimrunAndroidAdbTunnel, + createLimrunAndroidSession, + type LimrunAndroidSession, +} from './android.ts'; +import type { + LimrunAdbCommandOptions, + LimrunAdbProvider, + LimrunRuntimeDependencies, +} from './runtime-dependencies.ts'; const ok = { exitCode: 0, stdout: '', stderr: '' }; +/** + * The serial a reverse cleanup is ending is not a route a retained caller may take: an ordinary command + * arriving inside that window has to be refused, or the taken session keeps taking commands on the device. + */ +test('a command that arrives while cleanup holds its serial is refused', async () => { + const runs: string[][] = []; + const session = await createLimrunAndroidSession( + ANDROID_SESSION_OPTIONS, + androidDependencies(runs), + ); + // The first command raises the tunnel this session owns, so a later one has a serial to reach for. + await session.adbProvider.exec(['shell', 'echo', 'ready']); + const dispatched = runs.length; + let reportList!: () => void; + reverse.listGate = new Promise((resolve) => { + reportList = resolve; + }); + try { + // Cleanup has retired this session and is inside the device with the serial it is ending. + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + await vi.waitFor(() => expect(reverse.list).toHaveBeenCalledOnce()); + + await expect(session.adbProvider.exec(['shell', 'echo', 'again'])).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + + reportList(); + await cleanup; + } finally { + reverse.listGate = undefined; + } + // The only commands this session dispatched are its own cleanup's: the listing and the disconnect. + expect(runs.slice(dispatched).filter((argv) => argv.includes('shell'))).toEqual([]); + expect(tunnels.start).toHaveBeenCalledOnce(); +}); + +/** + * The serial a cleanup is ending is reached only by the cleanup's own dispatches. A reverse command that + * arrives while that cleanup is mid-request goes back through the session's closed public route, so nothing + * gets mapped at a device that is being disconnected, behind the cleanup's one listing. + */ +test('a reverse command that arrives during cleanup is refused, not dispatched at the ending serial', async () => { + const runs: string[][] = []; + let reportList!: () => void; + reverse.listGate = new Promise((resolve) => { + reportList = resolve; + }); + const deps = androidDependencies(runs); + // The route the reverse manager dispatches through, which is what production hands it. + let reverseRoute!: (args: readonly string[]) => Promise; + deps.android.createPortReverse = async (adb) => { + reverseRoute = adb; + return reverse; + }; + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, deps); + // The first command raises the tunnel this session owns, giving cleanup a serial to end. + await session.adbProvider.exec(['shell', 'echo', 'ready']); + const dispatched = runs.length; + try { + // Cleanup holds its listing open: its teardown executor is published at the serial it is ending. + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + await vi.waitFor(() => expect(reverse.list).toHaveBeenCalledOnce()); + + await expect(reverseRoute(['reverse', 'tcp:9001', 'tcp:8001'])).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + + reportList(); + await cleanup; + } finally { + reverse.listGate = undefined; + } + // Nothing mapped an endpoint behind the cleanup's back: no reverse command was dispatched at all, because + // the listing and the removals are stubbed and the caller's command never reached the device. + expect(runs.slice(dispatched).filter((argv) => argv.includes('reverse'))).toEqual([]); +}); + +const tunnels = vi.hoisted(() => ({ + /** + * Held by a case that has to keep a tunnel in flight, which is the window a session's cleanup has to + * cover: the endpoint the SDK is still raising can answer after that cleanup answered. + */ + gate: undefined as Promise | undefined, + start: vi.fn(async () => { + await tunnels.gate; + return { address: { address: '127.0.0.1', port: 62_001 }, close: tunnels.close }; + }), + close: vi.fn(), +})); + +/** The reverse manager every session of a case gets, with its listing held by the case that needs a window. */ +const reverse = vi.hoisted(() => ({ + /** Held by a case that has to keep a cleanup inside the window in which it holds its serial. */ + listGate: undefined as Promise | undefined, + list: vi.fn(async () => { + await reverse.listGate; + return []; + }), + ensure: vi.fn(async () => {}), + remove: vi.fn(async () => {}), + removeAllOwned: vi.fn(async () => {}), +})); + +vi.mock('@limrun/api/instance-client', () => ({ + createInstanceClient: vi.fn(async () => ({ startAdbTunnel: tunnels.start })), +})); + +beforeEach(() => { + tunnels.gate = undefined; + tunnels.start.mockClear(); + tunnels.close.mockClear(); + reverse.listGate = undefined; + reverse.list.mockClear(); +}); + test('cleanup asks the platform to address the disconnect as a server-level command and hands it over unchanged', async () => { const calls: Array<{ argv: string[]; options: LimrunAdbCommandOptions | undefined }> = []; const close = vi.fn(); @@ -48,3 +176,123 @@ test('cleanup asks the platform to address the disconnect as a server-level comm expect(session.adbTunnel).toBeUndefined(); expect(session.adbTunnelPromise).toBeUndefined(); }); + +/** + * Cleanup yields before it can look at the session's endpoint, so it has to retire that endpoint in the step + * that started it: a command arriving inside that window would otherwise raise a tunnel the cleanup had no way + * to see, which then answers into a session nothing owns any more. + */ +test('a command that arrives while cleanup is starting raises no tunnel', async () => { + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, androidDependencies()); + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + // The cleanup has not looked at the endpoint yet, and this command has no retired flag to stop it. + const command = session.adbProvider.exec(['shell', 'echo', 'again']); + + await expect(command).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + await cleanup; + + expect(tunnels.start).not.toHaveBeenCalled(); + expect(session.adbSerial).toBeUndefined(); + expect(session.adbTunnel).toBeUndefined(); +}); + +/** + * A tunnel already being raised when cleanup arrives answers after cleanup returned, and the endpoint is + * process-wide: the session has to end it where it lands rather than publish it to nobody. + */ +test('a tunnel still being raised when cleanup arrives is ended and its command refused', async () => { + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, androidDependencies()); + let reportTunnel!: () => void; + tunnels.gate = new Promise((resolve) => { + reportTunnel = resolve; + }); + try { + const command = session.adbProvider.exec(['shell', 'echo', 'held']); + await vi.waitFor(() => expect(tunnels.start).toHaveBeenCalledOnce()); + + const cleanup = cleanupLimrunAndroidAdbTunnel(session); + reportTunnel(); + + await expect(command).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + await cleanup; + } finally { + tunnels.gate = undefined; + } + // The tunnel this session raised was closed, and nothing published it for a later command to use. + expect(tunnels.close).toHaveBeenCalledOnce(); + expect(session.adbSerial).toBeUndefined(); + expect(session.adbTunnel).toBeUndefined(); +}); + +/** + * The host ADB endpoint is process-wide, so an interactor kept from a cleaned-up session has to be refused + * rather than handed a fresh tunnel: nothing owns that tunnel any more, and it survives the daemon's work + * on the device. + */ +test('an interactor kept from a cleaned-up session is refused instead of raising a second tunnel', async () => { + const session = await createLimrunAndroidSession(ANDROID_SESSION_OPTIONS, androidDependencies()); + // The first command raises the tunnel this session owns. + await session.adbProvider.exec(['shell', 'echo', 'ready']); + expect(tunnels.start).toHaveBeenCalledOnce(); + + await cleanupLimrunAndroidAdbTunnel(session); + + await expect(session.adbProvider.exec(['shell', 'echo', 'again'])).rejects.toMatchObject({ + code: 'SESSION_NOT_FOUND', + details: { reason: 'limrun_adb_tunnel_cleaned_up' }, + }); + expect(tunnels.start).toHaveBeenCalledOnce(); +}); + +const ANDROID_SESSION_OPTIONS = { + lease: { + leaseId: 'lease-android', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease, + instanceId: 'android-instance-1', + ownership: 'created' as const, + device: { + platform: 'android', + id: 'limrun:android:lease-android', + name: 'Android 14', + kind: 'emulator', + target: 'mobile', + booted: true, + } satisfies DeviceInfo, + apiUrl: 'https://android.example', + adbUrl: 'wss://adb.example', + token: 'instance-token', +}; + +function androidDependencies( + runs?: string[][], +): Pick { + return { + android: { + deviceAdbInvocation: (serial: string, command: readonly string[]) => + androidAdbInvocation(androidAdbSerialTarget(serial), command), + hostAdbInvocation: (command: readonly string[]) => + androidAdbInvocation(androidAdbHostTarget(), command), + adbError: async (message: string) => new Error(message), + createPortReverse: async () => reverse, + } as unknown as LimrunRuntimeDependencies['android'], + host: { + runAdb: async (invocation: AndroidAdbInvocation) => { + runs?.push([...serializeAndroidAdbInvocation(invocation)]); + return ok; + }, + } as unknown as LimrunRuntimeDependencies['host'], + }; +} diff --git a/packages/provider-limrun/src/android.ts b/packages/provider-limrun/src/android.ts index a30c243b1b..e532f5e512 100644 --- a/packages/provider-limrun/src/android.ts +++ b/packages/provider-limrun/src/android.ts @@ -1,4 +1,5 @@ import path from 'node:path'; +import { AsyncLocalStorage } from 'node:async_hooks'; import type Limrun from '@limrun/api'; import { createInstanceClient as createAndroidInstanceClient, @@ -16,6 +17,7 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import type { LimrunAdbCommandOptions, LimrunAdbCommandResult, + LimrunAdbExecutor, LimrunAdbProvider, LimrunPortReverseEndpoint, LimrunRuntimeDependencies, @@ -42,6 +44,18 @@ type LimrunAndroidAdbSession = { adbTunnel?: LimrunAdbTunnel; adbSerial?: string; adbTunnelPromise?: Promise; + /** + * Set once teardown has begun for this session, before it yields. The host ADB endpoint is + * process-wide, so neither a caller that kept an interactor from this session nor a tunnel this session had + * already started may leave one no owner can tear down any more. + */ + adbTunnelRetired?: boolean; + /** + * Where teardown points this session's ADB route while it ends an endpoint: at that endpoint, whatever the + * session's own public route says about it. Only this session's cleanup sets it, and only for as long as it + * is dispatching, so no caller that kept an interactor can reach a serial that is on its way out. + */ + adbReverseExecutor?: LimrunAdbExecutor; readonly dependencies: Pick; }; @@ -85,9 +99,13 @@ export async function createLimrunAndroidSession( await createLimrunAndroidTextInjector(client)(request); }, }; - adbProvider.reverse = await dependencies.android.createPortReverse(adbProvider.exec, { - noRebind: options.ownership === 'attached', - }); + // The reverse manager has to outlive this session's own ADB route: its cleanup clears reverse mappings off a + // device the session is disconnecting, after retirement has closed that route to every caller. So it dispatches + // through a route of its own, which is the session's route until cleanup takes it over. + adbProvider.reverse = await dependencies.android.createPortReverse( + async (args, options) => await runLimrunAndroidReverseAdb(session, args, options), + { noRebind: options.ownership === 'attached' }, + ); return Object.assign(session, { adbProvider }); } @@ -150,21 +168,52 @@ export async function configureLimrunAndroidPortReverse( } export async function cleanupLimrunAndroidAdbTunnel(session: LimrunAndroidSession): Promise { - await session.adbTunnelPromise?.catch(() => {}); - const serial = session.adbSerial; - if (serial) { - await cleanupAndroidPortReverse(session); - await session.dependencies.host - .runAdb(session.dependencies.android.hostAdbInvocation(['disconnect', serial]), { - allowFailure: true, - timeoutMs: 10_000, - }) - .catch(() => {}); + // Retire first, before yielding: the host ADB endpoint is process-wide, so from this step on no retained + // interactor can raise a tunnel, and a tunnel already in flight ends where it lands. + session.adbTunnelRetired = true; + // Cleanup keeps the serial it finds, or the one the in-flight start publishes while this step waits on it: + // a start answering into a retired session records its endpoint here first, because this cleanup is the only + // route left that can reach it. + const started = await session.adbTunnelPromise?.catch(() => undefined); + session.adbTunnelPromise = undefined; + const serial = session.adbSerial ?? started; + try { + if (serial) { + // Clearing this session's reverse mappings means reaching the device this cleanup is about to disconnect, + // and this session's public ADB route is closed from the step above. So its reverse manager is pointed at + // an executor of this cleanup's own, addressed at the serial being ended — a serial no retained caller can + // ask for. + session.adbReverseExecutor = async (args, options) => + await runLimrunAndroidAdbAtSerial(session, serial, args, options); + // The teardown's marker covers this route and every dispatch this cleanup starts from it, including a + // reverse manager that answers a later request of its own inside the same await chain. A caller command + // that reaches this route through any other path carries no marker. + await limrunAdbTeardownScope.run(true, async () => { + await cleanupAndroidPortReverse(session); + }); + await session.dependencies.host + .runAdb(session.dependencies.android.hostAdbInvocation(['disconnect', serial]), { + allowFailure: true, + timeoutMs: 10_000, + }) + .catch(() => {}); + } + } finally { + session.adbReverseExecutor = undefined; + closeLimrunAndroidAdbTunnel(session); } - session.adbTunnel?.close(); +} + +/** + * Ends the tunnel this session holds, once. A tunnel that answered only after its session was retired + * lands here, because retirement alone cannot reach it; nothing else is left to end it. + */ +function closeLimrunAndroidAdbTunnel(session: LimrunAndroidSession): void { + const tunnel = session.adbTunnel; + if (tunnel === undefined) return; session.adbTunnel = undefined; session.adbSerial = undefined; - session.adbTunnelPromise = undefined; + tunnel.close(); } async function cleanupAndroidPortReverse(session: LimrunAndroidSession): Promise { @@ -189,25 +238,51 @@ async function runLimrunAndroidAdb( args: readonly string[], options?: LimrunAdbCommandOptions, ): Promise { - const { invocation, result } = await executeLimrunAndroidAdb(session, args, options); - return await requireSuccessfulLimrunAndroidAdb( - invocation, - result, - options?.allowFailure, - session.dependencies, - ); + const serial = await ensurePersistentAndroidAdbSerial(session); + return await runLimrunAndroidAdbAtSerial(session, serial, args, options); } /** - * The tunnel serial is this provider's own addressing decision: it goes on the invocation's - * target, so the command array the Android cluster handed over reaches the host unchanged. + * Whether one dispatch belongs to a session's teardown: true only for the dispatches that teardown itself + * started. A provider that hands a caller command to the reverse route — a reverse command queued inside an + * operation of its own — carries no marker, so the manager cannot have a caller's mapping bound at a serial + * that is on its way out. A marker set by one session never authorises another session's route. + * + * Requirement this rests on: the reverse manager must dispatch its caller's commands within the caller's + * async context. A manager that answered from work it scheduled outside this call (an internal queue drained + * by a timer, or a driver loop made at construction) would run the teardown's removals without this marker, + * and the mappings would survive the session they were made for. */ -async function executeLimrunAndroidAdb( +const limrunAdbTeardownScope = new AsyncLocalStorage(); + +/** + * The reverse manager's own route to the device. It is the session's route while the session is live, and + * cleanup's route of its own once the session is retired: clearing reverse mappings means reaching the device + * that is about to be disconnected, and that has to stay possible after every caller of this session was + * refused. Only a dispatch that carries its session's teardown scope may use the teardown's executor, so no + * caller command is ever dispatched at the serial the teardown is ending. + */ +async function runLimrunAndroidReverseAdb( session: LimrunAndroidAdbSession, args: readonly string[], options?: LimrunAdbCommandOptions, -): Promise<{ invocation: AndroidAdbInvocation; result: LimrunAdbCommandResult }> { - const serial = await ensurePersistentAndroidAdbSerial(session); +): Promise { + const atSerial = limrunAdbTeardownScope.getStore() ? session.adbReverseExecutor : undefined; + if (atSerial !== undefined) return await atSerial(args, options); + return await runLimrunAndroidAdb(session, args, options); +} + +/** + * Dispatches one command at a serial the caller already holds, which is where both of this provider's ADB routes + * end up. The tunnel serial is this provider's own addressing decision: it goes on the invocation's target, so + * the command array the Android cluster handed over reaches the host unchanged. + */ +async function runLimrunAndroidAdbAtSerial( + session: LimrunAndroidAdbSession, + serial: string, + args: readonly string[], + options?: LimrunAdbCommandOptions, +): Promise { const invocation = session.dependencies.android.deviceAdbInvocation(serial, args); const result = await session.dependencies.host.runAdb(invocation, { allowFailure: options?.allowFailure, @@ -216,7 +291,12 @@ async function executeLimrunAndroidAdb( timeoutMs: options?.timeoutMs ?? 30_000, signal: options?.signal, }); - return { invocation, result }; + return await requireSuccessfulLimrunAndroidAdb( + invocation, + result, + options?.allowFailure, + session.dependencies, + ); } async function requireSuccessfulLimrunAndroidAdb( @@ -236,6 +316,11 @@ async function requireSuccessfulLimrunAndroidAdb( } async function ensurePersistentAndroidAdbSerial(session: LimrunAndroidAdbSession): Promise { + // Retirement is checked before any serial is read, including one this session cached: a caller that kept an + // interactor from a session whose cleanup answered is refused the same as one that arrives first. The serial + // of a session being torn down is that teardown's alone, and its reverse manager reaches it through the + // executor it was handed, not through this route. + if (session.adbTunnelRetired) throw alreadyCleanedUp(session); if (session.adbSerial) return session.adbSerial; const pending = session.adbTunnelPromise ?? startAndroidAdbTunnel(session); session.adbTunnelPromise = pending; @@ -250,11 +335,29 @@ async function ensurePersistentAndroidAdbSerial(session: LimrunAndroidAdbSession async function startAndroidAdbTunnel(session: LimrunAndroidAdbSession): Promise { const tunnel = await session.client.startAdbTunnel(); const serial = `${tunnel.address.address}:${tunnel.address.port}`; + // Nothing between this call's start and the publication below lets a second tunnel be raised, so a session + // retired while this tunnel was being raised records it all the same: its cleanup is the one route that can + // still reach it, and it is awaiting this promise. Refusal alone would leave the endpoint on the host with + // nobody left to disconnect it. session.adbTunnel = tunnel; session.adbSerial = serial; + if (session.adbTunnelRetired) throw alreadyCleanedUp(session); return serial; } +function alreadyCleanedUp(session: LimrunAndroidAdbSession): AppError { + return new AppError( + 'SESSION_NOT_FOUND', + `Limrun ADB tunnel for instance '${session.instanceId}' was already cleaned up`, + { + reason: 'limrun_adb_tunnel_cleaned_up', + // The tunnel is gone whether the instance's delete confirmed or is still held for retry, so the hint + // sends the caller to a new lease, and to Limrun only for the state where a release would refuse again. + hint: `This interactor's ADB tunnel is closed for lease '${session.lease.leaseId}'. Open a new lease. If this lease's release was refused as an unconfirmed delete, ask Limrun to delete the instance first.`, + }, + ); +} + function tcpEndpoint(port: number): LimrunPortReverseEndpoint { if (!Number.isInteger(port) || port < 1 || port > 65_535) { throw new AppError('INVALID_ARGS', `Invalid Android tcp reverse port: ${port}`); diff --git a/packages/provider-limrun/src/capture-lifecycle.test.ts b/packages/provider-limrun/src/capture-lifecycle.test.ts new file mode 100644 index 0000000000..7710144cf5 --- /dev/null +++ b/packages/provider-limrun/src/capture-lifecycle.test.ts @@ -0,0 +1,204 @@ +import { test, vi, expect } from 'vitest'; +import assert from 'node:assert/strict'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { LimrunCaptureLifecycle, type LimrunCaptureHost } from './capture-lifecycle.ts'; +import type { LimrunLeaseOwner } from './lease-ownership.ts'; +import type { NetworkCapture } from './network-har.ts'; +import { watchUnhandledRejections } from './lifecycle.fixtures.ts'; + +/** Longer than the lifecycle's budget for a destination tunnel that is still being opened. */ +const CAPTURE_OPEN_WAIT_MS = 25_000; + +/** + * A repeat allocation joins the start its owner is already running, within the budget a read or a teardown + * gives that same handshake. + */ +test('a repeat allocation stops waiting on a destination tunnel that never answers', async () => { + vi.useFakeTimers(); + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + let joined = false; + const joining = lifecycle.start(owner, undefined).then(() => { + joined = true; + }); + try { + // Below the budget the join is still waiting: only its expiry ends it, not an early return. + await vi.advanceTimersByTimeAsync(19_000); + await Promise.resolve(); + assert.ok(!joined, 'expected the join to keep waiting inside its budget'); + await vi.advanceTimersByTimeAsync(CAPTURE_OPEN_WAIT_MS); + await joining; + } finally { + vi.useRealTimers(); + } + + assert.ok(joined, 'expected the joining allocation to stop waiting on the stalled tunnel'); + // What the stalled start goes on to open stays the owner's to publish, and its teardown's to end: the + // joiner leaves the capture where the allocation that started it put it. + assert.equal(owner.capture?.phase, 'starting'); +}); + +/** + * An `artifacts` read for a capture whose destination tunnel is still being opened answers on the spot: + * the read has nothing to wait for — the tunnel belongs to the lease's release, and a stalled handshake + * would otherwise hold the request for the whole open budget to say nothing new. + */ +test('an artifacts read answers at once for a capture whose tunnel is still opening', async () => { + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + const listed = await Promise.race([ + lifecycle.list(owner.lease.leaseId, owner.capture), + new Promise<'stalled'>((resolve) => { + const timer = setTimeout(() => resolve('stalled'), 250); + timer.unref?.(); + }), + ] as const); + assert.equal(listed, undefined); +}); + +/** + * A destination tunnel that answers after the release stopped waiting is written and ended all the same. When + * that late end refuses — a capture whose traffic will not be read, a tunnel that will not close — the refusal + * is observed where it lands: the continuation outlives every budget and every caller, so nothing above it is + * left to catch it. + */ +test('a capture that answers late and then refuses is not loosed on the process', async () => { + // Loaded before the clock is replaced: the late end writes its HAR through this module, and a dynamic + // import that had to resolve under fake timers would never answer at all. + await import('./network-har.ts'); + const rejections = watchUnhandledRejections(); + const close = vi.fn(); + let reportOpened!: (capture: NetworkCapture) => void; + const opening = new Promise((opened) => { + reportOpened = opened; + }); + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + owner.capture = { phase: 'starting', opening }; + vi.useFakeTimers(); + try { + const ended = lifecycle.end(owner); + // The release gives up on the handshake and answers for its lease; the tunnel is still its own to end. + await vi.advanceTimersByTimeAsync(CAPTURE_OPEN_WAIT_MS); + expect(await ended).toBeUndefined(); + // The tunnel arrives late and refuses to be read: the end of it fails with nobody left above it. + reportOpened({ + phase: 'capturing', + harPath: '/tmp/sm4-late-capture.har', + capture: { + entries: () => [], + failures: async () => { + throw new Error('the destination tunnel refused to answer'); + }, + close, + }, + }); + await vi.advanceTimersByTimeAsync(1_000); + // Asserted while the listener is still attached, and after a yield: that is the only window in which a + // rejection this continuation loosed can be seen at all. + vi.useRealTimers(); + await rejections.assertIsSilent(); + expect(close).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + rejections.stop(); + } +}); + +test('a refused capture record carries the close failure beside the write error', async () => { + const rejections = watchUnhandledRejections(); + const writeError = new Error('HAR inspection refused'); + const close = vi.fn(() => { + throw new Error('capture tunnel refused to close'); + }); + const lifecycle = new LimrunCaptureLifecycle(captureHost()); + const owner = ownerWithCapture(); + owner.capture = { + phase: 'capturing', + harPath: '/tmp/sm4-refused-capture.har', + capture: { + entries: () => [], + failures: async () => { + throw writeError; + }, + close, + }, + }; + try { + await expect(lifecycle.end(owner)).rejects.toMatchObject({ + message: 'HAR inspection refused; Capture close failed: capture tunnel refused to close', + cause: writeError, + }); + expect(close).toHaveBeenCalledOnce(); + await rejections.assertIsSilent(); + } finally { + rejections.stop(); + } +}); + +/** + * The record limit counts a lease whose owner still holds a capture, so the very same test spares that + * record and no other. A lease the runtime still holds for an unconfirmed delete, whose capture is already + * ended, keeps nothing the limit should spare: its record competes for the limit like any released one, and + * the map cannot grow past it. + */ +test("a released lease's record competes for the limit even while its lease is still held", async () => { + // The runtime still holds this lease; its owner no longer holds a capture. + const stillHeld = ownerWithCapture('lease-held-no-capture'); + stillHeld.capture = undefined; + const lifecycle = new LimrunCaptureLifecycle(captureHost([stillHeld])); + await lifecycle.end( + Object.assign(ownerWithCapture(stillHeld.lease.leaseId), { + capture: { phase: 'unavailable', reason: 'capture-dir-unset' }, + }), + ); + // Past the limit, with the held lease's record the oldest of them. + for (let index = 0; index < 33; index += 1) { + const released = ownerWithCapture(`lease-released-${String(index)}`); + released.capture = { phase: 'unavailable', reason: 'capture-dir-unset' }; + await lifecycle.end(released); + } + + // The newest record still answers, and the record of the lease that holds no capture is the one that went. + expect(await lifecycle.list('lease-released-32', undefined)).toMatchObject({ + reason: 'capture-dir-unset', + }); + expect(await lifecycle.list(stillHeld.lease.leaseId, undefined)).toBeUndefined(); +}); + +function captureHost(held: readonly LimrunLeaseOwner[] = []): LimrunCaptureHost { + return { + harDir: undefined, + liveSessionOf: () => undefined, + heldOwners: () => held, + }; +} + +/** + * An owner whose lease has a destination tunnel that has not answered, and never will. A case that drives a + * different lease overrides the id it needs. + */ +function ownerWithCapture(leaseId = 'lease-capture-join'): LimrunLeaseOwner { + const lease = { + leaseId, + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease; + const opening = new Promise(() => {}); + return { + lease, + platform: 'android', + creation: Promise.resolve({ status: 'ready', session: undefined, appAlias: undefined }), + state: { phase: 'live' }, + rentedInstanceId: undefined, + session: undefined, + capture: { phase: 'starting', opening }, + appAlias: undefined, + deletedInstanceId: undefined, + } as unknown as LimrunLeaseOwner; +} diff --git a/packages/provider-limrun/src/capture-lifecycle.ts b/packages/provider-limrun/src/capture-lifecycle.ts new file mode 100644 index 0000000000..99ab1a3252 --- /dev/null +++ b/packages/provider-limrun/src/capture-lifecycle.ts @@ -0,0 +1,235 @@ +import type { + LimrunLeaseOwner, + LimrunOwnerCapture, + LimrunRuntimeSession, +} from './lease-ownership.ts'; +import type { NetworkCapture, SettledNetworkCapture } from './network-har.ts'; +import { awaitLimrunWithinBudget } from './lifecycle-budget.ts'; +import { errorMessage } from '@agent-device/kernel/errors'; + +/** + * How long a read or a teardown waits for a destination tunnel that is still being opened. The SDK gives this + * handshake fifteen seconds of its own; the extra five seconds is this runtime's margin over it. Exported + * because the disposal that joins a capture end must give it more than this: `end` waits the handshake inside + * its own call, and a join shorter than it would report a stalled-but-in-budget handshake as a refused record. + */ +export const CAPTURE_OPEN_BUDGET_MS = 20_000; + +/** Released captures stay listable; the oldest settled records are forgotten past this count. */ +const SETTLED_NETWORK_CAPTURE_LIMIT = 32; + +export type LimrunCaptureHost = Readonly<{ + /** Where each lease's HAR goes. Without it a capture is unavailable and nothing is opened. */ + harDir: string | undefined; + /** The live session of an owner, and nothing once a teardown has taken the lease. */ + liveSessionOf(owner: LimrunLeaseOwner): LimrunRuntimeSession | undefined; + /** The leases this runtime holds, which is what the record limit spares from forgetting. */ + heldOwners(): Iterable; +}>; + +/** + * What one lease's network capture goes through: starting it, ending it, and answering what `artifacts` can + * say about it. The owner is where a capture lives, including a start whose destination tunnel is still being + * opened, so the allocation that started one, a release that took the lease over, and an artifacts read all + * look at the same thing and none of them can end up owning a tunnel nobody can close. + */ +export class LimrunCaptureLifecycle { + /** The capture records this lifecycle settled, which outlive their lease so `artifacts` can still name them. */ + private readonly settled = new Map(); + private readonly host: LimrunCaptureHost; + + constructor(host: LimrunCaptureHost) { + this.host = host; + } + + /** + * Starts this lease's capture, or joins the start its owner is already running. The owner records that start + * before anything is awaited, so a repeat allocation cannot open a second destination tunnel whose + * `already_active` refusal would be reported over the capture that is working, and a teardown that takes the + * lease meanwhile finds a tunnel that is not there yet and ends it where it lands. + */ + async start(owner: LimrunLeaseOwner, hostList: unknown): Promise { + const held = owner.capture; + if (held?.phase === 'starting') { + // Joining another allocation's destination tunnel is waited for within the budget a read or a teardown + // gives that same handshake, and its outcome changes nothing here: the start keeps its own owner to + // publish to, and this lease's teardown is the route that ends whatever tunnel it opens. So a handshake + // that stops answering costs this caller its wait, and nothing else. + await awaitLimrunWithinBudget(held.opening, CAPTURE_OPEN_BUDGET_MS); + return; + } + const harDir = this.host.harDir; + if (!harDir) { + owner.capture = { phase: 'unavailable', reason: 'capture-dir-unset' }; + return; + } + // Only a live lease can capture: a release or a shutdown that took this owner is not a caller a new + // destination tunnel could be handed to, and nothing here would own it afterwards. + const session = this.host.liveSessionOf(owner); + if (!session) return; + let reportOpened!: (capture: NetworkCapture) => void; + const opening = new Promise((opened) => { + reportOpened = opened; + }); + owner.capture = { phase: 'starting', opening }; + const hosts = hostsToCapture(hostList); + // Started after the publication above, and nothing between the two lets a second start through. A start + // that could not be attempted at all is reported the same way, so nothing waits on a tunnel that is never + // coming, and the outcome is published before the waiting allocation is answered. + const publish = (capture: NetworkCapture): void => { + // A teardown took the capture over, and what this tunnel came to is that teardown's to end: nothing + // here hands a destination tunnel to a lease that is on its way out. + const current = owner.capture; + if (current !== undefined && current.phase === 'starting' && current.opening === opening) { + owner.capture = capture; + } + reportOpened(capture); + }; + void openCapture(session.client, harDir, owner.lease.leaseId, hosts).then( + publish, + (error: unknown) => + publish({ + phase: 'unavailable', + reason: 'tunnel-start-failed', + message: errorMessage(error), + }), + ); + await opening; + } + + /** + * Takes this owner's capture off it and ends it, keeping the record it came to so `artifacts` can still + * answer for the HAR after the lease is gone. Whatever its phase, the capture leaves the owner here: this is + * the disposal route, and the one route that owns a tunnel whose open had not answered yet. + */ + async end(owner: LimrunLeaseOwner): Promise { + const leaseId = owner.lease.leaseId; + const state = owner.capture; + if (state === undefined) return this.settled.get(leaseId); + owner.capture = undefined; + if (state.phase !== 'starting') return await endCapture(leaseId, state, this.record); + // A destination tunnel still being opened is waited for within a budget. A tunnel that answers later is + // written and ended by the continuation below all the same, so it is never left without an owner. + const opened = await awaitLimrunWithinBudget(state.opening, CAPTURE_OPEN_BUDGET_MS); + if (opened.ok) return await endCapture(leaseId, opened.value, this.record); + // Observed end to end: this continuation outlives its budget, so neither a late tunnel that refuses to be + // written nor one whose close refuses may reach the process as an unhandled rejection. The tunnel it could + // not end stays the host's, reported by nothing else. + void state.opening + .then(async (late) => { + await endCapture(leaseId, late, this.record); + }) + .catch(() => undefined); + return this.settled.get(leaseId); + } + + /** + * What `artifacts` answers for one lease: its capture written now while the lease is held, and the record a + * release settled after the lease is gone. A lease this lifecycle never held answers nothing. + */ + async list( + leaseId: string, + held: LimrunOwnerCapture | undefined, + ): Promise { + if (held === undefined) return this.settled.get(leaseId); + if (held.phase !== 'starting') { + const { writeCaptureRecord } = await import('./network-har.ts'); + return await writeCaptureRecord(held); + } + // A capture whose tunnel is still being opened has no traffic to report yet, and this read is not the + // route that owns that tunnel: its release is. Waiting the open budget here would hold an `artifacts` + // request for a handshake whose outcome this read cannot act on, so the read answers now with whatever + // record this lease already has and lets the release write the rest. + return this.settled.get(leaseId); + } + + /** Keeps one released capture's record, forgetting the oldest once the record limit is passed. */ + private readonly record = ( + leaseId: string, + capture: SettledNetworkCapture, + ): SettledNetworkCapture => { + // Re-insertion keeps the records in settle order, so the oldest released one is the first to go. + this.settled.delete(leaseId); + this.settled.set(leaseId, capture); + this.forgetRecordsOverLimit(); + return capture; + }; + + /** + * Forgets the oldest released capture once every record this lifecycle can answer for passes the limit. A + * lease whose owner still holds a capture keeps its own record: the limit counts those captures, so the very + * same test spares them, and a record of any other lease is the oldest thing here to throw away. + */ + private forgetRecordsOverLimit(): void { + const held = this.heldCaptures(); + while (this.settled.size + held.size > SETTLED_NETWORK_CAPTURE_LIMIT) { + const released = [...this.settled.keys()].find((leaseId) => !held.has(leaseId)); + if (released === undefined) return; + this.settled.delete(released); + } + } + + /** The leases whose owners still hold a capture: counted by the record limit and spared by it, one test both. */ + private heldCaptures(): Set { + const leaseIds = new Set(); + for (const owner of this.host.heldOwners()) { + if (owner.capture !== undefined) leaseIds.add(owner.lease.leaseId); + } + return leaseIds; + } +} + +/** Opens one capture's destination tunnel, or says why it could not. */ +async function openCapture( + client: LimrunRuntimeSession['client'], + harDir: string, + leaseId: string, + hosts: readonly string[], +): Promise { + const { limrunHarPath, startLimrunHarCapture } = await import('./network-har.ts'); + const result = await startLimrunHarCapture(client, hosts.length > 0 ? hosts : undefined); + return result.ok + ? { phase: 'capturing', harPath: limrunHarPath(harDir, leaseId), capture: result.capture } + : { phase: 'unavailable', reason: result.reason, message: result.message }; +} + +/** + * Ends one capture: writes its HAR, closes its tunnel, and keeps the record listable. The record is written + * before the tunnel is closed, because its failure counts are read from the capture that is about to go. + */ +async function endCapture( + leaseId: string, + capture: NetworkCapture, + record: (leaseId: string, capture: SettledNetworkCapture) => SettledNetworkCapture, +): Promise { + const { writeCaptureRecord } = await import('./network-har.ts'); + const written = await writeCaptureRecord(capture).then( + (value) => ({ ok: true, value }) as const, + (error: unknown) => ({ ok: false, error }) as const, + ); + let closeFailure: string | undefined; + if (capture.phase === 'capturing') { + try { + capture.capture.close(); + } catch (error) { + closeFailure = errorMessage(error); + } + } + if (!written.ok) { + if (closeFailure === undefined) throw written.error; + throw new Error(`${errorMessage(written.error)}; Capture close failed: ${closeFailure}`, { + cause: written.error, + }); + } + const settled = written.value; + return record(leaseId, closeFailure === undefined ? settled : { ...settled, closeFailure }); +} + +function hostsToCapture(hostList: unknown): string[] { + return typeof hostList === 'string' + ? hostList + .split(',') + .map((host) => host.trim()) + .filter(Boolean) + : []; +} diff --git a/packages/provider-limrun/src/client-options.ts b/packages/provider-limrun/src/client-options.ts index b424486e1b..7f0997f335 100644 --- a/packages/provider-limrun/src/client-options.ts +++ b/packages/provider-limrun/src/client-options.ts @@ -1,5 +1,6 @@ import type Limrun from '@limrun/api'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import type { LimrunDeleteResult } from './lease-ownership.ts'; export const LIMRUN_CLIENT_ID = 'agent-device-cli'; @@ -8,6 +9,49 @@ export function limrunUserAgent(clientVersion: string): string { return `${LIMRUN_CLIENT_ID}/${clientVersion}`; } +/** + * Asks the SDK for one attempt at a request whose failure this runtime cannot tell apart from Limrun having + * done the work: the SDK retries a lost answer by POSTing again, and a repeated instance create rents a + * second instance. The SDK defaults to two retries, so a request that must not repeat carries this. + * + * The option is the SDK's request-scoped `maxRetries` (`RequestOptions`, @limrun/api 0.64.1), which every + * resource method takes as its second argument. `session-allocation.test.ts` pins that this is what the + * create honours: it drives the real SDK against a local server that drops the answer and asserts one POST. + */ +export const LIMRUN_SINGLE_ATTEMPT = Object.freeze({ maxRetries: 0 }); + +/** + * Ends one instance, answering with no failure once Limrun has it gone and with a failure string and its error + * otherwise. Like the create, the delete is asked for in one attempt: a lost answer is ambiguous, and this + * runtime's retries happen at its own budgets, not the SDK's. A 404 is Limrun answering that the instance is + * already gone, which is exactly what the delete was asked to achieve, so it confirms rather than refuses. + */ +export async function deleteLimrunInstance( + limrun: Limrun, + platform: 'ios' | 'android', + instanceId: string, +): Promise { + try { + if (platform === 'ios') { + await limrun.iosInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); + } else { + await limrun.androidInstances.delete(instanceId, LIMRUN_SINGLE_ATTEMPT); + } + return { failure: undefined, cause: undefined }; + } catch (error) { + // A 404 names the outcome this request was after, however it came about. + if (readHttpStatus(error) === 404) return { failure: undefined, cause: undefined }; + return { failure: errorMessage(error), cause: error }; + } +} + +/** The HTTP status of an SDK failure, for a route that can answer for one particular refusal. */ +function readHttpStatus(error: unknown): number | undefined { + if (!error || typeof error !== 'object') return undefined; + const status = (error as { status?: unknown }).status; + return typeof status === 'number' ? status : undefined; +} + /** The organization client, or a typed refusal for an operation that needs the API key. */ export function requireLimrunOrgClient( limrun: Limrun | undefined, diff --git a/packages/provider-limrun/src/lease-ownership.ts b/packages/provider-limrun/src/lease-ownership.ts new file mode 100644 index 0000000000..699e4aaf8d --- /dev/null +++ b/packages/provider-limrun/src/lease-ownership.ts @@ -0,0 +1,163 @@ +import type { DeviceLease } from '@agent-device/contracts/device'; +import type { LimrunPlatform } from './device.ts'; +import type { LimrunAndroidSession } from './android.ts'; +import type { LimrunIosSession } from './ios.ts'; +import type { NetworkCapture, SettledNetworkCapture } from './network-har.ts'; + +/** + * One lease's resources in this runtime, and the phase it is in: the create that is making its instance + * (`allocating`), the instance that create made (`live`), or the one disposal that is ending it + * (`releasing`). This is the only place a lease's instance, keep-alive timer, network capture and app alias + * are recorded, so every route that can end one — release, shutdown, expired-lease recovery, and the + * allocation still waiting on the create — reads it here. + * + * The owner is published before its allocation can await anything, so a duplicate allocation joins that + * create instead of making a second instance, and a release or shutdown arriving while the create runs + * finds the instance it is on its way to making. `releasing` is entered in the same synchronous step that + * starts the disposal, and the owner is dropped only once that disposal has confirmed, so a lease is never + * between owners while one of its instances may be live: an allocation that arrives mid-release is refused, + * a second release joins the disposal in flight, and shutdown awaits it rather than starting another. + * + * Only `LimrunRuntimeImplementation` holds owners, and only its routes write them, so the fields stay + * mutable and no owner reaches a caller. + */ +export type LimrunLeaseOwner = { + readonly lease: DeviceLease; + /** Which platform's instance this lease's create is making, and which one its delete must name. */ + readonly platform: LimrunPlatform; + /** + * The create that is making this lease's instance; never rejects unobserved — the runtime reads it. It + * carries its own outcome, including a refusal to bill anything for the flags of the caller that started it. + * A joiner whose own flags were answered for replaces an outcome that billed nothing with its own create. + */ + creation: Promise; + state: LimrunLeaseState; + /** + * The instance this lease's create rented, recorded the moment Limrun answered, before any session was + * built on it. A create that could not finish still owes an end for it, and its disposal reads it here. + */ + rentedInstanceId: string | undefined; + /** + * The session this lease's create built, recorded the moment it existed. It is the handle that holds the + * instance connection and the host ADB endpoint open, so it stays on the owner even when a later step of + * the allocation fails: whoever ends this lease ends those resources too, not only the instance. + */ + session: LimrunRuntimeSession | undefined; + /** + * The one release of this owner's host resources: the instance connection and the Android ADB endpoint. + * Disposal starts it, and the continuation of a create that answered after its disposal refused answers + * to the same promise, so a session that arrived late is never released twice and never skipped. + */ + localRelease: Promise | undefined; + /** The keep-alive interval for the live instance; started when it went live, cleared by its disposal. */ + keepAlive: NodeJS.Timeout | undefined; + /** What this lease has in capture terms, including a start whose tunnel is still being opened. */ + capture: LimrunOwnerCapture | undefined; + /** The uploaded app the instance was created with, and the identifier Limrun installed it under. */ + appAlias: LimrunAppAlias | undefined; + /** The instance Limrun has already confirmed deleted, so a retried disposal never bills a second delete. */ + deletedInstanceId: string | undefined; + /** + * The delete this owner has in flight: the instance it names, and the request Limrun has not answered. A + * route that owes the end of that instance waits on this request rather than sending Limrun a second one — + * a delete whose answer this disposal stopped waiting for is still the delete that instance is getting. + */ + pendingDelete: + | Readonly<{ instanceId: string; requested: Promise }> + | undefined; +}; + +/** What one lease has live in this runtime, whatever its platform. */ +export type LimrunRuntimeSession = LimrunIosSession | LimrunAndroidSession; + +/** Whichever route can end one lease: a release, a daemon close, or the create that failed. */ +export type LimrunTeardownRoute = 'release' | 'shutdown' | 'create'; + +/** + * What one lease's app alias maps: the uploaded asset's name, and what Limrun installed it under. */ +export type LimrunAppAlias = Readonly<{ assetName: string; installedAppId: string }>; + +/** + * What one lease has in capture terms: the start that is still opening its destination tunnel + * (`starting`), the capture that start came to, or what a settled capture was written as. `starting` is + * published before the start can answer, so a duplicate allocation joins it instead of opening a second + * tunnel, and a teardown finds a tunnel that is not there yet and ends it where it lands. + */ +export type LimrunOwnerCapture = + | Readonly<{ phase: 'starting'; readonly opening: Promise }> + | NetworkCapture; + +/** + * What ending one lease came to. `cleanup` answers whether this lease's instance is certainly gone: + * `confirmed` once Limrun deleted it, `attached` when the instance was never this runtime's to delete, + * `uncertain` when deleting it failed or is still owed, and `none` when the lease never had an instance of + * its own. + */ +export type LimrunDisposalOutcome = Readonly<{ + cleanup: 'confirmed' | 'attached' | 'uncertain' | 'none'; + /** + * Creation outlived this disposal's wait. An uncertain outcome re-takes the owner after creation settles; + * a confirmed delete already ended the rented instance, and its local teardown awaits the late session. + */ + creationPending?: true; + /** The instance the lease held, known as soon as the create named it. */ + instanceId: string | undefined; + /** Why a disposal could not be confirmed; absent once the instance is gone. */ + failure: string | undefined; + /** + * The refusal behind that failure, when one exists: the delete's own error, or the crash of the disposal + * itself when it could not report anything. Reported as an error's cause, where it renders, rather than + * only as a string in details. + */ + cause: unknown; + /** The released capture's record, so a refusal can still carry the HAR that was written. */ + capture: SettledNetworkCapture | undefined; +}>; + +/** + * What one instance delete came to: whether Limrun has the instance gone, and what refused it otherwise. + */ +export type LimrunDeleteResult = Readonly<{ + failure: string | undefined; + /** The refusal behind that failure, for whoever has to report it. */ + cause: unknown; +}>; + +/** + * What one create amounted to: the session it made, why it failed, or that the caller who started it was + * refused before anything was billed. `refused_caller` is that caller's own answer and not the lease's: an + * owner that rented nothing stays allocatable for a caller whose flags this provider does read, and the + * distinction is what keeps one caller's refusal from becoming another caller's outcome. An instance a create + * rented before it failed is not carried here — the owner records it as soon as Limrun names it, which is + * earlier than any outcome could be reported. + */ +export type LimrunCreateOutcome = + | Readonly<{ + status: 'ready'; + session: LimrunRuntimeSession; + appAlias: LimrunAppAlias | undefined; + }> + | Readonly<{ status: 'failed'; error: unknown }> + | Readonly<{ status: 'refused_caller'; error: unknown }>; + +type LimrunLeaseState = + | Readonly<{ phase: 'allocating' }> + | Readonly<{ phase: 'live'; session: LimrunRuntimeSession }> + | LimrunReleasingState; + +/** + * The state of a lease one teardown route owns. Only the take writes it, and only while that route's + * disposal runs: `settled` is what that disposal reported, and `instanceId` grows to name the instance the + * create announced while the teardown was already under way. + */ +type LimrunReleasingState = { + phase: 'releasing'; + /** Which route took the lease from its allocation; one route owns the owner until it confirms. */ + by: LimrunTeardownRoute; + /** The instance being ended, known as soon as the create named it. */ + instanceId: string | undefined; + /** The one disposal of this owner, joined by every caller that arrives while it runs. */ + readonly disposal: Promise; + /** What that disposal reported; missing while it runs, so a caller can tell gone from unseen. */ + settled: LimrunDisposalOutcome | undefined; +}; diff --git a/packages/provider-limrun/src/lease-release-observation.test.ts b/packages/provider-limrun/src/lease-release-observation.test.ts new file mode 100644 index 0000000000..f1beb07e1c --- /dev/null +++ b/packages/provider-limrun/src/lease-release-observation.test.ts @@ -0,0 +1,376 @@ +import { test, vi, expect } from 'vitest'; +import fs from 'node:fs'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { createLimrunRuntime } from './runtime.ts'; +import { stubLimrunRuntimeDependencies } from './runtime-dependencies.fixtures.ts'; +import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; +import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; +import { watchUnhandledRejections } from './lifecycle.fixtures.ts'; + +/* + * A disposal starts its capture end and its local teardown beside the instance delete and joins them only + * after it. A step of either one that refuses inside that window has this disposal as its observer from the + * moment it started: nothing a disposal begins may reach the process as an unhandled rejection, and a + * refused capture or teardown never changes what the delete answered for. + */ + +const state = vi.hoisted(() => ({ + /** Instance deletes this runtime asked Limrun for; each waits for its case to answer it. */ + deletes: [] as Array<{ instanceId: string; report: () => void }>, + creates: 0, + clientStarts: 0, + clientGate: undefined as Promise | undefined, + createGate: undefined as Promise | undefined, + autoDelete: false, + disconnects: 0, + /** Made to throw by the case whose session disconnect refuses while its delete is still owed. */ + disconnectThrows: false, + captureCloses: 0, + /** Made to throw by the case whose capture tunnel close refuses as the release writes its record. */ + captureCloseThrows: false, +})); + +vi.mock('@limrun/api', () => ({ + default: class MockLimrun { + readonly iosInstances = { create: vi.fn(), list: vi.fn(), delete: vi.fn() }; + readonly androidInstances = { + create: async () => { + state.creates += 1; + await state.createGate; + return { + metadata: { id: 'android-instance-1' }, + status: { + token: 'instance-token', + apiUrl: 'https://android.example', + adbWebSocketUrl: 'wss://adb.example', + }, + }; + }, + list: vi.fn(async () => ({ getPaginatedItems: () => [] })), + // Every delete waits for its case to answer it, so a release can be held at its delete while the + // local teardown it started beside has long since refused. + delete: vi.fn(async (instanceId: string) => { + if (state.autoDelete) return; + let report!: () => void; + const answered = new Promise((answeredNow) => { + report = answeredNow; + }); + state.deletes.push({ instanceId, report }); + await answered; + }), + }; + readonly assets = { getOrUpload: vi.fn(), list: vi.fn(async () => []) }; + }, +})); + +vi.mock('@limrun/api/instance-client', () => ({ + createInstanceClient: vi.fn(async () => { + state.clientStarts += 1; + await state.clientGate; + return { + disconnect: () => { + state.disconnects += 1; + if (state.disconnectThrows) throw new Error('instance client refused to disconnect'); + }, + keepAlive: () => {}, + openUrl: vi.fn(), + sendAsset: vi.fn(), + pressKey: vi.fn(), + setText: vi.fn(), + tap: vi.fn(), + startAdbTunnel: vi.fn(async () => ({ + address: { address: '127.0.0.1', port: 62001 }, + close: () => {}, + })), + startTunnel: vi.fn(async () => ({ + tunnelId: 't1', + selectors: [], + close: () => { + state.captureCloses += 1; + if (state.captureCloseThrows) throw new Error('capture tunnel refused to close'); + }, + })), + getTunnelStatus: vi.fn(async () => ({})), + }; + }), +})); + +test('a local teardown that refuses beside a pending delete is reported, not loosed on the process', async () => { + const rejections = watchUnhandledRejections(); + state.deletes.length = 0; + const disconnectsBefore = state.disconnects; + state.disconnectThrows = true; + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-throwing-disconnect'); + let reportDelete: (() => void) | undefined; + try { + await runtime.leaseLifecycle.allocate?.(lease); + // The release holds at its delete while the session disconnect it started beside refuses. + const released = runtime.leaseLifecycle.release?.(lease); + await vi.waitFor(() => expect(state.deletes).toHaveLength(1)); + expect(state.disconnects - disconnectsBefore).toBe(1); + reportDelete = state.deletes[0]?.report; + reportDelete?.(); + + // The delete's confirmation is the release's answer; what the local teardown came to rides beside it + // as the warning it is: a host resource this process could not put down. + const record = (await released) as Record; + expect(record.limrunInstanceId).toBe('android-instance-1'); + const warnings = warningsOf(record); + expect(warnings.map((warning) => warning.code)).toEqual(['LIMRUN_TEARDOWN_INCOMPLETE']); + expect(String(warnings[0]?.message)).toBe( + 'Local teardown refused to finish: instance client refused to disconnect', + ); + await rejections.assertIsSilent(); + } finally { + rejections.stop(); + reportDelete?.(); + state.deletes.forEach(({ report }) => report()); + state.deletes.length = 0; + state.disconnectThrows = false; + await runtime.shutdown(); + } +}); + +test('release clears reverse mappings before disconnect closes their ADB endpoint', async () => { + const disconnectsBefore = state.disconnects; + const removed = vi.fn(async () => {}); + const dependencies = testDependencies(); + dependencies.android.createPortReverse = async (adb) => ({ + ensure: async () => { + await adb(['reverse', 'tcp:18441', 'tcp:18441']); + }, + list: async () => { + if (state.disconnects !== disconnectsBefore) throw new Error('ADB endpoint is disconnected'); + return [{ local: 'tcp:18441', remote: 'tcp:18441' }]; + }, + remove: removed, + removeAllOwned: async () => {}, + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, dependencies); + const lease = androidLease('lease-reverse-before-disconnect'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + await runtime.configurePortReverse?.({ + leaseId: lease.leaseId, + provider: 'limrun', + devicePort: 18441, + hostPort: 18441, + name: 'live-regression', + }); + const released = runtime.leaseLifecycle.release?.(lease); + await vi.waitFor(() => expect(state.deletes).toHaveLength(1)); + state.deletes[0]?.report(); + expect(await released).toMatchObject({ limrunInstanceId: 'android-instance-1' }); + expect(removed).toHaveBeenCalledWith('tcp:18441'); + expect(state.disconnects - disconnectsBefore).toBe(1); + } finally { + state.deletes.forEach(({ report }) => report()); + state.deletes.length = 0; + await runtime.shutdown(); + } +}); + +test('a capture close that refuses still leaves the delete it ran beside answered for', async () => { + const rejections = watchUnhandledRejections(); + const harDir = mkdtempForTestSync('limrun-har-observe-'); + state.deletes.length = 0; + const captureClosesBefore = state.captureCloses; + state.captureCloseThrows = true; + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', networkHarDir: harDir }, + testDependencies(), + ); + const lease = androidLease('lease-throwing-capture-close'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + await runtime.leaseLifecycle.allocate?.(lease, { flags: { providerNetworkLogs: true } }); + let reportDelete: (() => void) | undefined; + try { + const released = runtime.leaseLifecycle.release?.(lease); + await vi.waitFor(() => expect(state.deletes).toHaveLength(1)); + reportDelete = state.deletes[0]?.report; + reportDelete?.(); + + // The record's write ends with the tunnel's close, and that close refuses inside the disposal's + // join window: the instance the delete confirmed is gone, and the release answers for it. + const record = (await released) as Record; + expect(record.limrunInstanceId).toBe('android-instance-1'); + expect(state.captureCloses - captureClosesBefore).toBe(1); + expect(warningsOf(record).map((warning) => warning.message)).toEqual([ + 'Capture close failed: capture tunnel refused to close', + ]); + const harPath = `${harDir}/${lease.leaseId}.har`; + expect(record).toMatchObject({ + cloudArtifacts: { status: 'ready', cloudArtifacts: [{ url: harPath }] }, + }); + expect(fs.existsSync(harPath)).toBe(true); + const listed = await runtime.cloudArtifacts?.listCloudArtifacts?.({ leaseId: lease.leaseId }); + expect(listed).toMatchObject({ + status: 'ready', + message: expect.stringContaining('capture tunnel refused to close'), + }); + await rejections.assertIsSilent(); + } finally { + reportDelete?.(); + } + } finally { + rejections.stop(); + state.deletes.forEach(({ report }) => report()); + state.deletes.length = 0; + state.captureCloseThrows = false; + await runtime.shutdown(); + } +}); + +test('a late create with failed local cleanup is freed by the normal take with confirmed deletion', async () => { + await import('./lease-release.ts'); + const rejections = watchUnhandledRejections(); + const createsBefore = state.creates; + state.deletes.length = 0; + state.disconnectThrows = true; + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-late-failed-local'); + const allocating = Promise.resolve(runtime.leaseLifecycle.allocate?.(lease)).catch( + (error: unknown) => error, + ); + try { + await vi.waitFor(() => expect(state.creates - createsBefore).toBe(1)); + const expired = Promise.resolve(runtime.leaseLifecycle.release?.(lease)).catch( + (error: unknown) => error, + ); + vi.useFakeTimers(); + await vi.advanceTimersByTimeAsync(130_000); + expect(await expired).toMatchObject({ + details: { reason: 'limrun_instance_delete_unconfirmed' }, + }); + reportCreate(); + await vi.advanceTimersByTimeAsync(0); + expect(state.deletes).toHaveLength(1); + let answered = false; + const joining = Promise.resolve(runtime.leaseLifecycle.allocate?.(lease)) + .catch((error: unknown) => error) + .then((answer) => { + answered = true; + return answer; + }); + await vi.advanceTimersByTimeAsync(0); + expect(answered).toBe(false); + const release = runtime.leaseLifecycle.release?.(lease); + state.deletes[0]?.report(); + await vi.advanceTimersByTimeAsync(0); + const record = (await release) as Record; + expect(record.limrunInstanceId).toBe('android-instance-1'); + expect(warningsOf(record)).toMatchObject([ + { + code: 'LIMRUN_TEARDOWN_INCOMPLETE', + message: 'Local teardown refused to finish: instance client refused to disconnect', + }, + ]); + expect(await joining).toMatchObject({ + details: { + reason: 'limrun_lease_released_mid_allocation', + hint: expect.stringContaining('was ended by the release'), + }, + }); + await allocating; + state.disconnectThrows = false; + state.createGate = undefined; + await runtime.leaseLifecycle.allocate?.(lease); + expect(state.creates - createsBefore).toBe(2); + vi.useRealTimers(); + await rejections.assertIsSilent(); + } finally { + reportCreate(); + state.createGate = undefined; + state.disconnectThrows = false; + state.autoDelete = true; + state.deletes.forEach(({ report }) => report()); + vi.useRealTimers(); + await runtime.shutdown(); + state.autoDelete = false; + state.deletes.length = 0; + rejections.stop(); + } +}); + +test('a confirmed delete during creation still disconnects the late session without another rent', async () => { + await import('./lease-release.ts'); + const rejections = watchUnhandledRejections(); + const createsBefore = state.creates; + const clientsBefore = state.clientStarts; + const disconnectsBefore = state.disconnects; + state.deletes.length = 0; + let reportClient!: () => void; + state.clientGate = new Promise((resolve) => { + reportClient = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-confirmed-before-session'); + const allocating = Promise.resolve(runtime.leaseLifecycle.allocate?.(lease)).catch( + (error: unknown) => error, + ); + try { + // Reaching the instance client means the one POST has already named the rented instance. + await vi.waitFor(() => expect(state.clientStarts - clientsBefore).toBe(1)); + const released = runtime.leaseLifecycle.release?.(lease); + vi.useFakeTimers(); + await vi.advanceTimersByTimeAsync(125_000); + expect(state.deletes).toHaveLength(1); + state.deletes[0]?.report(); + await vi.advanceTimersByTimeAsync(50_000); + expect(await released).toMatchObject({ + limrunInstanceId: 'android-instance-1', + warnings: [{ code: 'LIMRUN_TEARDOWN_INCOMPLETE' }], + }); + expect(state.disconnects - disconnectsBefore).toBe(0); + // The owner has been closed, but its observed, memoized local release still owns this session. + reportClient(); + await vi.advanceTimersByTimeAsync(0); + expect(await allocating).toMatchObject({ + details: { reason: 'limrun_lease_released_mid_allocation' }, + }); + expect(state.creates - createsBefore).toBe(1); + expect(state.deletes).toHaveLength(1); + expect(state.disconnects - disconnectsBefore).toBe(1); + vi.useRealTimers(); + await rejections.assertIsSilent(); + } finally { + reportClient(); + state.clientGate = undefined; + state.deletes.forEach(({ report }) => report()); + vi.useRealTimers(); + await runtime.shutdown(); + state.deletes.length = 0; + rejections.stop(); + } +}); + +function warningsOf(record: Record): Array> { + return Array.isArray(record.warnings) ? (record.warnings as Record[]) : []; +} + +function androidLease(leaseId: string): DeviceLease { + return { + leaseId, + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + }; +} + +/** + * The host and platform adapters these cases run on: every one of them answers a fixed value. The client + * connection and the instance delete are driven from the mock `state` above. + */ +function testDependencies(): LimrunRuntimeDependencies { + return stubLimrunRuntimeDependencies(); +} diff --git a/packages/provider-limrun/src/lease-release.test.ts b/packages/provider-limrun/src/lease-release.test.ts new file mode 100644 index 0000000000..00f2a90058 --- /dev/null +++ b/packages/provider-limrun/src/lease-release.test.ts @@ -0,0 +1,450 @@ +import { test, vi, expect } from 'vitest'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { AppError } from '@agent-device/kernel/errors'; +import type Limrun from '@limrun/api'; +import { + allocationRefusedForTake, + cleanupLeftInstancesRunning, + createLeftInstanceRunning, + disposeLimrunLeaseOwner, + releaseLimrunLease, + type LimrunDisposalHost, +} from './lease-release.ts'; +import type { + LimrunCreateOutcome, + LimrunDisposalOutcome, + LimrunLeaseOwner, +} from './lease-ownership.ts'; + +/* + * Each wait is longer than the budgets it steps past: a disposal's create wait, and everything a disposal + * left without a create answer may still wait for — one delete, the local teardown, one capture record. + */ +const CREATE_ANSWER_WAIT_MS = 130_000; +const DISPOSAL_SETTLE_WAIT_MS = 110_000; + +/** + * A create that answers after its disposal stopped waiting owes its end to the disposal already over. If that + * disposal's delete is still with Limrun, the late end joins it: one instance is deleted by one request. + */ +test('a create that answers late joins the delete its disposal already started', async () => { + // Loaded before the clock is replaced, so the disposal's own budget timers are fake timers. + await import('./lifecycle-budget.ts'); + let reportDelete!: () => void; + const deleteHost = vi.fn(async () => { + await new Promise((resolve) => { + reportDelete = resolve; + }); + return { failure: undefined, cause: undefined }; + }); + const host: LimrunDisposalHost = { + deleteInstance: deleteHost, + endCapture: async () => undefined, + }; + let reportCreate!: () => void; + const creation = new Promise((resolve) => { + // A create that rented the instance and then failed, after its disposal had given up on it. + reportCreate = () => resolve({ status: 'failed', error: new Error('instance boot failed') }); + }); + const owner = ownerWithRentedInstance(creation); + + vi.useFakeTimers(); + try { + const disposal = disposeLimrunLeaseOwner(host, owner); + // The disposal stops waiting on the create and asks Limrun once to end the instance it did rent. + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS); + expect(deleteHost).toHaveBeenCalledOnce(); + // Its own delete then runs out its deadline, and the disposal refuses its caller. + await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + await expect(disposal).resolves.toMatchObject({ cleanup: 'uncertain' }); + + // The create answers now. The end it owes is this disposal's, and it joins the request Limrun already + // has for this instance rather than sending a second one. + reportCreate(); + const retaken = disposeLimrunLeaseOwner(host, owner); + await vi.advanceTimersByTimeAsync(0); + expect(deleteHost).toHaveBeenCalledOnce(); + + // The one request confirming is enough: the joined end records the instance as deleted. + reportDelete(); + await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + await expect(retaken).resolves.toMatchObject({ cleanup: 'confirmed' }); + } finally { + vi.useRealTimers(); + } + expect(owner.deletedInstanceId).toBe('android-instance-1'); +}); + +/** + * A delete Limrun refused keeps its own error on the disposal outcome, and the refusals every route raises + * carry it as their cause — where a cause renders — beside the text that says the delete was unconfirmed. + */ +test("a refused delete reports its own error as the refusal's cause", async () => { + await import('./lifecycle-budget.ts'); + const refusal = new Error('Limrun refused to delete the instance'); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ + failure: 'Limrun refused to delete the instance', + cause: refusal, + }), + endCapture: async () => undefined, + }; + const owner = ownerWithRentedInstance( + Promise.resolve({ status: 'failed', error: new Error('boot failed') }), + ); + const outcome = await disposeLimrunLeaseOwner(host, owner); + expect(outcome.cleanup).toBe('uncertain'); + expect(outcome.cause).toBe(refusal); + + // The allocation's refusal names both what it could not settle and why its create failed at all. + const lease = (owner as unknown as { lease: DeviceLease }).lease; + const thrown = captureThrow(() => + createLeftInstanceRunning(lease, outcome, new Error('boot failed')), + ); + expect(thrown.details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect(String(thrown.details?.cause)).toContain('boot failed'); + expect(thrown.cause).toBe(refusal); +}); + +/** + * A daemon close that could not end two leases names both of them and why each is still held, not only the + * first — every one of them may need to be named to Limrun by hand. + */ +test('a close that could not end two leases reports both reasons', async () => { + const refusal = new Error('Limrun refused to delete the instance'); + const outcomes: LimrunDisposalOutcome[] = [ + { + cleanup: 'uncertain', + instanceId: 'android-instance-1', + failure: 'delete one refused', + cause: refusal, + capture: undefined, + }, + { + cleanup: 'uncertain', + instanceId: undefined, + failure: undefined, + cause: undefined, + capture: undefined, + }, + ]; + const thrown = captureThrow(() => cleanupLeftInstancesRunning(outcomes)); + expect(thrown.details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect(thrown.details?.limrunInstanceIds).toEqual(['android-instance-1']); + expect(String(thrown.details?.cause)).toContain('delete one refused'); + expect(String(thrown.details?.cause)).toContain('an unconfirmed delete'); + expect(thrown.cause).toBe(refusal); +}); + +/** + * The label-based recovery route deletes through the same helper every other route deletes with: a 404 is + * Limrun answering that the instance is gone, and a delete it could not confirm refuses with the typed error + * naming every instance the labels carried, because the sweep that retries this route has nothing else to go on. + */ +test('recovery by labels confirms a gone instance and reports every id it ended', async () => { + const deletes: string[] = []; + const limrun = labeledInstancesClient( + ['android-instance-a', 'android-instance-b'], + async (id) => { + deletes.push(id); + }, + ); + const record = await releaseLimrunLease(limrun, leaseForRecovery(), undefined); + expect(deletes).toEqual(['android-instance-a', 'android-instance-b']); + expect(record).toMatchObject({ + limrunInstanceId: 'android-instance-a', + limrunInstanceIds: ['android-instance-a', 'android-instance-b'], + limrunInstanceCount: 2, + }); +}); + +test('recovery stops waiting for a delete whose response never finishes', async () => { + await import('./lifecycle-budget.ts'); + let reportDelete!: () => void; + const pending = new Promise((resolve) => { + reportDelete = resolve; + }); + const limrun = labeledInstancesClient(['android-instance-a'], async () => await pending); + let outcome: unknown; + vi.useFakeTimers(); + try { + const released = releaseLimrunLease(limrun, leaseForRecovery(), undefined).then( + (value) => { + outcome = value; + }, + (error: unknown) => { + outcome = error; + }, + ); + await vi.advanceTimersByTimeAsync(30_001); + expect(outcome).toBeInstanceOf(AppError); + expect((outcome as AppError).details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect((outcome as AppError).details?.hint).toContain('android-instance-a'); + reportDelete(); + await released; + } finally { + reportDelete(); + vi.useRealTimers(); + } +}); + +test('recovery by labels treats a 404 as confirmed, the way every other delete route does', async () => { + const limrun = labeledInstancesClient(['android-instance-a'], async () => { + throw Object.assign(new Error('not found'), { status: 404 }); + }); + const record = await releaseLimrunLease(limrun, leaseForRecovery(), undefined); + expect(record).toMatchObject({ limrunInstanceId: 'android-instance-a' }); +}); + +/** + * A label delete Limrun refused is a lease the daemon's sweep must retry: the refusal is the typed one, it + * names the instance still in question beside every id the labels had, and the confirmed ones are said apart. + */ +test('recovery by labels refuses with every id when a delete is not confirmed', async () => { + const limrun = labeledInstancesClient( + ['android-instance-a', 'android-instance-b', 'android-instance-c'], + async (id) => { + if (id === 'android-instance-b') throw new Error('Limrun refused to delete the instance'); + return undefined; + }, + ); + const thrown = await captureReject(() => + releaseLimrunLease(limrun, leaseForRecovery(), undefined), + ); + expect(thrown.details?.reason).toBe('limrun_instance_delete_unconfirmed'); + expect(thrown.details?.limrunInstanceId).toBe('android-instance-b'); + expect(thrown.details?.limrunInstanceIds).toEqual([ + 'android-instance-a', + 'android-instance-b', + 'android-instance-c', + ]); + expect(String(thrown.details?.hint)).toContain('android-instance-c'); + expect(String(thrown.details?.hint)).toContain( + 'Ask Limrun to delete: android-instance-b, android-instance-c.', + ); + expect(String(thrown.details?.hint)).toContain('android-instance-a was confirmed deleted'); + expect(String(thrown.cause)).toContain('refused'); +}); + +/** + * What a close says when the work it could not finish has no instance to name yet: the lease and the work. + * A take-refusal for an instance whose delete was not confirmed says so rather than calling it ended: + * the refused caller is the last route that will hear the instance may still be billing. + */ +test('a take-refusal for an unconfirmed delete does not call the instance ended', () => { + const lease = leaseForRecovery(); + const confirmed = captureThrow(() => + allocationRefusedForTake(lease, 'release', { + cleanup: 'confirmed', + instanceId: 'android-instance-1', + failure: undefined, + cause: undefined, + capture: undefined, + }), + ); + expect(String(confirmed.details?.hint)).toContain('was ended by the release'); + + const uncertain = captureThrow(() => + allocationRefusedForTake(lease, 'shutdown', { + cleanup: 'uncertain', + instanceId: 'android-instance-1', + failure: 'delete refused', + cause: new Error('refused'), + capture: undefined, + }), + ); + expect(String(uncertain.details?.hint)).toContain('could not confirm'); + expect(String(uncertain.details?.hint)).toContain('android-instance-1'); + expect(uncertain.details?.retriable).toBe(true); +}); + +/** + * A late route that could not give the instance back must keep the lease held: the next release, recovery or + * shutdown is the route that retries the delete, and only it can tell its caller the instance is gone. + */ +test('a late create whose delete Limrun refused keeps disposal uncertain', async () => { + await import('./lifecycle-budget.ts'); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ + failure: 'Limrun refused to delete the instance', + cause: undefined, + }), + endCapture: async () => undefined, + }; + let reportCreate!: () => void; + const creation = new Promise((resolve) => { + reportCreate = () => resolve({ status: 'failed', error: new Error('instance boot failed') }); + }); + const owner = ownerWithRentedInstance(creation); + vi.useFakeTimers(); + try { + const disposal = disposeLimrunLeaseOwner(host, owner); + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS + DISPOSAL_SETTLE_WAIT_MS); + await expect(disposal).resolves.toMatchObject({ cleanup: 'uncertain' }); + // The create answers after that, and the end it owes is refused by Limrun as well: the lease stays held. + reportCreate(); + const retaken = disposeLimrunLeaseOwner(host, owner); + await vi.advanceTimersByTimeAsync(DISPOSAL_SETTLE_WAIT_MS); + await expect(retaken).resolves.toMatchObject({ + cleanup: 'uncertain', + failure: 'Limrun refused to delete the instance', + }); + } finally { + vi.useRealTimers(); + } + expect(owner.deletedInstanceId).toBeUndefined(); +}); + +/** + * The disposal's join for a capture record contains the destination tunnel's own open budget: a capture end + * that spends almost all of that handshake and still comes to in time is answered for without a warning, + * because the join is longer than the work inside it. + */ +test('a capture end that spends its tunnel budget is still joined in time', async () => { + await import('./lifecycle-budget.ts'); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ failure: undefined, cause: undefined }), + // The tunnel answers at the very end of its own 20 s budget, then the record is written. + endCapture: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 20_000); + }); + return { phase: 'written', harPath: '/tmp/late.har', failures: { count: 0, truncated: 0 } }; + }, + }; + const owner = ownerWithRentedInstance( + // A create that failed but named the instance (rentedInstanceId is set by the fixture): the disposal + // ends that instance and joins the capture record it started beside the delete. + Promise.resolve({ + status: 'failed', + error: new Error('create answered after its session was gone'), + }), + ); + vi.useFakeTimers(); + let settled: LimrunDisposalOutcome | undefined; + try { + const disposal = disposeLimrunLeaseOwner(host, owner).then((outcome) => { + settled = outcome; + }); + await vi.advanceTimersByTimeAsync(31_000); + await disposal; + } finally { + vi.useRealTimers(); + } + expect(settled?.cleanup).toBe('confirmed'); + expect(settled?.failure).toBeUndefined(); +}); + +test('a rejected capture report preserves its error without changing confirmed deletion', async () => { + const refusal = new Error('HAR inspection refused'); + const owner = ownerWithRentedInstance( + Promise.resolve({ status: 'failed', error: new Error('boot failed') }), + ); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ failure: undefined, cause: undefined }), + endCapture: async () => { + throw refusal; + }, + }; + const outcome = await disposeLimrunLeaseOwner(host, owner); + expect(outcome.cleanup).toBe('confirmed'); + expect(outcome.failure).toBe('Capture record refused to be written: HAR inspection refused'); +}); + +test('an unanswered create does not make a confirmed instance delete uncertain', async () => { + await import('./lifecycle-budget.ts'); + let reportCreate!: () => void; + const creation = new Promise((resolve) => { + reportCreate = () => resolve({ status: 'failed', error: new Error('boot failed') }); + }); + const owner = ownerWithRentedInstance(creation); + const host: LimrunDisposalHost = { + deleteInstance: async () => ({ failure: undefined, cause: undefined }), + endCapture: async () => undefined, + }; + vi.useFakeTimers(); + try { + const disposal = disposeLimrunLeaseOwner(host, owner); + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS + DISPOSAL_SETTLE_WAIT_MS); + await expect(disposal).resolves.toMatchObject({ + cleanup: 'confirmed', + instanceId: 'android-instance-1', + }); + } finally { + reportCreate(); + await vi.advanceTimersByTimeAsync(0); + vi.useRealTimers(); + } +}); + +function captureReject(raise: () => Promise): Promise { + return raise().then( + () => { + throw new Error('expected the refusal to reject'); + }, + (error: unknown) => error as AppError, + ); +} + +/** A Limrun whose labels name these instances, answering every delete the way a case says. */ +function labeledInstancesClient( + instanceIds: readonly string[], + remove: (instanceId: string) => Promise, +): Limrun { + const page = { getPaginatedItems: () => instanceIds.map((id) => ({ metadata: { id } })) }; + const resource = { + list: async () => page, + delete: async (instanceId: string) => await remove(instanceId), + }; + return { iosInstances: resource, androidInstances: resource } as unknown as Limrun; +} + +function leaseForRecovery(): DeviceLease { + return { + leaseId: 'lease-labeled-recovery', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + }; +} + +function captureThrow(raise: () => never): AppError { + try { + raise(); + } catch (error) { + return error as AppError; + } + throw new Error('expected the refusal to throw'); +} + +/** An owner whose create has rented an instance but has not answered, taken by a disposal. */ +function ownerWithRentedInstance(creation: Promise): LimrunLeaseOwner { + const lease = { + leaseId: 'lease-release-join', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease; + return { + lease, + platform: 'android', + creation, + state: { phase: 'releasing', by: 'release' }, + rentedInstanceId: 'android-instance-1', + session: undefined, + localRelease: undefined, + keepAlive: undefined, + capture: undefined, + appAlias: undefined, + deletedInstanceId: undefined, + pendingDelete: undefined, + } as unknown as LimrunLeaseOwner; +} diff --git a/packages/provider-limrun/src/lease-release.ts b/packages/provider-limrun/src/lease-release.ts new file mode 100644 index 0000000000..bb621d5468 --- /dev/null +++ b/packages/provider-limrun/src/lease-release.ts @@ -0,0 +1,597 @@ +import type Limrun from '@limrun/api'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import { cleanupLimrunAndroidAdbTunnel } from './android.ts'; +import { CAPTURE_OPEN_BUDGET_MS } from './capture-lifecycle.ts'; +import { deleteLimrunInstance } from './client-options.ts'; +import type { LimrunBudgetOutcome } from './lifecycle-budget.ts'; +import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend, type LimrunPlatform } from './device.ts'; +import type { + LimrunDeleteResult, + LimrunDisposalOutcome, + LimrunLeaseOwner, + LimrunRuntimeSession, + LimrunTeardownRoute, +} from './lease-ownership.ts'; +import type { SettledNetworkCapture } from './network-har.ts'; + +/** + * Everything this runtime does once a lease has been taken for good: end what its owner holds, and answer with + * what came of it. One lease's disposal runs here, and every route that ends a lease — release, daemon close, + * expired-lease recovery, and the create that failed — answers from the same outcome, so none of them can + * report a lease as released on anything short of Limrun confirming the delete. + * + * Every wait here is bounded. Work that outlives its deadline keeps running and stays observed: this module + * ends what it eventually makes, and reports what it could not, rather than holding a caller open. + */ + +/** + * How long a disposal waits for the create to say what it rented. The create itself keeps Limrun's own + * timeouts: this bounds only the disposal's wait, so an instance is never held open by a create that stopped + * answering, and its owner disposes what that create eventually names. + */ +const CREATE_ANSWER_BUDGET_MS = 120_000; + +/** + * How long a disposal waits for one delete. The SDK clears a request's timer before it reads the response body, + * so a DELETE whose body never arrives would otherwise hold the release, the daemon close and every joining + * allocation open forever. + */ +const DELETE_ANSWER_BUDGET_MS = 30_000; + +/** + * How long a disposal waits for the local resources it started beside the delete: the instance connection and + * the Android ADB endpoint, whose `adb connect` the SDK runs without a deadline. + */ +const LOCAL_TEARDOWN_BUDGET_MS = 45_000; + +/** + * How long a disposal waits for a capture's HAR record, which is reporting and never a reason to hold a lease. + * The disposal joins a capture end that contains the destination tunnel's own open budget, so this is that + * handshake plus a window of its own for writing the HAR: a join shorter than the work inside it would report + * a handshake that is still inside its own stated budget as a record that refused. + */ +const CAPTURE_RECORD_BUDGET_MS = CAPTURE_OPEN_BUDGET_MS + 10_000; + +/** + * What a disposal may ask of the runtime that owns the lease, and nothing else: the delete of one instance, and + * the end of one lease's capture. Keeping the disposal to these two is what lets it end an owner without + * reaching back into the routes that allocate and read one, and without deciding when a lease is free again. + */ +export type LimrunDisposalHost = Readonly<{ + /** One instance delete: whether Limrun has the instance gone, and what refused it otherwise. */ + deleteInstance(platform: LimrunPlatform, instanceId: string): Promise; + /** Ends one lease's capture and answers with its record, for a runtime that has captured something. */ + endCapture(owner: LimrunLeaseOwner): Promise; +}>; + +/** + * Ends what one taken owner stands for: the instance its create made or rented, with every resource the owner + * holds beside it. This is the only place an instance is ended, and an owner is taken by exactly one route at a + * time, so one instance is ended exactly once. + */ +export async function disposeLimrunLeaseOwner( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + const { + awaitLimrunWithinBudget, + joinOutcomeFailures, + keepLimrunObserved, + workOutstandingFailure, + } = await import('./lifecycle-budget.ts'); + const create = await awaitLimrunWithinBudget(owner.creation, CREATE_ANSWER_BUDGET_MS); + // The create answered too late for this disposal to wait on it; what it goes on to name is still this + // disposal's to end. + if (!create.ok) return await disposeUnansweredCreate(host, owner); + // The owner records what its create actually made — the instance the moment Limrun named it, the session the + // moment it was built — so a create that failed or refused its caller is disposed from those, exactly like + // one that answered. + const session = owner.session; + const instanceId = session?.instanceId ?? owner.rentedInstanceId; + if (instanceId === undefined) return await disposeUnrented(host, owner); + // The capture's record is optional reporting, and local teardown can outlive its own steps: neither is + // allowed to stand between this disposal and the delete that stops the instance from being billed, so both + // are started here and joined after it. Both are observed in this same step: a step of either one that + // refuses before the join below is reached has this disposal as its observer from the moment it started, + // so no refusal of a resource this route already owns can reach the process as an unhandled rejection. + const capture = keepLimrunObserved(host.endCapture(owner)); + const local = keepLimrunObserved(releaseOwnerResources(owner)); + const instance = await endInstance(host, owner, instanceId); + const stuck = await awaitLocalTeardown(local); + // The record is optional reporting joined last: a capture that refuses to be written cannot keep a release + // from answering for the instance it ended, and is still listable once it does come to. + const joined = await awaitLimrunWithinBudget(capture, CAPTURE_RECORD_BUDGET_MS); + return { + cleanup: instance.cleanup, + instanceId, + failure: joinOutcomeFailures( + instance.failure, + joinOutcomeFailures(stuck, captureRecordOutcome(joined, workOutstandingFailure)), + ), + cause: instance.cause, + capture: joined.ok ? joined.value : undefined, + }; +} + +/** + * What the joined capture record leaves to report: nothing, a refusal of the record itself, or a wait + * that ran out. A refusal is reported as what it is, because a record that will never be written is not + * the same thing as one that may still arrive. + */ +function captureRecordOutcome( + joined: LimrunBudgetOutcome, + outstanding: (work: string, budgetMs: number) => string, +): string | undefined { + if (joined.ok) { + return joined.value?.closeFailure === undefined + ? undefined + : `Capture close failed: ${joined.value.closeFailure}`; + } + return 'rejected' in joined + ? `Capture record refused to be written: ${errorMessage(joined.error)}` + : outstanding('Capture record', CAPTURE_RECORD_BUDGET_MS); +} + +/** + * Ends what a disposal can name after creation outlives its wait. An uncertain outcome keeps the lease for + * a re-take after creation settles. A confirmed delete frees it: that create already named its only rented + * instance, and the memoized local teardown still awaits any session the create goes on to build. + */ +async function disposeUnansweredCreate( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + const { joinOutcomeFailures, keepLimrunObserved, workOutstandingFailure } = + await import('./lifecycle-budget.ts'); + // Observed the step it starts in, like the disposal's other early work: this route answers only after + // the local teardown below, so a capture end that refuses meanwhile has its observer already beside it. + const capture = keepLimrunObserved(awaitEndedCapture(host, owner)); + const outstanding = workOutstandingFailure( + `Instance create for lease '${owner.lease.leaseId}'`, + CREATE_ANSWER_BUDGET_MS, + ); + const instanceId = owner.rentedInstanceId; + if (instanceId === undefined) { + return { + cleanup: 'uncertain', + creationPending: true, + instanceId: undefined, + failure: outstanding, + cause: undefined, + capture: await capture, + }; + } + // The create has named its only rented instance, so confirming this delete ends its billing even while + // session construction is still pending. + const instance = await endInstance(host, owner, instanceId); + // Reported, not thrown: a local step that refuses is one this disposal can no longer fix, and the delete + // it ran beside has already been answered for. + const stuck = await awaitLocalTeardown(releaseOwnerResources(owner)); + return { + cleanup: instance.cleanup, + creationPending: true, + instanceId, + failure: joinOutcomeFailures(instance.failure, stuck ?? outstanding), + cause: instance.cause, + capture: await capture, + }; +} + +/** + * Ends what a disposal holds when its create answered and no instance is named for the lease: nothing this + * runtime rented is running, so the lease can be given back. + */ +async function disposeUnrented( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + return { + cleanup: 'none', + instanceId: undefined, + failure: undefined, + cause: undefined, + capture: await awaitEndedCapture(host, owner), + }; +} + +/** + * Ends one lease's capture within the budget a record is given everywhere else, so a HAR write that stops + * answering cannot hold a disposal that has already ended, or owes nothing, for longer than reporting is worth. + */ +async function awaitEndedCapture( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, +): Promise { + const { awaitLimrunWithinBudget } = await import('./lifecycle-budget.ts'); + const ended = await awaitLimrunWithinBudget(host.endCapture(owner), CAPTURE_RECORD_BUDGET_MS); + return ended.ok ? ended.value : undefined; +} + +/** What ending one instance came to: whether Limrun confirmed it gone, and what it said otherwise. */ +type LimrunInstanceEnd = Readonly<{ + cleanup: LimrunDisposalOutcome['cleanup']; + failure: string | undefined; + cause: unknown; +}>; + +/** + * Ends one owner's instance within a deadline of its own. An attached instance is the caller's own and is never + * deleted here; a delete Limrun already confirmed answers for itself; and a delete that never answers keeps the + * lease held and stays observed, because what it confirms late is what frees the lease. Ownership is read from + * the owner and not from an argument, so no route that reaches an instance can default an attached one to + * being ours to delete. + */ +async function endInstance( + host: LimrunDisposalHost, + owner: LimrunLeaseOwner, + instanceId: string, +): Promise { + const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); + if (owner.deletedInstanceId === instanceId) + return { cleanup: 'confirmed', failure: undefined, cause: undefined }; + if ((owner.session?.ownership ?? 'created') === 'attached') { + return { cleanup: 'attached', failure: undefined, cause: undefined }; + } + const requested = + owner.pendingDelete?.instanceId === instanceId + ? // This owner already has a delete with Limrun for this instance, however far from answering. One + // instance is deleted by one request: a second route that owes its end waits on that request. + owner.pendingDelete.requested + : host.deleteInstance(owner.platform, instanceId); + if (owner.pendingDelete?.instanceId !== instanceId) { + const inFlight = { instanceId, requested }; + owner.pendingDelete = inFlight; + // A delete that answered leaves nothing for a later route to join: a confirmed one is recorded as + // deleted, and a refused one is that later route's to retry. + const forgotten = (): void => { + if (owner.pendingDelete === inFlight) owner.pendingDelete = undefined; + }; + void requested.then(forgotten, forgotten); + } + const deleted = await awaitLimrunWithinBudget(requested, DELETE_ANSWER_BUDGET_MS); + if (!deleted.ok) { + // Both branches of the request are answered for here: this continuation outlives its budget, and the + // route that started it has stopped waiting, so nothing it reports later may reach the process as an + // unhandled rejection. + void requested.then( + (answer) => { + if (answer.failure === undefined) confirmLateDelete(owner, instanceId); + }, + () => undefined, + ); + return { + cleanup: 'uncertain', + failure: + 'rejected' in deleted + ? `Delete of instance '${instanceId}' could not be attempted` + : workOutstandingFailure(`Delete of instance '${instanceId}'`, DELETE_ANSWER_BUDGET_MS), + cause: undefined, + }; + } + if (deleted.value.failure === undefined) owner.deletedInstanceId = instanceId; + return { + cleanup: deleted.value.failure === undefined ? 'confirmed' : 'uncertain', + failure: deleted.value.failure, + cause: deleted.value.cause, + }; +} + +/** + * Records a delete that confirmed after its disposal stopped waiting for it. The lease stays held for its next + * route, which is the one that has to tell its caller the instance is gone; this only makes sure that route + * never asks Limrun to delete the same instance twice. + */ +function confirmLateDelete(owner: LimrunLeaseOwner, instanceId: string): void { + owner.deletedInstanceId = instanceId; +} + +/** + * Ends the resources one owner holds besides its instance: the keep-alive beat that would otherwise ping a + * deleted instance forever, the instance connection, and the Android ADB tunnel with its reverse mappings. One + * release answers for one owner, whichever route asked first, so a session that arrives after this started is + * still released by it rather than by a second pass nobody is promised. + */ +function releaseOwnerResources(owner: LimrunLeaseOwner): Promise { + owner.localRelease ??= releaseLocalResources(owner); + return owner.localRelease; +} + +async function releaseLocalResources(owner: LimrunLeaseOwner): Promise { + if (owner.keepAlive) clearInterval(owner.keepAlive); + owner.keepAlive = undefined; + // The session is what holds a host endpoint open. The owner has kept it from the moment it was built, whether + // or not the allocation that built it answered, so a create this disposal stopped waiting for is awaited here + // rather than passed over: the session it eventually builds has nowhere else to be disconnected. + const session = owner.session ?? (await sessionTheCreateBuilt(owner)); + if (session === undefined) return; + const { awaitLimrunWithinBudget } = await import('./lifecycle-budget.ts'); + const tunnel = + session.platform === 'android' ? cleanupLimrunAndroidAdbTunnel(session) : Promise.resolve(); + // Disconnect closes the ADB endpoint: reverse cleanup gets its window, but cannot postpone disconnect + // past that window. Both promises are observed before either can settle. + const disconnect = awaitLimrunWithinBudget(tunnel, LOCAL_TEARDOWN_BUDGET_MS).then(() => { + session.client.disconnect(); + }); + const results = await Promise.allSettled([tunnel, disconnect]); + const refused = results.find((result) => result.status === 'rejected'); + if (refused?.status === 'rejected') throw refused.reason; +} + +/** The session a create answered for, once it answers, for a release that got there first. */ +async function sessionTheCreateBuilt( + owner: LimrunLeaseOwner, +): Promise { + await owner.creation.catch(() => undefined); + return owner.session; +} + +/** + * Reports the local teardown a disposal had to stop waiting for. A step that refused and a step that never + * answered are both reported as what they are: neither is something this runtime can still fix, and the + * delete that ran beside it has already been answered. + */ +async function awaitLocalTeardown(local: Promise): Promise { + const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); + const ended = await awaitLimrunWithinBudget(local, LOCAL_TEARDOWN_BUDGET_MS); + if (ended.ok) return undefined; + return 'rejected' in ended + ? `Local teardown refused to finish: ${errorMessage(ended.error)}` + : workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS); +} + +/** What a release answers for one lease: the instance it ended, with what it left and wrote. */ +type LimrunReleaseRecord = Record; + +/** + * Ends one lease from the disposal its runtime already started for it, or from what the lease's own labels + * name when the runtime no longer holds an owner for it. A delete Limrun did not confirm keeps the lease held: + * the release is refused, and the next release, recovery or shutdown retries the same instance rather than + * reporting an instance released. + * + * The disposal is handed in rather than started here because the caller took the lease in the same step the + * release was asked for, and no loading in between may let the allocation still waiting on its create answer + * first. + */ +export async function releaseLimrunLease( + limrun: Limrun | undefined, + lease: DeviceLease, + disposal: Promise | undefined, +): Promise { + // No disposal means this runtime holds nothing for the lease: it was never allocated here, or its disposal + // confirmed and gave it back. What the lease has under its labels is the recovery route's. + if (disposal === undefined) return await releaseLabeledInstances(limrun, lease); + const outcome = await disposal; + if (outcome.cleanup === 'uncertain') throw releaseLeftInstanceRunning(lease.leaseId, outcome); + if (outcome.instanceId === undefined) return undefined; + const { networkArtifacts } = await import('./network-har.ts'); + return { + limrunInstanceId: outcome.instanceId, + // The instance is gone, so the lease is released; what the release stopped waiting for is a warning the + // caller still has to see, because it is a host resource this process left behind. + ...(outcome.failure === undefined + ? {} + : { + warnings: [ + { + code: 'LIMRUN_TEARDOWN_INCOMPLETE', + message: outcome.failure, + limrunInstanceId: outcome.instanceId, + }, + ], + }), + ...(outcome.capture ? { cloudArtifacts: networkArtifacts(outcome.capture) } : {}), + }; +} + +/** + * Ends every instance a lease's own labels name, for a lease this runtime no longer holds an owner for. + * Every delete goes through the same helper every other route deletes with, so a 404 confirms here exactly + * as it does there; a delete the helper could not confirm refuses with the typed unconfirmed-delete error, + * naming every instance the labels carried and which of them are already confirmed gone. The refusal keeps + * the lease held: the daemon's expiry sweep is the retry, and nothing else knows these ids. + */ +async function releaseLabeledInstances( + limrun: Limrun | undefined, + lease: DeviceLease, +): Promise { + const platform = platformForLimrunLeaseBackend(lease.backend); + if (!platform || !limrun) return undefined; + const labelSelector = `provider=${LIMRUN_PROVIDER},leaseId=${lease.leaseId}`; + const instances = + platform === 'ios' + ? await limrun.iosInstances.list({ labelSelector }) + : await limrun.androidInstances.list({ labelSelector }); + const instanceIds = instances.getPaginatedItems().map((instance) => instance.metadata.id); + const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts'); + const confirmed: string[] = []; + for (const instanceId of instanceIds) { + const answer = await awaitLimrunWithinBudget( + deleteLimrunInstance(limrun, platform, instanceId), + DELETE_ANSWER_BUDGET_MS, + ); + const deleted = answer.ok + ? answer.value + : { + failure: workOutstandingFailure( + `Delete of instance '${instanceId}'`, + DELETE_ANSWER_BUDGET_MS, + ), + cause: undefined, + }; + if (deleted.failure !== undefined) { + throwUnconfirmedDelete( + `Recovery could not confirm instance '${instanceId}' of lease '${lease.leaseId}' deleted`, + { limrunInstanceId: instanceId, limrunInstanceIds: instanceIds }, + deleted.failure, + recoveryHint(instanceIds, confirmed), + deleted.cause, + ); + } + confirmed.push(instanceId); + } + if (instanceIds.length === 0) return undefined; + return { + limrunInstanceId: instanceIds[0], + limrunInstanceIds: instanceIds, + limrunInstanceCount: instanceIds.length, + }; +} + +/** What the recovery refusal tells the operator to name to Limrun: the instance left, and those gone. */ +function recoveryHint(instanceIds: readonly string[], confirmed: readonly string[]): string { + const outstanding = instanceIds.filter((id) => !confirmed.includes(id)); + const request = `Ask Limrun to delete: ${outstanding.join(', ')}.`; + if (confirmed.length === 0) return request; + const gone = `${confirmed.length > 1 ? 'Instances' : 'Instance'} ${confirmed.join(', ')} ${confirmed.length > 1 ? 'were' : 'was'} confirmed deleted.`; + return `${request} ${gone}`; +} + +/** + * What a release reports when Limrun did not confirm the delete, so the lease is still held. An unended lease + * whose create never named an instance is answered for by the lease alone, because the operator has nothing to + * name to Limrun yet. + */ +function releaseLeftInstanceRunning(leaseId: string, outcome: LimrunDisposalOutcome): never { + const { instanceId } = outcome; + return throwUnconfirmedDelete( + `Limrun instance for lease '${leaseId}' was not confirmed deleted`, + instanceId === undefined ? {} : { limrunInstanceId: instanceId }, + outcome.failure, + instanceId === undefined + ? `Release lease '${leaseId}' again once its instance create names the instance it rented.` + : `Release lease '${leaseId}' again, or ask Limrun to delete instance '${instanceId}'.`, + outcome.cause, + ); +} + +/** + * The refusal an allocation gets when a teardown took its lease while the create ran. The caller passes the + * disposal's outcome rather than reading the owner, so this answers with what that teardown came to — + * including a delete Limrun did not confirm, which the refused caller is the last route left to hear about. + */ +export function allocationRefusedForTake( + lease: DeviceLease, + by: LimrunTeardownRoute, + outcome: LimrunDisposalOutcome, +): never { + const { instanceId } = outcome; + // The hint says what the disposal actually confirmed: an unconfirmed delete leaves the instance possibly + // still billing, and that caller is the last route left that will hear about it. + const ended = + instanceId === undefined + ? undefined + : outcome.cleanup === 'uncertain' + ? `The ${by} of this lease could not confirm instance '${instanceId}' deleted. Release the lease again, or ask Limrun to delete it.` + : outcome.cleanup === 'attached' + ? `Instance '${instanceId}' is the caller's own attached instance and was left running. Open a new lease.` + : `Instance '${instanceId}' was ended by the ${by} of this lease. Open a new lease.`; + throw new AppError( + 'DEVICE_IN_USE', + `Limrun lease '${lease.leaseId}' was released while its instance was being prepared`, + { + reason: 'limrun_lease_released_mid_allocation', + ...(instanceId === undefined + ? { hint: `Instance create for lease '${lease.leaseId}' may still rent one. Check Limrun.` } + : { limrunInstanceId: instanceId, ...(ended === undefined ? {} : { hint: ended }) }), + ...(outcome.cleanup === 'uncertain' + ? { retriable: true, ...(outcome.failure === undefined ? {} : { cause: outcome.failure }) } + : {}), + }, + outcome.cause, + ); +} + +/** + * What an allocation reports when the create it waited on failed and left an instance running. The hint names + * the instance when the create got as far as renting one, and the lease when it did not. + */ +export function createLeftInstanceRunning( + lease: DeviceLease, + outcome: LimrunDisposalOutcome, + createError: unknown, +): never { + const { instanceId } = outcome; + return throwUnconfirmedDelete( + `Limrun lease '${lease.leaseId}' could not be allocated, and its instance could not be deleted`, + instanceId === undefined ? {} : { limrunInstanceId: instanceId }, + // Both refusals belong to this one answer: why the allocation failed, and why its instance is still + // billed. The delete's text leads the report because it is what the operator still has to settle. + joinFailureText( + outcome.failure, + createError === undefined ? undefined : `create failed: ${errorMessage(createError)}`, + ), + instanceId === undefined + ? `Ask Limrun what lease '${lease.leaseId}' rented — this runtime could not name the instance.` + : `Ask Limrun to delete instance '${instanceId}' — this runtime could not confirm the delete.`, + outcome.cause ?? createError, + ); +} + +/** Two reasons one report carries, delete first, when both exist. */ +function joinFailureText( + first: string | undefined, + second: string | undefined, +): string | undefined { + if (first === undefined) return second; + return second === undefined ? first : `${first}; ${second}`; +} + +/** + * The refusal every route that could not confirm an instance delete raises. The instance ids go into details, + * why the delete failed goes into `details.cause` as text, and the delete's own error goes where a cause + * renders — the error's own cause. Each caller names its instances the way its own report reads: a single + * lease its one id, a close the list of what it left. + */ +function throwUnconfirmedDelete( + message: string, + instanceDetails: Readonly>, + failure: string | undefined, + hint: string, + cause: unknown, +): never { + throw new AppError( + 'COMMAND_FAILED', + message, + { + reason: 'limrun_instance_delete_unconfirmed', + retriable: true, + ...instanceDetails, + ...(failure === undefined ? {} : { cause: failure }), + hint, + }, + cause, + ); +} + +/** + * What a daemon close reports for every lease it could not confirm ended. A lease whose create never named an + * instance is counted the same as one whose delete was refused: this process moved on without stopping it, and + * the operator needs both the count and the reason. + */ +export function cleanupLeftInstancesRunning(outcomes: readonly LimrunDisposalOutcome[]): never { + const known = outcomes.map((outcome) => outcome.instanceId).filter((id) => id !== undefined); + const plural = outcomes.length === 1 ? '' : 's'; + // Why every lease this close could not end, not only the first: each failure is a separate thing the + // operator may have to name to Limrun. + return throwUnconfirmedDelete( + `Limrun cleanup left ${String(outcomes.length)} lease${plural} unended, ${String(known.length)} with a known instance`, + { limrunInstanceIds: known }, + joinFailureReasons(outcomes), + known.length === 0 ? unendedLeaseHint(outcomes) : `Ask Limrun to delete: ${known.join(', ')}.`, + outcomes.find((outcome) => outcome.cause !== undefined)?.cause, + ); +} + +/** Every lease this close could not end, with why, joined for one report. */ +function joinFailureReasons(outcomes: readonly LimrunDisposalOutcome[]): string | undefined { + if (outcomes.every((outcome) => outcome.failure === undefined)) return undefined; + return outcomes.map((outcome) => outcome.failure ?? 'an unconfirmed delete').join('; '); +} + +/** What a close says when the work it could not finish has no instance to name yet: the lease and the work. */ +function unendedLeaseHint(outcomes: readonly LimrunDisposalOutcome[]): string { + return `Limrun may still bill work this runtime could not name: ${outcomes + .map((outcome) => outcome.failure ?? 'an unconfirmed delete') + .join('; ')}.`; +} diff --git a/packages/provider-limrun/src/lifecycle-budget.ts b/packages/provider-limrun/src/lifecycle-budget.ts new file mode 100644 index 0000000000..bef349acae --- /dev/null +++ b/packages/provider-limrun/src/lifecycle-budget.ts @@ -0,0 +1,75 @@ +/** + * One bounded wait, for the work a lease's lifecycle awaits but does not own: a create that may be + * retrying, a session connection still handshaking, an ADB `connect` the SDK started without a deadline, + * or a capture still opening its destination tunnel. Every one of these can answer later than a release + * or a daemon close may wait, and none of them may keep an instance billed while its caller is unresolved. + * + * Expiry is a reported outcome rather than a thrown one, and the work being waited on is never cancelled: + * it runs to its end, and its owner disposes what it eventually makes. + */ + +/** + * How the wait ended, kept so a caller states which work was still outstanding rather than guessing. + * A work item that refused is an outcome of its own: every wait this type answers for is reporting, + * cleanup, or a handshake somebody else can still act on, so a refusal is never this route's failure + * to re-raise — it maps to this variant and the caller says what it could not use. + */ +export type LimrunBudgetOutcome = + | Readonly<{ ok: true; value: T }> + | Readonly<{ ok: false; expired: true }> + | Readonly<{ ok: false; rejected: true; error: unknown }>; + +/** + * Waits for `work` within `budgetMs`, unref'd so a daemon close is never held open by the timer alone. + * Both of the work's answers are mapped to outcomes in this step, so a work item that rejects can + * never reach the process as an unhandled rejection, whether it answers inside the budget or after it. + */ +export async function awaitLimrunWithinBudget( + work: Promise, + budgetMs: number, +): Promise> { + const settled = Promise.resolve(work).then( + (value): LimrunBudgetOutcome => ({ ok: true, value }), + (error: unknown): LimrunBudgetOutcome => ({ ok: false, rejected: true, error }), + ); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + settled, + new Promise>((expired) => { + timer = setTimeout(() => expired({ ok: false, expired: true }), budgetMs); + timer.unref?.(); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +/** + * Keeps work that was started without being awaited yet observed for as long as that lasts: the handler + * attached here is what stops a refusal that arrives before the awaiting step from reaching the process as + * an unhandled rejection. The same promise comes back, so the route that starts the work is still the one + * that awaits what it answers with. + */ +export function keepLimrunObserved(work: Promise): Promise { + void work.catch(() => undefined); + return work; +} + +/** + * Why an outcome is not confirmed: the work this runtime stopped waiting for. It is reported rather than + * dropped, because whoever reads it has to say whether the resource may still be running. + */ +export function workOutstandingFailure(work: string, budgetMs: number): string { + return `${work} had not answered within ${String(budgetMs / 1000)} s`; +} + +/** Joins why a delete failed with the local work that outlived its budget, delete first. */ +export function joinOutcomeFailures( + deleteFailure: string | undefined, + outstanding: string | undefined, +): string | undefined { + if (deleteFailure === undefined) return outstanding; + return outstanding === undefined ? deleteFailure : `${deleteFailure}; ${outstanding}`; +} diff --git a/packages/provider-limrun/src/lifecycle.fixtures.ts b/packages/provider-limrun/src/lifecycle.fixtures.ts new file mode 100644 index 0000000000..5279bf5c49 --- /dev/null +++ b/packages/provider-limrun/src/lifecycle.fixtures.ts @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; + +/** + * Collects the rejections the process would die on: the daemon's failure mode for one is to take every other + * session down with it. The check yields to a macrotask first, because Node reports a rejection only after the + * current microtask drain — asserting before that yield, or removing the listener without one, could pass a + * leaked rejection unseen. + */ +export function watchUnhandledRejections(): { + assertIsSilent: () => Promise; + stop: () => void; +} { + const seen: unknown[] = []; + const watch = (reason: unknown): void => { + seen.push(reason); + }; + process.on('unhandledRejection', watch); + return { + assertIsSilent: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + if (seen.length > 0) assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`); + }, + stop: () => process.off('unhandledRejection', watch), + }; +} diff --git a/packages/provider-limrun/src/network-har.ts b/packages/provider-limrun/src/network-har.ts index cde0178378..d470025aef 100644 --- a/packages/provider-limrun/src/network-har.ts +++ b/packages/provider-limrun/src/network-har.ts @@ -4,8 +4,10 @@ import { DestinationTunnelSessionError } from '@limrun/api/destination-tunnel-di import type { DestinationTunnelHAREntry } from '@limrun/api/destination-tunnel-har'; import type { DestinationTunnelInspectionEvent } from '@limrun/api/destination-tunnel-inspection'; import type { DestinationTcpTunnel } from '@limrun/api/destination-tunnel-dialer'; +import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import fs from 'node:fs'; import path from 'node:path'; +import { LIMRUN_PROVIDER } from './device.ts'; /** * The Limrun network capture: one destination tunnel with inspection. Completed requests stay in @@ -148,7 +150,7 @@ export async function startLimrunHarCapture( } /** Writes the entries as a HAR 1.2 file that only the daemon's user can read. */ -export async function writeLimrunHar( +async function writeLimrunHar( harPath: string, entries: readonly DestinationTunnelHAREntry[], ): Promise { @@ -182,3 +184,103 @@ function defaultPort(url: URL | null): number | undefined { if (url.port) return Number(url.port); return url.protocol === 'https:' ? 443 : url.protocol === 'http:' ? 80 : undefined; } + +/** What one lease's capture came to once it was written or refused. */ +export type SettledNetworkCapture = ( + | Readonly<{ phase: 'written'; harPath: string; failures: LimrunNetworkFailures }> + | Readonly<{ + phase: 'unavailable'; + reason: 'capture-dir-unset' | 'tunnel-in-use' | 'tunnel-start-failed' | 'har-write-failed'; + message?: string; + }> +) & + Readonly<{ closeFailure?: string }>; + +/** What one lease has in capture terms: what it is capturing now, or what its capture came to. */ +export type NetworkCapture = + | SettledNetworkCapture + | Readonly<{ phase: 'capturing'; harPath: string; capture: LimrunHarCapture }>; + +/** Where one lease's HAR goes, so the lease id is the only name a capture's file needs. */ +export function limrunHarPath(harDir: string, leaseId: string): string { + return path.join(harDir, `${leaseId}.har`); +} + +/** + * Writes one capture's HAR and answers with the record it came to. The capture itself is left alone, so an + * artifacts read can snapshot a live lease's traffic without ending the capture. A settled capture is + * already its own record. + */ +export async function writeCaptureRecord(capture: NetworkCapture): Promise { + if (capture.phase !== 'capturing') return capture; + const failures = await capture.capture.failures(); + try { + await writeLimrunHar(capture.harPath, capture.capture.entries()); + return { phase: 'written', harPath: capture.harPath, failures }; + } catch (error) { + return { + phase: 'unavailable', + reason: 'har-write-failed', + message: error instanceof Error ? error.message : String(error), + }; + } +} + +const NETWORK_ARTIFACT_NAME = 'Network traffic (HAR on the daemon host)'; + +/** What `artifacts` answers for one lease's capture record. */ +export function networkArtifacts(capture: SettledNetworkCapture): CloudArtifactsResult { + const artifacts = captureRecordArtifacts(capture); + if (capture.closeFailure === undefined) return artifacts; + return { + ...artifacts, + message: [artifacts.message, `Capture close failed: ${capture.closeFailure}`] + .filter((message) => message !== undefined) + .join('; '), + }; +} + +function captureRecordArtifacts(capture: SettledNetworkCapture): CloudArtifactsResult { + if (capture.phase === 'unavailable') { + return { + provider: LIMRUN_PROVIDER, + status: 'unavailable', + message: `Network capture unavailable (${capture.reason})${capture.message ? `: ${capture.message}` : ''}`, + cloudArtifacts: [ + { + provider: LIMRUN_PROVIDER, + kind: 'network', + name: NETWORK_ARTIFACT_NAME, + availability: 'unavailable', + metadata: { reason: capture.reason }, + }, + ], + }; + } + const { count, last, lastDialFailure, truncated } = capture.failures; + const summary = [ + `Network capture failures: ${count}`, + ...(last + ? [`last ${last.kind}${last.host ? ` ${last.host}${last.port ? `:${last.port}` : ''}` : ''}`] + : []), + ...(lastDialFailure ? [`last dial failure ${lastDialFailure}`] : []), + ...(truncated > 0 ? [`${truncated} oldest requests left the HAR`] : []), + ].join('; '); + return { + provider: LIMRUN_PROVIDER, + status: 'ready', + ...(count > 0 || lastDialFailure || truncated > 0 ? { message: summary } : {}), + cloudArtifacts: [ + { + provider: LIMRUN_PROVIDER, + kind: 'network', + name: NETWORK_ARTIFACT_NAME, + url: capture.harPath, + extension: 'har', + contentType: 'application/json', + availability: 'ready', + metadata: { failures: capture.failures }, + }, + ], + }; +} diff --git a/packages/provider-limrun/src/runtime-dependencies.fixtures.ts b/packages/provider-limrun/src/runtime-dependencies.fixtures.ts new file mode 100644 index 0000000000..82323635f6 --- /dev/null +++ b/packages/provider-limrun/src/runtime-dependencies.fixtures.ts @@ -0,0 +1,63 @@ +import type { AppsFilter } from '@agent-device/contracts/device'; +import type { Interactor } from '@agent-device/contracts/interactor-types'; +import { AppError } from '@agent-device/kernel/errors'; +import { + androidAdbHostTarget, + androidAdbInvocation, + androidAdbSerialTarget, +} from '@agent-device/platform-android/mechanics'; +import type { LimrunAdbExecutor, LimrunRuntimeDependencies } from './runtime-dependencies.ts'; + +/** What no case here exercises: the keyboard is never visible and nothing owns input. */ +const NO_KEYBOARD = Object.freeze({ + visible: false as const, + inputOwner: 'unknown' as const, +}); + +/** + * The host and platform adapters the runtime tests hand `createLimrunRuntime`: every one of them answers a + * fixed or a pass-through value, so a case that reaches one of them has left the path it means to drive. The + * pass-throughs are the ones that shape what they were handed into an invocation or an error, and they shape + * it the same way every time. A case with a reading of its own spreads this object and replaces the member it + * drives. + */ +export function stubLimrunRuntimeDependencies(): LimrunRuntimeDependencies { + return { + clientVersion: 'test-version', + android: { + createInteractor: () => ({}) as Interactor, + createPortReverse: async () => ({ + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + list: async () => [], + }), + inferAppName: async () => 'Example', + listApps: async (_adb: LimrunAdbExecutor, _filter: AppsFilter) => [], + getForegroundApp: async () => undefined, + getKeyboardState: async () => ({ ...NO_KEYBOARD }), + dismissKeyboard: async () => ({ + ...NO_KEYBOARD, + attempts: 0, + wasVisible: false, + dismissed: false, + }), + readLogs: async () => '', + forceStopApp: async () => {}, + deviceAdbInvocation: (serial, command) => + androidAdbInvocation(androidAdbSerialTarget(serial), command), + hostAdbInvocation: (command) => androidAdbInvocation(androidAdbHostTarget(), command), + adbError: async (message) => new AppError('COMMAND_FAILED', message), + }, + host: { + runAdb: async () => ({ stdout: '', stderr: '', exitCode: 0 }), + archiveDirectory: async () => undefined, + downloadFile: async () => undefined, + }, + ios: { + applySimctlSetting: async () => undefined, + resolveAppAlias: async (app) => app, + readBundleAppName: async () => undefined, + }, + }; +} diff --git a/packages/provider-limrun/src/runtime.test.ts b/packages/provider-limrun/src/runtime.test.ts index 108d4946f8..021c06f341 100644 --- a/packages/provider-limrun/src/runtime.test.ts +++ b/packages/provider-limrun/src/runtime.test.ts @@ -7,15 +7,15 @@ import type { Interactor } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformRequestScope } from '@agent-device/contracts/platform-runtime-host'; -import { - androidAdbHostTarget, - androidAdbInvocation, - androidAdbSerialTarget, -} from '@agent-device/platform-android/mechanics'; import { createLimrunRuntime } from './runtime.ts'; +import { stubLimrunRuntimeDependencies } from './runtime-dependencies.fixtures.ts'; import { unusedLimrunRuntimeHost } from './runtime.fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; -import type { LimrunAdbExecutor, LimrunRuntimeDependencies } from './runtime-dependencies.ts'; +import type { + LimrunAdbExecutor, + LimrunAdbProvider, + LimrunRuntimeDependencies, +} from './runtime-dependencies.ts'; /** * The provider's platform module is the daemon's real load path: `createLimrunRuntime(..., @@ -32,50 +32,519 @@ const state = vi.hoisted(() => ({ tunnelClose: vi.fn(), captureTunnelClose: vi.fn(), onInspectionEvent: undefined as ((event: unknown) => void) | undefined, + androidCreate: vi.fn(async () => ({ + metadata: { id: 'android-instance-1' }, + status: { + token: 'instance-token', + apiUrl: 'https://android.example', + adbWebSocketUrl: 'wss://adb.example', + }, + })), + androidDelete: vi.fn(async (_instanceId: string) => { + await state.deleteGate; + if (state.deleteFails) throw new Error('Limrun refused to delete the instance'); + }), + /** Made to reject by a case whose instance delete is refused, the one failure a lease may not swallow. */ + deleteFails: false, + /** Held by a case whose instance delete must answer later than its own route's deadline. */ + deleteGate: undefined as Promise | undefined, + keepAlive: vi.fn(), + /** Instance-client connections the provider made, so one lease can be shown to make one. */ + connectCount: 0, + /** Held by a case that keeps a create in flight after its instance was rented. */ + createGate: undefined as Promise | undefined, + /** Held by a case that keeps a destination tunnel or ADB tunnel in flight. */ + tunnelGate: undefined as Promise | undefined, + /** Made to throw by a case whose session connection fails after its instance was rented. */ + connectFails: false, + /** Uploaded apps the catalog answers for a case that asks for an app to preinstall. */ + uploadedApps: [] as unknown[], + /** What the device reports installed, for a case whose app identifier cannot be resolved. */ + installedApps: [] as Array<{ id: string; name?: string }>, + /** Destination tunnels and ADB tunnels this runtime asked the SDK to open. */ + captureStarts: 0, + adbTunnelStarts: 0, + /** The ADB provider last handed to an interactor: the handle kept for a lease that has since ended. */ + heldAdbProvider: undefined as LimrunAdbProvider | undefined, })); vi.mock('@limrun/api', () => ({ default: class MockLimrun { readonly iosInstances = { create: vi.fn(), list: vi.fn(), delete: vi.fn() }; readonly androidInstances = { - create: vi.fn(async () => ({ - metadata: { id: 'android-instance-1' }, - status: { - token: 'instance-token', - apiUrl: 'https://android.example', - adbWebSocketUrl: 'wss://adb.example', - }, - })), - list: vi.fn(), - delete: vi.fn(async () => undefined), + create: async () => { + const instance = await state.androidCreate(); + await state.createGate; + return instance; + }, + // Expired-lease recovery finds nothing under this lease's labels, so a release of a lease the + // runtime lost answers nothing rather than reaching an unstubbed SDK call. + list: vi.fn(async () => ({ getPaginatedItems: () => [] })), + delete: state.androidDelete, + }; + readonly assets = { + getOrUpload: vi.fn(), + list: vi.fn(async () => state.uploadedApps), }; - readonly assets = { getOrUpload: vi.fn(), list: vi.fn() }; }, })); vi.mock('@limrun/api/instance-client', () => ({ - createInstanceClient: vi.fn(async () => ({ - disconnect: state.disconnect, - keepAlive: vi.fn(), - openUrl: vi.fn(), - sendAsset: vi.fn(), - pressKey: vi.fn(), - setText: vi.fn(), - tap: vi.fn(), - startAdbTunnel: vi.fn(async () => ({ - address: { address: '127.0.0.1', port: 62_001 }, - close: state.tunnelClose, - })), - startTunnel: vi.fn(async (options: { onInspectionEvent: (event: unknown) => void }) => { - state.onInspectionEvent = options.onInspectionEvent; - return { tunnelId: 't1', selectors: [], close: state.captureTunnelClose }; - }), - getTunnelStatus: vi.fn(async () => ({ - lastDialFailure: { tunnelId: 't1', reason: 'dns_not_found' }, - })), - })), + createInstanceClient: vi.fn(async () => { + state.connectCount += 1; + if (state.connectFails) throw new Error('instance connection refused'); + return { + disconnect: state.disconnect, + keepAlive: state.keepAlive, + openUrl: vi.fn(), + sendAsset: vi.fn(), + pressKey: vi.fn(), + setText: vi.fn(), + tap: vi.fn(), + startAdbTunnel: vi.fn(async () => { + state.adbTunnelStarts += 1; + await state.tunnelGate; + return { address: { address: '127.0.0.1', port: 62_001 }, close: state.tunnelClose }; + }), + startTunnel: vi.fn(async (options: { onInspectionEvent: (event: unknown) => void }) => { + state.captureStarts += 1; + state.onInspectionEvent = options.onInspectionEvent; + await state.tunnelGate; + return { tunnelId: 't1', selectors: [], close: state.captureTunnelClose }; + }), + getTunnelStatus: vi.fn(async () => ({ + lastDialFailure: { tunnelId: 't1', reason: 'dns_not_found' }, + })), + }; + }), })); +/** + * One lease has one set of resources, however many routes arrive at it: a leased instance cannot be rented + * twice, left untracked by a release or a daemon close that raced its create, or kept alive after it is + * gone. The mock `state` is shared by this file, so each case measures from its own baseline. + */ + +test('two allocations of one lease rent one instance and start one keep-alive timer', async () => { + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const { connects, creates, deletes, disconnects } = limrunActivity(); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', keepAlive: true }, + testDependencies(), + ); + let started: ReturnType | undefined; + try { + // The second call arrives while the first one's create is still running. + const first = runtime.leaseLifecycle.allocate?.(androidLease('lease-race')); + const second = runtime.leaseLifecycle.allocate?.(androidLease('lease-race')); + await waitForCreate(creates + 1); + started = vi.spyOn(globalThis, 'setInterval'); + reportCreate(); + + assert.equal((await first)?.limrunInstanceId, 'android-instance-1'); + assert.equal((await second)?.limrunInstanceId, 'android-instance-1'); + assert.equal(state.connectCount - connects, 1); + assert.equal(started.mock.calls.length, 1); + } finally { + started?.mockRestore(); + // Whatever this case asserted, the create is answered before it is torn down, so a failed assertion + // surfaces as a failure rather than as a shutdown that waits out its create budget. + reportCreate(); + await runtime.shutdown(); + state.createGate = undefined; + } + // The second allocation is not an orphaned session behind the first: one create, one connection, one delete. + const activity = limrunActivity(); + assert.equal(activity.creates - creates, 1); + assert.equal(activity.connects - connects, 1); + assert.deepEqual( + state.androidDelete.mock.calls.slice(deletes).map(([instanceId]) => instanceId), + ['android-instance-1'], + ); + assert.equal(activity.disconnects - disconnects, 1); +}); + +test('a release that arrives before the create answered ends the instance that create made', async () => { + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const { creates, deletes, disconnects } = limrunActivity(); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', keepAlive: true }, + testDependencies(), + ); + const lease = androidLease('lease-race-release'); + let started: ReturnType | undefined; + try { + const allocating = runtime.leaseLifecycle.allocate?.(lease); + const refused = refusalOf(allocating); + await waitForCreate(creates + 1); + started = vi.spyOn(globalThis, 'setInterval'); + + const released = runtime.leaseLifecycle.release?.(lease); + reportCreate(); + + // The release cannot answer before the instance it is waiting for exists, so it names what it ended. + assert.deepEqual(await released, { limrunInstanceId: 'android-instance-1' }); + // The allocation whose instance was taken from it is refused rather than handed a dead session. + const error = await refused; + assert.ok( + error instanceof AppError && + error.details?.reason === 'limrun_lease_released_mid_allocation' && + error.details.limrunInstanceId === 'android-instance-1', + `expected the allocation to be refused for the release, got ${String(error)}`, + ); + assert.equal(started.mock.calls.length, 0); + assert.deepEqual( + state.androidDelete.mock.calls.slice(deletes).map(([instanceId]) => instanceId), + ['android-instance-1'], + ); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + } finally { + started?.mockRestore(); + reportCreate(); + await runtime.shutdown(); + state.createGate = undefined; + } + // One instance, so one delete: the allocation's own refusal does not end a second one. + assert.equal(state.androidDelete.mock.calls.length - deletes, 1); +}); + +test('a shutdown while the create runs ends the instance that create made', async () => { + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const { creates, deletes } = limrunActivity(); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', keepAlive: true }, + testDependencies(), + ); + let started: ReturnType | undefined; + let closing: Promise | undefined; + try { + const allocating = runtime.leaseLifecycle.allocate?.(androidLease('lease-race-shutdown')); + const refused = refusalOf(allocating); + await waitForCreate(creates + 1); + started = vi.spyOn(globalThis, 'setInterval'); + + closing = runtime.shutdown(); + reportCreate(); + await closing; + + const error = await refused; + assert.ok( + error instanceof AppError && error.details?.reason === 'limrun_lease_released_mid_allocation', + `expected the allocation to be refused for the shutdown, got ${String(error)}`, + ); + assert.equal(started.mock.calls.length, 0); + // Nothing may allocate into a runtime that was told to end every instance it holds. + await assert.rejects( + async () => await runtime.leaseLifecycle.allocate?.(androidLease('lease-after-shutdown')), + (error) => error instanceof AppError && error.details?.reason === 'limrun_runtime_shut_down', + ); + assert.deepEqual( + state.androidDelete.mock.calls.slice(deletes).map(([instanceId]) => instanceId), + ['android-instance-1'], + ); + } finally { + started?.mockRestore(); + reportCreate(); + state.createGate = undefined; + // The close this case raced is also its teardown; a failed assertion before it still ends the runtime. + await (closing ?? runtime.shutdown()); + } +}); + +/** Lets an allocation reach the instance create, which is a few module loads away, without clock waits. */ +async function waitForCreate(calls: number): Promise { + await vi.waitFor(() => assert.equal(state.androidCreate.mock.calls.length, calls)); +} + +test('a release that arrives before the allocation yields ends the instance its create rents', async () => { + const { creates, deletes } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-release-before-yield'); + try { + // Neither call is awaited between: the release arrives at the allocation's first suspension point. + const allocating = runtime.leaseLifecycle.allocate?.(lease); + const refused = refusalOf(allocating); + const released = runtime.leaseLifecycle.release?.(lease); + + // The release had an owner to take, so it answers for the instance that create went on renting. + assert.deepEqual(await released, { limrunInstanceId: 'android-instance-1' }); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + const error = await refused; + assert.ok( + error instanceof AppError && + error.details?.reason === 'limrun_lease_released_mid_allocation' && + error.details.limrunInstanceId === 'android-instance-1', + `expected the allocation to be refused for the release, got ${String(error)}`, + ); + } finally { + await runtime.shutdown(); + } + // One instance was rented and one was ended; the refused allocation did not rent another. + assert.equal(state.androidCreate.mock.calls.length - creates, 1); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); +}); + +test('the create of a new lease refuses a profile flag it is given no field for', async () => { + const { creates } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + try { + await assert.rejects( + async () => + await runtime.leaseLifecycle.allocate?.(androidLease('lease-refused-flag'), { + flags: { providerNetworkLogs: true, providerOsVersion: '15' }, + }), + (error) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); + } finally { + await runtime.shutdown(); + } + // Nothing was billed before the refused flag was answered for. + assert.equal(state.androidCreate.mock.calls.length - creates, 0); +}); + +test('a create that failed after renting keeps its unconfirmed delete for the next route', async () => { + const { deletes } = limrunActivity(); + state.connectFails = true; + state.deleteFails = true; + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + try { + // The instance was rented before the session connection failed, and Limrun refused this delete. + const error = await refusalOf( + runtime.leaseLifecycle.allocate?.(androidLease('lease-rented-then-failed')), + ); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.ok( + error instanceof AppError && + error.details?.reason === 'limrun_instance_delete_unconfirmed' && + error.details.limrunInstanceId === 'android-instance-1', + `expected the allocation to report the unended instance, got ${String(error)}`, + ); + + // The lease is still held, so the daemon close retries the delete it could not confirm. + state.deleteFails = false; + await runtime.shutdown(); + } finally { + state.connectFails = false; + state.deleteFails = false; + await runtime.shutdown(); + } + // The retried delete ended the same instance the create rented, rather than forgetting it was owed. + assert.deepEqual(deletedInstances(deletes), ['android-instance-1', 'android-instance-1']); +}); + +test('two allocations that ask for traffic logs open one destination tunnel', async () => { + const { captures, captureCloses } = limrunActivity(); + let reportTunnel!: () => void; + state.tunnelGate = new Promise((resolve) => { + reportTunnel = resolve; + }); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', networkHarDir: mkdtempForTestSync('limrun-har-') }, + testDependencies(), + ); + const lease = androidLease('lease-capture-race'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + // Both repeat allocations ask for the capture while the first one's tunnel is still being opened. + const first = refusalOf( + runtime.leaseLifecycle.allocate?.(lease, { + flags: { providerNetworkLogs: true }, + }), + ); + const second = refusalOf( + runtime.leaseLifecycle.allocate?.(lease, { + flags: { providerNetworkLogs: true }, + }), + ); + await waitForNewOpen(captures, () => state.captureStarts); + const pending = await runtime.cloudArtifacts?.listCloudArtifacts?.({ leaseId: lease.leaseId }); + assert.equal(pending?.status, 'pending'); + assert.equal(pending?.provider, 'limrun'); + reportTunnel(); + assert.deepEqual(await Promise.all([first, second]), [undefined, undefined]); + + // The second allocation joined the start in flight rather than opening a tunnel whose `already_active` + // refusal would be reported over the capture that is working. + assert.equal(state.captureStarts - captures, 1); + } finally { + // Opened before teardown for the same reason as the create gate above. + reportTunnel(); + state.tunnelGate = undefined; + await runtime.shutdown(); + } + assert.equal(state.captureTunnelClose.mock.calls.length - captureCloses, 1); +}); + +test('a release that arrives while a tunnel is opening ends the tunnel that answers late', async () => { + const { captures, captureCloses, deletes } = limrunActivity(); + let reportTunnel!: () => void; + state.tunnelGate = new Promise((resolve) => { + reportTunnel = resolve; + }); + const harDir = mkdtempForTestSync('limrun-har-'); + const runtime = createLimrunRuntime( + { apiKey: 'lim_test_key', networkHarDir: harDir }, + testDependencies(), + ); + const lease = androidLease('lease-capture-teardown'); + try { + await runtime.leaseLifecycle.allocate?.(lease); + + // The capture's destination tunnel is still being opened when the release arrives. + const capturing = refusalOf( + runtime.leaseLifecycle.allocate?.(lease, { flags: { providerNetworkLogs: true } }), + ); + await waitForNewOpen(captures, () => state.captureStarts); + const released = runtime.leaseLifecycle.release?.(lease); + reportTunnel(); + + const record = (await released) as Record; + assert.equal(record.limrunInstanceId, 'android-instance-1'); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + // The release wrote the capture's record as well, so `artifacts` can still name the HAR. + assert.equal( + (record.cloudArtifacts as { cloudArtifacts: { url?: string }[] }).cloudArtifacts[0]?.url, + path.join(harDir, `${lease.leaseId}.har`), + ); + const error = await capturing; + assert.ok( + error instanceof AppError && error.details?.reason === 'limrun_lease_released_mid_allocation', + `expected the capture allocation to be refused for the release, got ${String(error)}`, + ); + } finally { + reportTunnel(); + state.tunnelGate = undefined; + await runtime.shutdown(); + } + // The release owned a tunnel whose open had not answered, and ended it where it landed. + assert.equal(state.captureTunnelClose.mock.calls.length - captureCloses, 1); +}); + +test('a release stops waiting on a stuck ADB tunnel, deletes the instance and says what it left', async () => { + await import('./lease-release.ts'); + const { deletes, disconnects, adbTunnels, tunnelCloses } = limrunActivity(); + let reportTunnel!: () => void; + state.tunnelGate = new Promise((resolve) => { + reportTunnel = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-stuck-tunnel'); + let refusedCommand: Promise | undefined; + vi.useFakeTimers(); + try { + const device = await allocateAndroidDevice(runtime, lease.leaseId); + if (!runtime.getInteractor(device)) throw new Error('Limrun runtime must return an interactor'); + // A retained interactor raises this session's ADB tunnel, an `adb connect` the SDK starts without a + // deadline; it is still stuck when the release arrives. + const command = heldAdbCommand(['shell', 'echo', 'held']); + refusedCommand = refusalOf(command); + await vi.advanceTimersByTimeAsync(0); + assert.equal(state.adbTunnelStarts - adbTunnels, 1); + + const released = runtime.leaseLifecycle.release?.(lease); + await vi.advanceTimersByTimeAsync(0); + // The delete does not queue behind the stuck tunnel: it was asked for while that tunnel was still open. + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 0); + + // The release's own budget for the local teardown elapses; the delete it ran beside is already answered. + await vi.advanceTimersByTimeAsync(LOCAL_TEARDOWN_WAIT_MS); + const [warning] = warningsOf(await released); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + assert.equal(warning?.code, 'LIMRUN_TEARDOWN_INCOMPLETE'); + // The report names the work this route stopped waiting for, and the instance it left a host endpoint on. + assert.match(String(warning?.message), /Local teardown/); + assert.equal(warning?.limrunInstanceId, 'android-instance-1'); + // The lease is given back: the stuck tunnel is reported, not held over the next allocation. + await runtime.leaseLifecycle.allocate?.(lease); + + reportTunnel(); + await vi.advanceTimersByTimeAsync(0); + const refused = await refusedCommand; + assert.ok( + refused instanceof AppError && refused.details?.reason === 'limrun_adb_tunnel_cleaned_up', + `expected the retained command to be refused, got ${String(refused)}`, + ); + } finally { + reportTunnel(); + vi.useRealTimers(); + state.tunnelGate = undefined; + await runtime.shutdown(); + await (refusedCommand ?? Promise.resolve()); + } + // The tunnel that answered into a retired session was closed where it landed, and no second one was raised. + assert.equal(state.adbTunnelStarts - adbTunnels, 1); + assert.equal(state.tunnelClose.mock.calls.length - tunnelCloses, 1); +}); + +/** The retained interactor's own ADB handle, driven outside any request this runtime is serving. */ +function heldAdbCommand(args: readonly string[]): Promise { + const provider = state.heldAdbProvider; + if (!provider) throw new Error('Expected the runtime to hand an interactor an ADB provider'); + return provider.exec(args); +} + +function warningsOf(record: Record | undefined): WarningRecord[] { + return Array.isArray(record?.warnings) ? (record.warnings as WarningRecord[]) : []; +} + +type WarningRecord = Readonly<{ + code?: string; + message?: string; + limrunInstanceId?: string; +}>; + +/* Each wait is longer than the budget its case waits out: the local teardown, one delete, one create. */ +const LOCAL_TEARDOWN_WAIT_MS = 60_000; +const DELETE_ANSWER_WAIT_MS = 40_000; +const CREATE_ANSWER_WAIT_MS = 130_000; +/** A profile flag this provider reads no field for, refused for whichever caller carries it. */ +const REFUSED_FLAG = { flags: { providerOsVersion: '15' } }; + +/** + * Why one racing call refused, captured from the moment it starts: two routes can answer in either order, + * and the one that answers first must not leave the other's refusal unobserved. + */ +function refusalOf(call: Promise | undefined): Promise { + return (call ?? Promise.resolve()).then( + () => undefined, + (error: unknown) => error, + ); +} + +/** Where the shared instance-control mocks stand, so a case can measure only what it caused. */ +function limrunActivity() { + return { + creates: state.androidCreate.mock.calls.length, + deletes: state.androidDelete.mock.calls.length, + disconnects: state.disconnect.mock.calls.length, + connects: state.connectCount, + captures: state.captureStarts, + captureCloses: state.captureTunnelClose.mock.calls.length, + adbTunnels: state.adbTunnelStarts, + tunnelCloses: state.tunnelClose.mock.calls.length, + }; +} + +/** The instance ids the runtime asked Limrun to delete after a case's own baseline. */ +function deletedInstances(from: number): string[] { + return state.androidDelete.mock.calls.slice(from).map(([instanceId]) => instanceId as string); +} + +/** Lets a case reach a resource a call is on its way to opening, without clock waits. */ +async function waitForNewOpen(starts: number, opened: () => number): Promise { + await vi.waitFor(() => assert.ok(opened() > starts, 'expected a new tunnel to be attempted')); +} + test('the platform module answers app state from the live Android provider session', async () => { const registration = createLimrunRuntime( { apiKey: 'lim_test_key', runtimeInstance: 'test-instance' }, @@ -187,6 +656,7 @@ test('the platform module surfaces an aborted request inside the app-state reade }); test('a network capture writes a private HAR with its failures on artifacts and at release', async () => { + const captureCloses = state.captureTunnelClose.mock.calls.length; const harDir = mkdtempForTestSync('limrun-har-'); const runtime = createLimrunRuntime( { apiKey: 'lim_test_key', networkHarDir: harDir }, @@ -228,7 +698,7 @@ test('a network capture writes a private HAR with its failures on artifacts and cloudArtifacts?: { cloudArtifacts: { url?: string }[]; message?: string }; }; assert.equal(released.cloudArtifacts?.cloudArtifacts[0]?.url, harPath); - assert.equal(state.captureTunnelClose.mock.calls.length, 1); + assert.equal(state.captureTunnelClose.mock.calls.length - captureCloses, 1); assert.match( String(released.cloudArtifacts?.message), /last request-failed missing\.example\.com:443/, @@ -281,6 +751,177 @@ test('a network capture that cannot start is an unavailable artifact with its ty } }); +// The SDK clears a request's timer before it reads the body, so the DELETE itself is what gets a deadline. +test('a release refuses a delete that never answers, within that delete’s own deadline', async () => { + const { deletes, disconnects } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-silent-delete'); + let reportDelete!: () => void; + state.deleteGate = new Promise((resolve) => { + reportDelete = resolve; + }); + vi.useFakeTimers(); + try { + await runtime.leaseLifecycle.allocate?.(lease); + const refused = refusalOf(runtime.leaseLifecycle.release?.(lease)); + await vi.advanceTimersByTimeAsync(DELETE_ANSWER_WAIT_MS); + // The lease is still held: Limrun never confirmed the instance was gone. + const refusedDelete = (await refused) as AppError; + assert.equal(refusedDelete.details?.reason, 'limrun_instance_delete_unconfirmed'); + assert.equal(refusedDelete.details.limrunInstanceId, 'android-instance-1'); + assert.match(String(refusedDelete.details.cause), /delete/i); + // What that delete goes on to answer is still observed: its late confirmation ends the lease. + reportDelete(); + await vi.advanceTimersByTimeAsync(0); + } finally { + reportDelete(); + state.deleteGate = undefined; + vi.useRealTimers(); + await runtime.shutdown(); + } + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); +}); + +test('a create that answers after its disposal gave up ends the instance it rents', async () => { + const { creates, deletes, connects, disconnects } = limrunActivity(); + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-late-create'); + try { + const refusedAllocation = refusalOf(runtime.leaseLifecycle.allocate?.(lease)); + await waitForCreate(creates + 1); + const refusedRelease = refusalOf(runtime.leaseLifecycle.release?.(lease)); + vi.useFakeTimers(); + await vi.advanceTimersByTimeAsync(CREATE_ANSWER_WAIT_MS); + // The release refused rather than waiting longer, and the create has still not named anything. + assert.ok((await refusedRelease) instanceof AppError, 'expected the expired release to refuse'); + assert.deepEqual(deletedInstances(deletes), []); + // The POST answers after that: the instance it names, and the session built on it, belong to the + // disposal that is already over, so that disposal ends them. + reportCreate(); + await vi.advanceTimersByTimeAsync(0); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + assert.ok((await refusedAllocation) instanceof AppError); + assert.equal(state.connectCount - connects, 1); + // The instance is gone and the session is down, so the lease is allocatable again: what the disposal + // that had already refused its caller owed is paid, and nothing is left owed to the next allocation. + state.createGate = undefined; + await runtime.leaseLifecycle.allocate?.(lease); + assert.equal(state.androidCreate.mock.calls.length - creates, 2); + } finally { + vi.useRealTimers(); + reportCreate(); + state.createGate = undefined; + await runtime.shutdown(); + } + assert.equal(state.androidCreate.mock.calls.length - creates, 2); + assert.deepEqual(deletedInstances(deletes), ['android-instance-1', 'android-instance-1']); +}); + +test('a create that fails after its session was built disconnects that session', async () => { + const { deletes, disconnects } = limrunActivity(); + state.uploadedApps = [{ id: 'asset-1', name: 'com.example.app.apk', md5: 'md5', os: 'android' }]; + // The app is installed under something this provider cannot match unambiguously. + state.installedApps = [{ id: 'com.other.app' }, { id: 'com.other.app.fork' }]; + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + try { + const refused = refusalOf( + runtime.leaseLifecycle.allocate?.(androidLease('lease-app-unresolved'), { + initialApp: 'com.example.app.apk', + }), + ); + assert.ok((await refused) instanceof AppError, 'expected the app resolution to fail'); + // The instance it rented is deleted, and so is the connection the session built beside it. + assert.deepEqual(deletedInstances(deletes), ['android-instance-1']); + assert.equal(state.disconnect.mock.calls.length - disconnects, 1); + } finally { + state.uploadedApps = []; + state.installedApps = []; + await runtime.shutdown(); + } +}); + +/** + * The alias a lease was created with answers for the asset name its caller reuses — through the binding + * the daemon actually resolves open targets with, and only while the lease is held: once it is released, + * the name answers as the plain string it is, mapped by nothing. + */ +test('a created app alias resolves the asset name back to the installed identifier', async () => { + state.uploadedApps = [{ id: 'asset-1', name: 'settings.apk', md5: 'md5', os: 'android' }]; + state.installedApps = [{ id: 'com.example.settings' }]; + const registration = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies(), { + includePlatformModule: true, + }); + try { + const device = await allocateAndroidDevice(registration.runtime, 'lease-alias', 'settings.apk'); + const owner = await registration.platformModule.loadRuntime(unusedLimrunRuntimeHost()); + const binding = await owner.bind({ + device, + intent: { kind: 'ordinary' }, + scope: requestScope(), + }); + const resolve = async (app: string) => + await binding.operations.resolveOpenTarget?.({ target: app, surface: 'app' }); + assert.equal((await resolve('settings.apk'))?.appBundleId, 'com.example.settings'); + assert.equal((await resolve('com.other.app'))?.appBundleId, 'com.other.app'); + + // The alias belonged to the lease, and the lease is gone: nothing maps the asset name any more. + await registration.runtime.shutdown(); + assert.equal((await resolve('settings.apk'))?.appBundleId, 'settings.apk'); + } finally { + state.uploadedApps = []; + state.installedApps = []; + await registration.runtime.shutdown(); + } +}); + +test('a joiner whose own flags are valid allocates despite another caller’s refused flag', async () => { + const { creates } = limrunActivity(); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-mixed-flags'); + try { + // The first call carries a flag this provider reads no field for; the second carries none. + const refused = refusalOf(runtime.leaseLifecycle.allocate?.(lease, REFUSED_FLAG)); + const joined = await runtime.leaseLifecycle.allocate?.(lease); + assert.equal(((await refused) as AppError).code, 'INVALID_ARGS'); + assert.equal(joined?.limrunInstanceId, 'android-instance-1'); + } finally { + await runtime.shutdown(); + } + // Only the valid caller's lease was billed, and only once. + assert.equal(state.androidCreate.mock.calls.length - creates, 1); +}); + +/** A joiner with unsupported flags is refused by a create that is still running, not handed its session. */ +test('a joiner whose own flags are refused is refused while another caller’s create runs', async () => { + const { creates } = limrunActivity(); + let reportCreate!: () => void; + state.createGate = new Promise((resolve) => { + reportCreate = resolve; + }); + const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies()); + const lease = androidLease('lease-refused-join'); + try { + const running = runtime.leaseLifecycle.allocate?.(lease); + await waitForCreate(creates + 1); + const joining = refusalOf(runtime.leaseLifecycle.allocate?.(lease, REFUSED_FLAG)); + reportCreate(); + assert.equal(((await joining) as AppError).code, 'INVALID_ARGS'); + assert.equal((await running)?.limrunInstanceId, 'android-instance-1'); + } finally { + reportCreate(); + state.createGate = undefined; + await runtime.shutdown(); + } + // The refused joiner did not start a second create for a lease it was refused on. + assert.equal(state.androidCreate.mock.calls.length - creates, 1); +}); + function completedRequest(url: string, error?: string): unknown { return { type: 'complete', @@ -315,8 +956,12 @@ function requestScope(signal: AbortSignal = new AbortController().signal): Platf async function allocateAndroidDevice( runtime: ReturnType, leaseId = 'lease-appstate', + initialApp?: string, ): Promise { - const allocation = await runtime.leaseLifecycle.allocate?.(androidLease(leaseId)); + const allocation = await runtime.leaseLifecycle.allocate?.( + androidLease(leaseId), + initialApp === undefined ? undefined : { initialApp }, + ); const device = allocation?.device; if (!device || typeof device !== 'object') throw new Error('Expected allocated device'); return device as DeviceInfo; @@ -336,44 +981,18 @@ function androidLease(leaseId: string): DeviceLease { } function testDependencies(): LimrunRuntimeDependencies { + const stub = stubLimrunRuntimeDependencies(); return { - clientVersion: 'test-version', + ...stub, android: { - createInteractor: () => ({}) as Interactor, - createPortReverse: async () => ({ - ensure: async () => {}, - remove: async () => {}, - removeAllOwned: async () => {}, - list: async () => [], - }), - inferAppName: async () => 'Example', - listApps: async (_adb: LimrunAdbExecutor, _filter: AppsFilter) => [], + ...stub.android, + // The ADB provider an interactor was handed is kept: a case drives it after its lease is gone. + createInteractor: (_device, adb) => { + state.heldAdbProvider = adb; + return {} as Interactor; + }, + listApps: async (_adb: LimrunAdbExecutor, _filter: AppsFilter) => state.installedApps, getForegroundApp: async () => (state.foreground ? { ...state.foreground } : undefined), - getKeyboardState: async () => ({ visible: false, inputOwner: 'unknown' as const }), - dismissKeyboard: async () => ({ - visible: false, - inputOwner: 'unknown' as const, - attempts: 0, - wasVisible: false, - dismissed: false, - }), - readLogs: async () => '', - forceStopApp: async () => {}, - deviceAdbInvocation: (serial: string, command: readonly string[]) => - androidAdbInvocation(androidAdbSerialTarget(serial), command), - hostAdbInvocation: (command: readonly string[]) => - androidAdbInvocation(androidAdbHostTarget(), command), - adbError: async (message: string) => new AppError('COMMAND_FAILED', message), }, - host: { - runAdb: async () => ({ stdout: '', stderr: '', exitCode: 0 }), - archiveDirectory: async () => undefined, - downloadFile: async () => undefined, - }, - ios: { - applySimctlSetting: async () => undefined, - resolveAppAlias: async (app: string) => app, - readBundleAppName: async () => undefined, - }, - } satisfies LimrunRuntimeDependencies; + }; } diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index 7a0f704f72..096157529b 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -13,9 +13,8 @@ import type { ProviderPortReverseOptions, } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, errorMessage } from '@agent-device/kernel/errors'; import { - cleanupLimrunAndroidAdbTunnel, configureLimrunAndroidPortReverse, createLimrunAndroidInteractor, createLimrunAndroidSession, @@ -30,12 +29,7 @@ import { platformForLimrunLeaseBackend, type LimrunPlatform, } from './device.ts'; -import { - createLimrunIosInteractor, - createLimrunIosSession, - installLimrunIosApp, - type LimrunIosSession, -} from './ios.ts'; +import { createLimrunIosInteractor, createLimrunIosSession, installLimrunIosApp } from './ios.ts'; import { createLimrunDeviceSession, type LimrunDeviceSession } from './device-session.ts'; import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; import type { @@ -46,7 +40,12 @@ import type { import { providerRuntimeOwner } from '@agent-device/contracts/platform-runtime'; import type { LimrunAppLogDescriptor } from './app-log-descriptor.ts'; import type { LimrunAppLogReader } from './app-log-runtime.ts'; -import { LIMRUN_CLIENT_ID, limrunUserAgent, requireLimrunOrgClient } from './client-options.ts'; +import { + deleteLimrunInstance, + LIMRUN_CLIENT_ID, + limrunUserAgent, + requireLimrunOrgClient, +} from './client-options.ts'; import { resolveLimrunRuntimeInstance } from './runtime-instance.ts'; import type { LimrunRequestOperationDrain } from './request-cancellation.ts'; import type { LimrunAppAsset } from './app-catalog.ts'; @@ -55,10 +54,17 @@ import type { CloudArtifactProvider, CloudArtifactsResult, } from '@agent-device/contracts/observability'; -import type { LimrunHarCapture, LimrunNetworkFailures } from './network-har.ts'; -import path from 'node:path'; - -type LimrunRuntimeSession = LimrunIosSession | LimrunAndroidSession; +import type { + LimrunAppAlias, + LimrunCreateOutcome, + LimrunDeleteResult, + LimrunDisposalOutcome, + LimrunLeaseOwner, + LimrunRuntimeSession, + LimrunTeardownRoute, +} from './lease-ownership.ts'; +import type { LimrunCaptureLifecycle } from './capture-lifecycle.ts'; +import type { LimrunDisposalHost } from './lease-release.ts'; const KEEP_ALIVE_INTERVAL_MS = 30_000; @@ -114,13 +120,21 @@ export function createLimrunRuntime( class LimrunRuntimeImplementation implements ProviderDeviceRuntime { private readonly limrun: Limrun | undefined; - private readonly sessions = new Map(); - private readonly keepAliveTimers = new Map(); - private readonly networkCaptures = new Map(); - private readonly appAliases = new Map< - string, - Readonly<{ assetName: string; installedAppId: string }> - >(); + /** + * A lease has an owner from the step it is allocated until the one disposal of it confirms. Allocate, + * join, release, recovery, shutdown and every reader of a live session go through this one map, so an + * in-flight create is never invisible to the route that has to end what it rents. + */ + private readonly leases = new Map(); + /** + * What each lease's network capture goes through, including the records a release settled. Absent until a + * lease asks for a capture, which is what keeps the capture's module out of a runtime that captures nothing: + * the lifecycle is only ever reached through `await import('./capture-lifecycle.ts')` in `captures()`, and + * this declaration stays a type for exactly that reason. + */ + private capturesInstance?: LimrunCaptureLifecycle; + /** Whether this runtime has shut down; it then holds nothing, so it must create nothing. */ + private shutDown = false; private readonly options: LimrunRuntimeOptions; private readonly dependencies: LimrunRuntimeDependencies; readonly provider = LIMRUN_PROVIDER; @@ -153,7 +167,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { readonly deviceInventoryProvider: DeviceInventoryProvider = async (request) => { if (request.leaseProvider !== this.provider || !request.leaseId) return null; - const session = this.sessions.get(request.leaseId); + const session = this.liveSessionForLease(request.leaseId); if (!session) return null; if (request.platform && request.platform !== session.platform) return []; return [session.device]; @@ -173,6 +187,24 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { : undefined; } + /** + * The capture lifecycle, made the first time a lease asks for a capture. The records a release settled live + * there, so an artifacts read for a lease this runtime no longer holds goes through the same instance that + * settled them, and a runtime that never captures anything never evaluates that module. + */ + private async captures(): Promise { + if (this.capturesInstance === undefined) { + const { LimrunCaptureLifecycle } = await import('./capture-lifecycle.ts'); + // Two allocations can be inside this load at once, and the settled records have to be one lifecycle's. + this.capturesInstance ??= new LimrunCaptureLifecycle({ + harDir: this.options.networkHarDir, + liveSessionOf: (owner) => this.liveSession(owner), + heldOwners: () => this.leases.values(), + }); + } + return this.capturesInstance; + } + ownsDevice(device: DeviceInfo): boolean { return parseLimrunDeviceId(device.id) !== undefined; } @@ -246,11 +278,29 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { return portReverseResult(options); } + /** + * Ends every instance this runtime holds or is on its way to holding, and says so when it could not. + * Each lease is taken, or joined, in one synchronous step, so a create that answers afterwards is + * disposed by the one disposal of that lease and by no other route, and a release already running is + * awaited rather than raced. + */ async shutdown(): Promise { - const sessions = [...this.sessions.values()]; - await Promise.allSettled(sessions.map(async (session) => await this.terminateSession(session))); - this.sessions.clear(); - this.appAliases.clear(); + this.shutDown = true; + // Snapshot into a binding before anything is awaited: `allocate` refuses once this flag is set and + // publishes an owner only where none exists, and nothing between the flag and this snapshot awaits. So + // no create can start here, and every create already running has an owner in this snapshot to take. + const held = [...this.leases.values()]; + const outcomes = await Promise.all( + held.map(async (owner) => await this.take(owner, 'shutdown')), + ); + // An instance Limrun would not give back is still billed, and this process will not stop it. The + // plugin's close is where an operator learns what a daemon close left running, so the instance and + // the provider's refusal travel on the error rather than being dropped. + const unended = outcomes.filter((outcome) => outcome.cleanup === 'uncertain'); + if (unended.length > 0) { + const { cleanupLeftInstancesRunning } = await import('./lease-release.ts'); + throw cleanupLeftInstancesRunning(unended); + } } private async allocate( @@ -260,88 +310,354 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (lease.leaseProvider !== this.provider) return undefined; const platform = platformForLimrunLeaseBackend(lease.backend); if (!platform) return undefined; - const { rejectRefusedLimrunProfileFields } = await import('./session-allocation.ts'); - // Before the reuse below, since a repeat allocation of a live lease carries flags of its own, and - // ahead of attach as well as create, since an attached instance reads none of them either. - rejectRefusedLimrunProfileFields(context); - const existing = this.sessions.get(lease.leaseId); - const session = - existing ?? - (await this.attachSession(platform, lease)) ?? - (await this.createSession(platform, lease, context)); - if (!existing) { - this.sessions.set(lease.leaseId, session); - if (this.options.keepAlive) this.startKeepAlive(lease.leaseId, session); + // Published without awaiting anything, so a lease is never unowned while its allocation is at its first + // await: a release or a shutdown arriving there finds this owner and takes it, and the create that owner + // stands on answers the caller's own profile flags before it bills anything. + let owner = this.leases.get(lease.leaseId); + if (owner === undefined) { + // Nothing between this lookup and the publication below awaits, so two allocations of one lease can + // never both pass it, and this runtime can be shut down only before or after this step. + if (this.shutDown) throw allocationRefusedAfterShutdown(lease); + owner = this.beginAllocation(lease, platform, context); + } else { + // A lease this runtime already holds, so this call answers for flags of its own, and settles them before + // it awaits anything else — including a create that is already running. Joining one is no way to be + // handed a session this caller was never entitled to; a new lease is validated as the first step of its + // own create instead, which is what keeps this publication free of any await. + await this.rejectRefusedProfileFields(context); } - // A repeat allocation can ask for the capture, or retry one that could not start. - if ( - context?.flags?.providerNetworkLogs === true && - this.networkCaptures.get(lease.leaseId)?.phase !== 'capturing' - ) { - await this.startNetworkCapture(lease.leaseId, session, context.flags.providerNetworkHosts); + await this.joinCreation(owner, context); + // A repeat allocation can ask for the capture, or retry one that could not start. A lease a teardown has + // already taken starts nothing: `captures.start` reads the owner's own live session, and what this lease + // has is that teardown's to end. + if (context?.flags?.providerNetworkLogs === true && owner.capture?.phase !== 'capturing') { + const captures = await this.captures(); + await captures.start(owner, context.flags.providerNetworkHosts); } + // A release or shutdown that took the lease while this call waited owns the instance the create made, and + // answers for it. This caller is refused for that teardown, with what it came to, rather than being handed + // an instance nothing will release for it. The refusal waits for the teardown to answer, so this caller + // never reports before the instance it was promised is gone. + const taken = owner.state; + if (taken.phase === 'releasing') { + const outcome = await taken.disposal; + const { allocationRefusedForTake } = await import('./lease-release.ts'); + throw allocationRefusedForTake(lease, taken.by, outcome); + } + const session = this.requireLiveSession(owner); return { limrunInstanceId: session.instanceId, device: session.device }; } - private async startNetworkCapture( - leaseId: string, - session: LimrunRuntimeSession, - hostList: unknown, + /** + * Brings one owner to the point its live session answers, and refuses this caller only for what it asked + * for itself. + */ + private async joinCreation( + owner: LimrunLeaseOwner, + context?: LeaseLifecycleContext, ): Promise { - const harDir = this.options.networkHarDir; - if (!harDir) { - this.networkCaptures.set(leaseId, { phase: 'unavailable', reason: 'capture-dir-unset' }); + if (owner.state.phase !== 'allocating') return; + const create = await this.creationFor(owner, context); + if (create === undefined) return; + if (create.status === 'ready') { + this.goLive(owner, create.session, create.appAlias); return; } - const hosts = - typeof hostList === 'string' - ? hostList - .split(',') - .map((host) => host.trim()) - .filter(Boolean) - : []; - const { startLimrunHarCapture } = await import('./network-har.ts'); - const result = await startLimrunHarCapture( - session.client, - hosts.length > 0 ? hosts : undefined, - ); - this.networkCaptures.set( - leaseId, - result.ok - ? { - phase: 'capturing', - harPath: path.join(harDir, `${leaseId}.har`), - capture: result.capture, - } - : { phase: 'unavailable', reason: result.reason, message: result.message }, - ); + if (create.status === 'refused_caller') throw create.error; + // A create that failed has no release coming for it, so this call is the only route left to end an + // instance it made before failing. What it came to outranks why the create did not finish: an instance + // Limrun is still billing is the caller's problem first. + const outcome = await this.settleFailedCreate(owner); + if (outcome?.cleanup === 'uncertain') { + const { createLeftInstanceRunning } = await import('./lease-release.ts'); + throw createLeftInstanceRunning(owner.lease, outcome, create.error); + } + throw create.error; + } + + /** + * The create one caller is entitled to: the one its lease's owner stands on, unless that create refused the + * caller that started it for a flag this provider reads no field for. That refusal billed nothing, so it is + * that caller's answer and not this lease's: a caller whose own flags this provider does read asks for the + * create the refusal foreclosed and makes it the lease's creation, on the owner the lease already has. Any + * later joiner of the same refusal follows that create rather than renting a second instance. + */ + private async creationFor( + owner: LimrunLeaseOwner, + context?: LeaseLifecycleContext, + ): Promise { + for (;;) { + const started = owner.creation; + const create = await started; + if (create.status !== 'refused_caller') return create; + // This caller's own flags were answered for before it got here, so the refusal below is another + // caller's. Another joiner of it may have replaced it while this caller was being answered: join that. + if (owner.creation !== started) continue; + // A teardown took the lease in the meantime, so what this lease has is that route's to end. + if (owner.state.phase !== 'allocating') return undefined; + const retried = this.createForLease(owner, context); + owner.creation = retried; + // Observed from the moment it is published, like the original creation: the owner's type promises a + // creation that cannot reach the process unobserved, and this keeps that promise by construction, not + // only by the fact that `createForLease` reports its failures as outcomes. + void retried.catch(() => undefined); + return await retried; + } + } + + /** The profile flags this provider reads nothing for, refused for a lease this runtime already holds. */ + private async rejectRefusedProfileFields(context?: LeaseLifecycleContext): Promise { + // Loaded here rather than eagerly so this package entry's declared closure stays unchanged. The create + // of a new allocation refuses the same flags as its first step, inside the owner it published. + const { rejectRefusedLimrunProfileFields } = await import('./session-allocation.ts'); + rejectRefusedLimrunProfileFields(context); + } + + /** + * Publishes the owner one allocation's create is standing on, in the same step that starts the create + * and before it can answer, so a duplicate allocation joins that create instead of making a second + * instance, and a release or shutdown arriving mid-create has somewhere to find the instance that create + * is on its way to making. The owner records that instance the moment Limrun names it, so whichever route + * takes the lease has what it has to end. + */ + private beginAllocation( + lease: DeviceLease, + platform: LimrunPlatform, + context?: LeaseLifecycleContext, + ): LimrunLeaseOwner { + let report!: (create: LimrunCreateOutcome) => void; + const creation = new Promise((settled) => { + report = settled; + }); + const owner: LimrunLeaseOwner = { + lease, + platform, + creation, + state: { phase: 'allocating' }, + rentedInstanceId: undefined, + session: undefined, + localRelease: undefined, + keepAlive: undefined, + capture: undefined, + appAlias: undefined, + deletedInstanceId: undefined, + pendingDelete: undefined, + }; + this.leases.set(lease.leaseId, owner); + // The create is started after the publication above, so its answer can never arrive without an owner + // beside it. Whichever route takes the lease in the meantime ends what this answer names. + void this.createForLease(owner, context).then(report); + return owner; } - /** Writes a live capture's HAR now; a released capture reports the file it wrote at release. */ - private async writeNetworkCapture(leaseId: string): Promise { - const state = this.networkCaptures.get(leaseId); - if (state?.phase !== 'capturing') return state; - const failures = await state.capture.failures(); + /** + * Attaches or makes this lease's instance and answers with the whole outcome rather than with a promise + * that resolves or rejects: a refused caller still has to say whether anything was left running, and the + * owner is where an instance this create rented gets recorded. Nothing here ends an instance, and nothing + * retries a create: a create whose answer was lost may have made an instance anyway, and that instance is + * reported and ended, never replaced by a second one. + */ + private async createForLease( + owner: LimrunLeaseOwner, + context?: LeaseLifecycleContext, + ): Promise { try { - const { writeLimrunHar } = await import('./network-har.ts'); - await writeLimrunHar(state.harPath, state.capture.entries()); - return { phase: 'written', harPath: state.harPath, failures }; - } catch (error) { - return { - phase: 'unavailable', - reason: 'har-write-failed', - message: error instanceof Error ? error.message : String(error), + const { + allocateLimrunAndroidSession, + allocateLimrunIosSession, + rejectRefusedLimrunProfileFields, + resolvePreinstalledAppId, + resolveRequestedLimrunAppAsset, + } = await import('./session-allocation.ts'); + // The first step of the create, so an allocation that published its owner before it could await + // anything still bills nothing for flags this provider reads no field for. That refusal belongs to the + // caller that carried the flag: it says nothing about the lease, which is why the owner's creation + // carries it as `refused_caller` rather than as a failed create. + try { + rejectRefusedLimrunProfileFields(context); + } catch (error) { + return { status: 'refused_caller', error }; + } + const platform = owner.platform; + // Attach first, and only ask for the organization client if there is nothing to attach: an attached + // instance needs no API key, and is not created with an app asset of ours. + const attached = await this.attachSession(platform, owner.lease); + if (attached) { + owner.session = attached; + return { status: 'ready', session: attached, appAlias: undefined }; + } + const limrun = this.orgClient('Creating an instance'); + const requestedAsset = await resolveRequestedLimrunAppAsset(limrun, platform, context); + const params = this.sessionAllocationParams(limrun, owner.lease, requestedAsset); + const rent = (instanceId: string): void => { + owner.rentedInstanceId = instanceId; + }; + // The session is recorded in the step that answers for it, so every later step of this allocation + // fails with its handles still owned: the instance client and the host ADB endpoint are released by + // whoever ends this lease, whether or not the allocation ever answered. + const session = + platform === 'ios' + ? await allocateLimrunIosSession(params, rent) + : await allocateLimrunAndroidSession(params, rent); + owner.session = session; + const appAlias: LimrunAppAlias | undefined = requestedAsset && { + assetName: requestedAsset.name, + installedAppId: await resolvePreinstalledAppId(session, requestedAsset), }; + return { status: 'ready', session, appAlias }; + } catch (error) { + return { status: 'failed', error }; } } + /** + * Puts the instance one create made, and the app alias that came with it, onto its owner. Another + * allocation that joined the same create can get here first, and a route that took the lease while the + * create ran owns the instance instead — either way this owner is already where it belongs, and nothing + * is published for a lease that no longer has an allocation. + */ + private goLive( + owner: LimrunLeaseOwner, + session: LimrunRuntimeSession, + appAlias: LimrunAppAlias | undefined, + ): void { + if (owner.state.phase !== 'allocating') return; + owner.appAlias = appAlias; + owner.state = { phase: 'live', session }; + if (this.options.keepAlive) this.startKeepAlive(owner, session); + } + + /** + * Closes an owner whose create made nothing, and ends the instance one that made an instance before it + * failed. A create that failed has no release coming for it — its caller is being refused — so this is + * the route that ends what it left running, through the same take every other teardown goes through. + * The outcome is reported to that caller, which is the only one left to hear it. + */ + private async settleFailedCreate( + owner: LimrunLeaseOwner, + ): Promise { + const state = owner.state; + if (state.phase === 'allocating') return await this.take(owner, 'create'); + // Another route took the lease while this call waited, and its disposal owns what the create made. + return state.phase === 'releasing' ? await state.disposal : undefined; + } + + /** + * Hands one lease's resources to the one route that may end them, or joins the disposal already ending + * them. The take and the disposal it starts are recorded before this step returns, so no route can reach + * the lease in between and find it free, and the owner leaves the map only once that disposal confirmed. + * A disposal that could not confirm keeps the owner here, so the next release, recovery or shutdown + * retries the delete rather than reporting an instance Limrun may still be billing. + */ + private take(owner: LimrunLeaseOwner, by: LimrunTeardownRoute): Promise { + const taken = owner.state; + const settled = taken.phase === 'releasing' ? taken.settled : undefined; + if (taken.phase === 'releasing' && !(settled?.cleanup === 'uncertain')) return taken.disposal; + let reportDisposal!: (outcome: LimrunDisposalOutcome) => void; + const disposal = new Promise((settledOutcome) => { + reportDisposal = settledOutcome; + }); + owner.state = { + phase: 'releasing', + by, + instanceId: + this.liveSession(owner)?.instanceId ?? owner.rentedInstanceId ?? settled?.instanceId, + disposal, + settled: undefined, + }; + const report = (outcome: LimrunDisposalOutcome): void => { + const state = owner.state; + if (state.phase === 'releasing') { + state.settled = outcome; + state.instanceId = outcome.instanceId ?? state.instanceId; + // Whatever the create made is now ended, or was never made, so this owner holds nothing and the + // lease is free again. An unconfirmed delete keeps the owner so its outcome can be retried. + if (outcome.cleanup !== 'uncertain') this.closeOwner(owner); + else if (outcome.creationPending) { + void owner.creation.then(() => this.take(owner, by)).catch(() => undefined); + } + } + reportDisposal(outcome); + }; + // `disposeOwner` reports the failures it can name, so the rejection branch is only for what it could not + // report at all. The disposal still answers, because a caller may be waiting on this lease's instance. + void this.disposeOwner(owner).then(report, (error: unknown) => { + report({ + cleanup: 'uncertain', + instanceId: owner.rentedInstanceId, + failure: errorMessage(error), + cause: error, + capture: undefined, + }); + }); + return disposal; + } + + /** + * Hands one taken owner to the module that ends what it stands for. The disposal may ask only two things of + * this runtime — delete one instance and end one lease's capture — so it never reaches back into the routes + * that allocate and read one, and the lease is still given back by the take that started this disposal. + */ + private async disposeOwner(owner: LimrunLeaseOwner): Promise { + const { disposeLimrunLeaseOwner } = await import('./lease-release.ts'); + return await disposeLimrunLeaseOwner(this.disposalHost, owner); + } + + private readonly disposalHost: LimrunDisposalHost = { + deleteInstance: async (platform, instanceId) => + await this.deleteCreatedInstance(platform, instanceId), + endCapture: async (owner) => await this.capturesInstance?.end(owner), + }; + + /** One instance delete. `undefined` is Limrun's confirmation that the instance is gone. */ + private async deleteCreatedInstance( + platform: LimrunPlatform, + instanceId: string, + ): Promise { + const limrun = this.orgClient('Deleting an instance'); + return await deleteLimrunInstance(limrun, platform, instanceId); + } + + /** Removes an owner once it holds nothing, and only while it is still the map's owner for that lease. */ + private closeOwner(owner: LimrunLeaseOwner): void { + if (this.leases.get(owner.lease.leaseId) === owner) this.leases.delete(owner.lease.leaseId); + } + + private startKeepAlive(owner: LimrunLeaseOwner, session: LimrunRuntimeSession): void { + const timer = setInterval(() => { + try { + session.client.keepAlive(); + } catch { + // A dead connection surfaces on the next real command. + } + }, KEEP_ALIVE_INTERVAL_MS); + timer.unref(); + owner.keepAlive = timer; + } + readonly cloudArtifacts: CloudArtifactProvider = { - listCloudArtifacts: async (query) => { - const capture = query.leaseId ? await this.writeNetworkCapture(query.leaseId) : undefined; - return capture && networkArtifacts(capture); - }, + listCloudArtifacts: async (query) => await this.listCloudArtifacts(query.leaseId), }; + /** + * What `artifacts` answers for one lease: its capture written now while the lease is held, and the + * record a release settled after the lease is gone. A runtime that never started a capture has no + * lifecycle to ask, so nothing is loaded for it. + */ + private async listCloudArtifacts( + leaseId: string | undefined, + ): Promise { + if (leaseId === undefined) return undefined; + const captures = this.capturesInstance; + if (captures === undefined) return undefined; + if (this.leases.get(leaseId)?.capture?.phase === 'starting') { + return { provider: LIMRUN_PROVIDER, status: 'pending', cloudArtifacts: [] }; + } + const { networkArtifacts } = await import('./network-har.ts'); + const capture = await captures.list(leaseId, this.leases.get(leaseId)?.capture); + return capture && networkArtifacts(capture); + } + private async attachSession( platform: LimrunPlatform, lease: DeviceLease, @@ -366,39 +682,6 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { ); } - private async createSession( - platform: LimrunPlatform, - lease: DeviceLease, - context?: LeaseLifecycleContext, - ): Promise { - const { - allocateLimrunAndroidSession, - allocateLimrunIosSession, - resolvePreinstalledAppId, - resolveRequestedLimrunAppAsset, - } = await import('./session-allocation.ts'); - const limrun = this.orgClient('Creating an instance'); - const requestedAsset = await resolveRequestedLimrunAppAsset(limrun, platform, context); - const params = this.sessionAllocationParams(limrun, lease, requestedAsset); - const session = - platform === 'ios' - ? await allocateLimrunIosSession(params) - : await allocateLimrunAndroidSession(params); - if (requestedAsset) { - try { - const installedAppId = await resolvePreinstalledAppId(session, requestedAsset); - this.appAliases.set(lease.leaseId, { - assetName: requestedAsset.name, - installedAppId, - }); - } catch (error) { - await this.terminateSession(session); - throw error; - } - } - return session; - } - private sessionAllocationParams(limrun: Limrun, lease: DeviceLease, app?: LimrunAppAsset) { return { limrun, @@ -424,99 +707,51 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { } private async release(lease: DeviceLease): Promise | undefined> { - const session = this.sessions.get(lease.leaseId); - if (!session) return await this.releaseRecoveredSession(lease); - const live = this.networkCaptures.get(lease.leaseId); - const capture = await this.writeNetworkCapture(lease.leaseId); - if (capture) { - // Re-insertion keeps the map in settle order, so the oldest released record goes first. - this.networkCaptures.delete(lease.leaseId); - this.networkCaptures.set(lease.leaseId, capture); - } - for (const leaseId of this.networkCaptures.keys()) { - if (this.networkCaptures.size <= SETTLED_NETWORK_CAPTURE_LIMIT) break; - if (!this.sessions.has(leaseId)) this.networkCaptures.delete(leaseId); - } - if (live?.phase === 'capturing') live.capture.close(); - await this.terminateSession(session); - this.sessions.delete(lease.leaseId); - this.appAliases.delete(lease.leaseId); - return { - limrunInstanceId: session.instanceId, - ...(capture ? { cloudArtifacts: networkArtifacts(capture) } : {}), - }; + // The take happens in this step, before anything is loaded: the release owns the lease from here, and no + // await in between can let the allocation still waiting on its create answer ahead of it. + const owner = this.leases.get(lease.leaseId); + const disposal = owner === undefined ? undefined : this.take(owner, 'release'); + const { releaseLimrunLease } = await import('./lease-release.ts'); + return await releaseLimrunLease(this.limrun, lease, disposal); } - private async releaseRecoveredSession( - lease: DeviceLease, - ): Promise | undefined> { - const platform = platformForLimrunLeaseBackend(lease.backend); - const limrun = this.limrun; - if (!platform || !limrun) return undefined; - const labelSelector = `provider=${LIMRUN_PROVIDER},leaseId=${lease.leaseId}`; - const instances = - platform === 'ios' - ? await limrun.iosInstances.list({ labelSelector }) - : await limrun.androidInstances.list({ labelSelector }); - const instanceIds = instances.getPaginatedItems().map((instance) => instance.metadata.id); - for (const instanceId of instanceIds) { - if (platform === 'ios') { - await limrun.iosInstances.delete(instanceId); - } else { - await limrun.androidInstances.delete(instanceId); - } - } - if (instanceIds.length === 0) return undefined; - return { limrunInstanceId: instanceIds[0], limrunInstanceCount: instanceIds.length }; + private orgClient(operation: string, hint?: string): Limrun { + return requireLimrunOrgClient(this.limrun, operation, hint); } - private startKeepAlive(leaseId: string, session: LimrunRuntimeSession): void { - const timer = setInterval(() => { - try { - session.client.keepAlive(); - } catch { - // A dead connection surfaces on the next real command. - } - }, KEEP_ALIVE_INTERVAL_MS); - timer.unref(); - this.keepAliveTimers.set(leaseId, timer); + /** The live session of one owner: what its create made, and only while the lease is held. */ + private liveSession(owner: LimrunLeaseOwner | undefined): LimrunRuntimeSession | undefined { + return owner?.state.phase === 'live' ? owner.state.session : undefined; } - private async terminateSession(session: LimrunRuntimeSession): Promise { - const leaseId = session.lease.leaseId; - clearInterval(this.keepAliveTimers.get(leaseId)); - this.keepAliveTimers.delete(leaseId); - const capture = this.networkCaptures.get(leaseId); - if (capture?.phase === 'capturing') { - this.networkCaptures.delete(leaseId); - capture.capture.close(); - } - session.client.disconnect(); - if (session.platform === 'android') await cleanupLimrunAndroidAdbTunnel(session); - if (session.ownership === 'attached') return; - const limrun = this.orgClient('Deleting an instance'); - if (session.platform === 'ios') { - await limrun.iosInstances.delete(session.instanceId); - } else { - await limrun.androidInstances.delete(session.instanceId); + /** The live session of one owner, for a route that has already refused every other phase. */ + private requireLiveSession(owner: LimrunLeaseOwner): LimrunRuntimeSession { + const session = this.liveSession(owner); + if (session === undefined) { + throw new AppError( + 'COMMAND_FAILED', + `Limrun lease '${owner.lease.leaseId}' has no live session`, + { reason: 'limrun_lease_not_live' }, + ); } + return session; } - private orgClient(operation: string, hint?: string): Limrun { - return requireLimrunOrgClient(this.limrun, operation, hint); + private liveSessionForLease(leaseId: string): LimrunRuntimeSession | undefined { + return this.liveSession(this.leases.get(leaseId)); } private getSessionForDevice(device: DeviceInfo): LimrunRuntimeSession | undefined { const parsed = parseLimrunDeviceId(device.id); if (!parsed) return undefined; - const session = this.sessions.get(parsed.leaseId); + const session = this.liveSessionForLease(parsed.leaseId); return session?.platform === parsed.platform ? session : undefined; } resolveAppReference(device: DeviceInfo, app: string): string { const parsed = parseLimrunDeviceId(device.id); if (!parsed) return app; - const alias = this.appAliases.get(parsed.leaseId); + const alias = this.leases.get(parsed.leaseId)?.appAlias; return alias?.assetName === app ? alias.installedAppId : app; } @@ -549,7 +784,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { } private requireAndroidPortReverseSession(leaseId: string): LimrunAndroidSession | undefined { - const session = this.sessions.get(leaseId); + const session = this.liveSessionForLease(leaseId); if (!session || session.platform === 'android') return session; throw unsupported( 'port reverse', @@ -642,64 +877,14 @@ function unsupported(command: string, message: string): never { throw new AppError('UNSUPPORTED_OPERATION', message, { command }); } -/** Released captures stay listable; the oldest records are forgotten past this count. */ -const SETTLED_NETWORK_CAPTURE_LIMIT = 32; - -type SettledNetworkCapture = - | Readonly<{ phase: 'written'; harPath: string; failures: LimrunNetworkFailures }> - | Readonly<{ - phase: 'unavailable'; - reason: 'capture-dir-unset' | 'tunnel-in-use' | 'tunnel-start-failed' | 'har-write-failed'; - message?: string; - }>; - -type NetworkCapture = - | SettledNetworkCapture - | Readonly<{ phase: 'capturing'; harPath: string; capture: LimrunHarCapture }>; - -const NETWORK_ARTIFACT_NAME = 'Network traffic (HAR on the daemon host)'; - -function networkArtifacts(capture: SettledNetworkCapture): CloudArtifactsResult { - if (capture.phase === 'unavailable') { - return { - provider: LIMRUN_PROVIDER, - status: 'unavailable', - message: `Network capture unavailable (${capture.reason})${capture.message ? `: ${capture.message}` : ''}`, - cloudArtifacts: [ - { - provider: LIMRUN_PROVIDER, - kind: 'network', - name: NETWORK_ARTIFACT_NAME, - availability: 'unavailable', - metadata: { reason: capture.reason }, - }, - ], - }; - } - const { count, last, lastDialFailure, truncated } = capture.failures; - const summary = [ - `Network capture failures: ${count}`, - ...(last - ? [`last ${last.kind}${last.host ? ` ${last.host}${last.port ? `:${last.port}` : ''}` : ''}`] - : []), - ...(lastDialFailure ? [`last dial failure ${lastDialFailure}`] : []), - ...(truncated > 0 ? [`${truncated} oldest requests left the HAR`] : []), - ].join('; '); - return { - provider: LIMRUN_PROVIDER, - status: 'ready', - ...(count > 0 || lastDialFailure || truncated > 0 ? { message: summary } : {}), - cloudArtifacts: [ - { - provider: LIMRUN_PROVIDER, - kind: 'network', - name: NETWORK_ARTIFACT_NAME, - url: capture.harPath, - extension: 'har', - contentType: 'application/json', - availability: 'ready', - metadata: { failures: capture.failures }, - }, - ], - }; +/** A lease cannot be allocated into a runtime that has been told to end every instance it holds. */ +function allocationRefusedAfterShutdown(lease: DeviceLease): never { + throw new AppError( + 'SESSION_NOT_FOUND', + `Cannot allocate Limrun lease '${lease.leaseId}': this runtime is shut down`, + { + reason: 'limrun_runtime_shut_down', + hint: 'Start the daemon again to allocate a Limrun lease.', + }, + ); } diff --git a/packages/provider-limrun/src/session-allocation.test.ts b/packages/provider-limrun/src/session-allocation.test.ts new file mode 100644 index 0000000000..60096de044 --- /dev/null +++ b/packages/provider-limrun/src/session-allocation.test.ts @@ -0,0 +1,219 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { test } from 'vitest'; +import Limrun from '@limrun/api'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { deleteLimrunInstance } from './client-options.ts'; +import { allocateLimrunAndroidSession, allocateLimrunIosSession } from './session-allocation.ts'; +import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts'; + +/** + * An instance create whose answer was lost is the one request this runtime must not repeat: Limrun may have + * made the instance already, and the SDK's own retries would POST again and rent a second instance no owner + * learns about. These cases put the real SDK in front of a local server, since that retry is decided below the + * request boundary every other Limrun suite replaces. + */ + +test('the Android instance create asks the SDK for one attempt, so a lost answer rents no second instance', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${request.path}`); + // The instance may well have been made; this answer never arrives. + request.drop(); + }); + const rented: string[] = []; + try { + await assert.rejects( + async () => + await allocateLimrunAndroidSession( + allocationParams(server), + (instanceId) => void rented.push(instanceId), + ), + ); + } finally { + await server.close(); + } + // One POST, so at most one instance: the SDK was not left to resolve the ambiguity by asking again. + assert.deepEqual(routes, ['POST /v1/android_instances']); + // Nothing was named, since no answer came back to name it. + assert.deepEqual(rented, []); +}); + +test('the iOS instance create asks the SDK for one attempt, so a lost answer rents no second instance', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${request.path}`); + request.drop(); + }); + const rented: string[] = []; + try { + await assert.rejects( + async () => + await allocateLimrunIosSession( + allocationParams(server), + (instanceId) => void rented.push(instanceId), + ), + ); + } finally { + await server.close(); + } + assert.deepEqual(routes, ['POST /v1/ios_instances']); + assert.deepEqual(rented, []); +}); + +test('an Android instance that answers without its endpoints is reported as rented and left for its owner', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${request.path}`); + await request.respond( + 200, + JSON.stringify({ metadata: { id: 'android-instance-9' }, status: { token: 'lim_token' } }), + ); + }); + const rented: string[] = []; + try { + await assert.rejects( + async () => + await allocateLimrunAndroidSession( + allocationParams(server), + (instanceId) => void rented.push(instanceId), + ), + (error) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /did not expose API and ADB websocket endpoints/.test(error.message), + ); + } finally { + await server.close(); + } + // The create names what it rented before any session is built on it, which is what lets the caller's owner + // end it; the helper does not delete it here, where a refusal could be swallowed. + assert.deepEqual(rented, ['android-instance-9']); + assert.deepEqual(routes, ['POST /v1/android_instances']); +}); + +/** + * The instance delete carries the same single-attempt contract as the create, and answers for the one refusal + * that names the outcome it was after: an instance Limrun already removed is gone, which is what the delete + * was asked to achieve. + */ + +test('the instance delete asks the SDK for one attempt, so a lost answer is not repeated behind this runtime’s back', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${String(request.path)}`); + // The instance may well be gone; this answer never arrives, and only the daemon's budgets retry it. + request.drop(); + }); + try { + const deleted = await deleteLimrunInstance(clientFor(server), 'android', 'android-instance-1'); + assert.deepEqual(routes, ['DELETE /v1/android_instances/android-instance-1']); + assert.match(String(deleted.failure), /connection|socket|abort|fetch|closed/i); + assert.ok(deleted.cause !== undefined, 'the refusal is reported, not swallowed'); + } finally { + await server.close(); + } +}); + +test('an instance delete that finds the instance already gone confirms it deleted', async () => { + const routes: string[] = []; + const server = await startInstanceServer(async (request) => { + routes.push(`${String(request.method)} ${String(request.path)}`); + await request.respond(404, JSON.stringify({ error: { message: 'instance not found' } })); + }); + try { + const deleted = await deleteLimrunInstance(clientFor(server), 'ios', 'ios-instance-1'); + // Gone is gone: the delete was not refused, so no later route re-issues it and no lease is held for it. + assert.equal(deleted.failure, undefined); + assert.equal(deleted.cause, undefined); + assert.deepEqual(routes, ['DELETE /v1/ios_instances/ios-instance-1']); + } finally { + await server.close(); + } +}); + +function clientFor(server: InstanceServer): Limrun { + return new Limrun({ apiKey: server.key, baseURL: server.baseUrl }); +} + +function allocationParams(server: InstanceServer) { + return { + limrun: clientFor(server), + lease: { + leaseId: 'lease-single-attempt', + tenantId: 'team-a', + runId: 'run-a', + backend: 'android-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + } satisfies DeviceLease, + metadata: { displayName: 'agent-device', labels: { provider: 'limrun' } }, + dependencies: unreachedDependencies, + }; +} + +/** + * The host and platform adapters none of these cases reaches: the create, or the endpoints it names, answer + * first. Any step through one of these means a case lost its premise and fails on the spot. + */ +const unreachedDependencies = { + android: { + createPortReverse: async (): Promise => { + throw new Error('expected no session for an instance whose endpoints never arrived'); + }, + }, +} as unknown as LimrunRuntimeDependencies; + +type InstanceServer = Readonly<{ + baseUrl: string; + key: string; + close(): Promise; +}>; + +type ReceivedRequest = Readonly<{ + method: string | undefined; + path: string; + /** Ends the connection without an answer, the ambiguity a retry must not resolve by POSTing again. */ + drop(): void; + respond(status: number, body: string): Promise; +}>; + +function startInstanceServer( + onRequest: (request: ReceivedRequest) => Promise, +): Promise { + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + void onRequest({ + method: req.method, + // The query is Limrun's own create parameters, not part of the route a retry would repeat. + path: (req.url ?? '').split('?')[0] ?? '', + drop: () => req.socket.destroy(), + respond: async (status, body) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(body); + }, + }); + }); + }); + return new Promise((started, failed) => { + server.once('error', failed); + server.listen(0, '127.0.0.1', () => { + const address = server.address() as AddressInfo; + started({ + baseUrl: `http://127.0.0.1:${String(address.port)}`, + key: 'lim_test_key', + close: async () => + await new Promise((closed, closeFailed) => { + server.close((error) => (error ? closeFailed(error) : closed())); + server.closeAllConnections(); + }), + }); + }); + }); +} diff --git a/packages/provider-limrun/src/session-allocation.ts b/packages/provider-limrun/src/session-allocation.ts index 7c9fda7219..07fe452ccf 100644 --- a/packages/provider-limrun/src/session-allocation.ts +++ b/packages/provider-limrun/src/session-allocation.ts @@ -3,6 +3,7 @@ import type { DeviceLease, LeaseLifecycleContext } from '@agent-device/contracts import { AppError } from '@agent-device/kernel/errors'; import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import { createLimrunAndroidSession, type LimrunAndroidSession } from './android.ts'; +import { LIMRUN_SINGLE_ATTEMPT } from './client-options.ts'; import { buildLimrunDevice, LIMRUN_PROFILE_FIELDS } from './device.ts'; import { createLimrunIosSession, type LimrunIosSession } from './ios.ts'; import { @@ -67,89 +68,99 @@ export async function resolvePreinstalledAppId( export async function allocateLimrunIosSession( params: SessionAllocationParams, + onRent: (instanceId: string) => void, ): Promise { - const instance = (await params.limrun.iosInstances.create({ - wait: true, - metadata: params.metadata, - spec: { - ...(params.region ? { region: params.region } : {}), - ...(params.app - ? { - initialAssets: [ - { - kind: 'App' as const, - source: 'AssetID' as const, - assetId: params.app.id, - launchMode: 'RelaunchIfRunning' as const, - }, - ], - } - : {}), - }, - })) as LimrunInstance; - try { - if (!instance.status.apiUrl) { - throw new AppError('COMMAND_FAILED', 'Limrun iOS instance did not expose apiUrl'); - } - return await createLimrunIosSession( - { - lease: params.lease, - instanceId: instance.metadata.id, - ownership: 'created', - device: buildLimrunDevice('ios', params.lease, instance.metadata.id), - apiUrl: instance.status.apiUrl, - token: instance.status.token, + const instance = (await params.limrun.iosInstances.create( + { + wait: true, + metadata: params.metadata, + spec: { + ...(params.region ? { region: params.region } : {}), + ...(params.app + ? { + initialAssets: [ + { + kind: 'App' as const, + source: 'AssetID' as const, + assetId: params.app.id, + launchMode: 'RelaunchIfRunning' as const, + }, + ], + } + : {}), }, - params.dependencies, - ); - } catch (error) { - await params.limrun.iosInstances.delete(instance.metadata.id).catch(() => {}); - throw error; + }, + // Limrun may have created this instance even though its answer was lost, so the SDK must not POST + // this call again: a second create would rent a second instance nobody is standing beside. + LIMRUN_SINGLE_ATTEMPT, + )) as LimrunInstance; + const instanceId = instance.metadata.id; + // Rented from this step: the caller's owner records the instance here, before any session is built on it, + // so its disposal can always end it — including when the endpoint check or the session below fails. + onRent(instanceId); + if (!instance.status.apiUrl) { + throw new AppError('COMMAND_FAILED', 'Limrun iOS instance did not expose apiUrl'); } + return await createLimrunIosSession( + { + lease: params.lease, + instanceId, + ownership: 'created', + device: buildLimrunDevice('ios', params.lease, instanceId), + apiUrl: instance.status.apiUrl, + token: instance.status.token, + }, + params.dependencies, + ); } export async function allocateLimrunAndroidSession( params: SessionAllocationParams, + onRent: (instanceId: string) => void, ): Promise { - const instance = (await params.limrun.androidInstances.create({ - wait: true, - metadata: params.metadata, - spec: { - ...(params.region ? { region: params.region } : {}), - ...(params.app - ? { - initialAssets: [ - { - kind: 'App' as const, - source: 'AssetIDs' as const, - assetIds: [params.app.id], - }, - ], - } - : {}), - }, - })) as LimrunInstance; - try { - if (!instance.status.apiUrl || !instance.status.adbWebSocketUrl) { - throw new AppError( - 'COMMAND_FAILED', - 'Limrun Android instance did not expose API and ADB websocket endpoints', - ); - } - return await createLimrunAndroidSession( - { - lease: params.lease, - instanceId: instance.metadata.id, - ownership: 'created', - device: buildLimrunDevice('android', params.lease, instance.metadata.id), - apiUrl: instance.status.apiUrl, - token: instance.status.token, - adbUrl: instance.status.adbWebSocketUrl, + const instance = (await params.limrun.androidInstances.create( + { + wait: true, + metadata: params.metadata, + spec: { + ...(params.region ? { region: params.region } : {}), + ...(params.app + ? { + initialAssets: [ + { + kind: 'App' as const, + source: 'AssetIDs' as const, + assetIds: [params.app.id], + }, + ], + } + : {}), }, - params.dependencies, + }, + // Limrun may have created this instance even though its answer was lost, so the SDK must not POST + // this call again: a second create would rent a second instance nobody is standing beside. + LIMRUN_SINGLE_ATTEMPT, + )) as LimrunInstance; + const instanceId = instance.metadata.id; + // Rented from this step: the caller's owner records the instance here, before any session is built on it, + // so its disposal can always end it — including when an endpoint is missing or the session below fails. + onRent(instanceId); + if (!instance.status.apiUrl || !instance.status.adbWebSocketUrl) { + throw new AppError( + 'COMMAND_FAILED', + 'Limrun Android instance did not expose API and ADB websocket endpoints', ); - } catch (error) { - await params.limrun.androidInstances.delete(instance.metadata.id).catch(() => {}); - throw error; } + return await createLimrunAndroidSession( + { + lease: params.lease, + instanceId, + ownership: 'created', + device: buildLimrunDevice('android', params.lease, instanceId), + apiUrl: instance.status.apiUrl, + token: instance.status.token, + adbUrl: instance.status.adbWebSocketUrl, + }, + params.dependencies, + ); } diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index dec506e77a..ed9f85793f 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -52,7 +52,7 @@ const limrunMockState = vi.hoisted(() => { iosList: vi.fn<() => Promise>(async () => ({ getPaginatedItems: () => [], })), - iosDelete: vi.fn(async () => undefined), + iosDelete: vi.fn(async (_instanceId: string, _options?: { maxRetries: number }) => undefined), androidCreate: vi.fn(async () => ({ metadata: { id: 'android-instance-1' }, status: { @@ -64,7 +64,9 @@ const limrunMockState = vi.hoisted(() => { androidList: vi.fn<() => Promise>(async () => ({ getPaginatedItems: () => [], })), - androidDelete: vi.fn(async () => undefined), + androidDelete: vi.fn( + async (_instanceId: string, _options?: { maxRetries: number }) => undefined, + ), }; }); @@ -254,7 +256,9 @@ test('Limrun reclaims a labeled iOS instance without an in-memory session', asyn assert.deepEqual(limrunMockState.iosList.mock.calls, [ [{ labelSelector: 'provider=limrun,leaseId=lease-recovered-ios' }], ]); - assert.deepEqual(limrunMockState.iosDelete.mock.calls, [['ios-instance-recovered']]); + assert.deepEqual(limrunMockState.iosDelete.mock.calls, [ + ['ios-instance-recovered', { maxRetries: 0 }], + ]); } finally { await runtime.shutdown(); } @@ -692,7 +696,7 @@ test('Limrun deletes iOS instance when post-create validation fails', async () = const allocateLease = runtime.leaseLifecycle.allocate; if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); await assert.rejects(() => allocateLease(lease), /did not expose apiUrl/); - assert.deepEqual(limrunMockState.iosDelete.mock.calls[0], ['ios-instance-missing-api']); + assert.equal(limrunMockState.iosDelete.mock.calls[0]?.[0], 'ios-instance-missing-api'); } finally { await runtime.shutdown(); } @@ -724,7 +728,7 @@ test('Limrun deletes Android instance when post-create validation fails', async const allocateLease = runtime.leaseLifecycle.allocate; if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); await assert.rejects(() => allocateLease(lease), /did not expose API and ADB/); - assert.deepEqual(limrunMockState.androidDelete.mock.calls[0], ['android-instance-missing-adb']); + assert.equal(limrunMockState.androidDelete.mock.calls[0]?.[0], 'android-instance-missing-adb'); } finally { await runtime.shutdown(); } @@ -754,7 +758,19 @@ test('Limrun keeps session tracked when release fails so release can be retried' await allocateLease(lease); limrunMockState.androidDelete.mockRejectedValueOnce(new Error('temporary delete failure')); - await assert.rejects(() => releaseLease(lease), /temporary delete failure/); + // An unconfirmed delete is reported as such, with the instance still held, rather than as a release. + await assert.rejects( + () => releaseLease(lease), + (error: unknown) => { + const details = (error as AppError).details; + return ( + error instanceof AppError && + details?.reason === 'limrun_instance_delete_unconfirmed' && + details.limrunInstanceId === 'android-instance-1' && + details.cause === 'temporary delete failure' + ); + }, + ); assert.deepEqual(await releaseLease(lease), { limrunInstanceId: 'android-instance-1' }); } finally { await runtime.shutdown(); @@ -897,7 +913,10 @@ test('Limrun instance access wins over the API key for its platform only', async assert.equal(limrunMockState.iosCreate.mock.calls.length, 0); assert.equal(limrunMockState.iosDelete.mock.calls.length, 0); assert.equal(limrunMockState.androidCreate.mock.calls.length, 1); - assert.deepEqual(limrunMockState.androidDelete.mock.calls, [['android-instance-1']]); + assert.deepEqual( + limrunMockState.androidDelete.mock.calls.map(([instanceId]) => instanceId), + ['android-instance-1'], + ); }); test('Limrun without an API key refuses operations that need one', async () => { diff --git a/src/provider-device-runtimes.test.ts b/src/provider-device-runtimes.test.ts index 887ec19236..a023aba458 100644 --- a/src/provider-device-runtimes.test.ts +++ b/src/provider-device-runtimes.test.ts @@ -13,7 +13,7 @@ const limrun = vi.hoisted(() => ({ status: { token: 'instance-token', apiUrl: 'https://ios.example' }, })), iosList: vi.fn(async () => ({ getPaginatedItems: () => [] as { metadata: { id: string } }[] })), - iosDelete: vi.fn(async (_instanceId: string) => undefined), + iosDelete: vi.fn(async (_instanceId: string, _options?: { maxRetries: number }) => undefined), iosDisconnect: vi.fn(), })); @@ -93,7 +93,10 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest assert.deepEqual(limrun.iosList.mock.calls.at(-1), [ { labelSelector: `provider=limrun,leaseId=${lease.leaseId}` }, ]); - assert.deepEqual(limrun.iosDelete.mock.calls.at(-1), ['ios-instance-recovered']); + assert.deepEqual(limrun.iosDelete.mock.calls.at(-1), [ + 'ios-instance-recovered', + { maxRetries: 0 }, + ]); assert.equal(fs.existsSync(pendingPath), false); } finally { dateNow.mockRestore();