diff --git a/packages/provider-webdriver/src/profile-fields.fixtures.ts b/packages/provider-webdriver/src/profile-fields.fixtures.ts index 2acc0b6bed..bd53b63379 100644 --- a/packages/provider-webdriver/src/profile-fields.fixtures.ts +++ b/packages/provider-webdriver/src/profile-fields.fixtures.ts @@ -40,3 +40,15 @@ export function consumedProfileFields({ fields }: ProviderProfileFieldDeclaratio .filter((field) => fields[field] === 'consumed') .sort(); } + +/** A hub that reads none of the profile fields, so every one of them is refused. */ +export function refuseAllProfileFields(provider: string): ProviderProfileFieldDeclaration { + const consumed = consumeAllProfileFields(provider); + return { + provider, + label: consumed.label, + fields: Object.fromEntries( + (Object.keys(consumed.fields) as ProviderProfileField[]).map((field) => [field, 'refused']), + ) as Record, + }; +} diff --git a/packages/provider-webdriver/src/runtime-session-allocation.test.ts b/packages/provider-webdriver/src/runtime-session-allocation.test.ts new file mode 100644 index 0000000000..4b5fcf9e9d --- /dev/null +++ b/packages/provider-webdriver/src/runtime-session-allocation.test.ts @@ -0,0 +1,973 @@ +import assert from 'node:assert/strict'; +import { afterEach, test } from 'vitest'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { createCloudWebDriverRuntime, type CloudWebDriverRuntimeOptions } from './runtime.ts'; +import { consumeAllProfileFields, refuseAllProfileFields } from './profile-fields.fixtures.ts'; + +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); + +/** + * `POST /session` is billed and the lease is the only handle that can stop it again. Two allocations + * that overlapped both passed the live-session check, so the second hub session overwrote the first in + * the manager's map and only one of the two was ever releasable. + */ +test('two concurrent allocations of one lease create one billed session', async () => { + const hub = startWebDriverHub(); + const preparation = startProviderPreparation(); + const runtime = makeRuntime({ prepareSession: preparation.prepareSession }); + const lease = makeLease(); + + try { + globalThis.fetch = hub.fetch; + const first = settle(() => runtime.leaseLifecycle.allocate!(lease)); + await preparation.waitFor(1); + const second = settle(() => runtime.leaseLifecycle.allocate!(lease)); + // One scheduler boundary while the first allocation is still held before the hub: the second has + // then reached the lease, and the first cannot have published its session yet. + await schedulerYield(); + + const [firstOutcome, secondOutcome] = await Promise.all([ + first, + second, + preparation.answerHeld(), + ]); + assert.equal(hub.created.length, 1, 'one lease bills one session'); + assertResolvedWith(firstOutcome, 'session-1'); + assertResolvedWith(secondOutcome, 'session-1'); + + await runtime.leaseLifecycle.release!(lease); + assert.deepEqual(hub.deleted, hub.created); + assert.equal(preparation.cleanups, 1); + } finally { + await preparation.answerHeld(); + await runtime.shutdown(); + } +}); + +/** + * An allocation that cannot be honoured is refused for its own flags, and one that joins a create in + * flight is refused for those flags without taking the create's caller down with it: the flags are the + * caller's, and the session belongs to whoever the lease was first allocated for. + */ +test('an allocation refused for its own flags leaves the create it joined alone', async () => { + const hub = startWebDriverHub(); + const preparation = startProviderPreparation(); + // This runtime reads none of the Cloud profile fields, so the flags below are refused. + const runtime = makeRuntime({ + prepareSession: preparation.prepareSession, + profileFields: refuseAllProfileFields('webdriver-test'), + }); + const lease = makeLease(); + // The flags bag is keyed by profile field, and `providerApp` is one this runtime refuses. + const refusedFlags = { providerApp: 'binary-123' }; + + try { + globalThis.fetch = hub.fetch; + const valid = settle(() => runtime.leaseLifecycle.allocate!(lease)); + await preparation.waitFor(1); + const refused = settle(() => + runtime.leaseLifecycle.allocate!(lease, { flags: refusedFlags } as never), + ); + await schedulerYield(); + await preparation.answerHeld(); + + assertRejectedWith(await refused, 'INVALID_ARGS', { flags: ['--provider-app'] }); + assertResolvedWith(await valid, 'session-1'); + assert.equal(hub.created.length, 1, 'a refused allocation bills nothing'); + } finally { + await preparation.answerHeld(); + await runtime.shutdown(); + } +}); + +/** + * A new allocation publishes its entry before it can await, and its own flags are the create's first + * step, so a call this provider cannot honour still bills nothing and never reaches preparation. + */ +test('an allocation refused for its own flags bills nothing before it reaches the provider', async () => { + const hub = startWebDriverHub(); + const preparation = startProviderPreparation(); + const runtime = makeRuntime({ + prepareSession: preparation.prepareSession, + profileFields: refuseAllProfileFields('webdriver-test'), + }); + const lease = makeLease(); + + try { + globalThis.fetch = hub.fetch; + assertRejectedWith( + await settle(() => + runtime.leaseLifecycle.allocate!(lease, { flags: { providerApp: 'b' } } as never), + ), + 'INVALID_ARGS', + { flags: ['--provider-app'] }, + ); + assert.deepEqual(hub.created, [], 'a refused allocation never asks the hub to bill a device'); + assert.equal(preparation.started, 0, 'a refused allocation never reaches the provider'); + } finally { + await runtime.shutdown(); + } +}); + +/** + * A release that landed mid-creation found no published session and answered that the lease held + * nothing, so the hub session that creation went on to make had no route left to stop it. The caller + * whose session was disposed is refused, rather than handed a session id nothing will release. + */ +test('a release during creation disposes the session that creation makes', async () => { + const hub = startWebDriverHub(); + const preparation = startProviderPreparation(); + const runtime = makeRuntime({ prepareSession: preparation.prepareSession }); + const lease = makeLease(); + + try { + globalThis.fetch = hub.fetch; + const allocating = settle(() => runtime.leaseLifecycle.allocate!(lease)); + await preparation.waitFor(1); + // The release takes the lease in its first synchronous step, while the create is held. + const releasing = settle(() => runtime.leaseLifecycle.release!(lease)); + await preparation.answerHeld(); + + assert.deepEqual(await releasing, { + status: 'resolved', + value: { provider: 'webdriver-test', providerSessionId: 'session-1' }, + }); + assert.deepEqual(hub.deleted, ['session-1']); + assert.equal(preparation.cleanups, 1); + assertRejectedWith(await allocating, 'provider_session_released_during_allocation', { + leaseId: lease.leaseId, + providerSessionId: 'session-1', + cleanup: 'confirmed', + }); + } finally { + await preparation.answerHeld(); + await runtime.shutdown(); + } +}); + +/** The same race on the route that ends every session this process holds. */ +test('shutdown during creation disposes the session that creation makes', async () => { + const hub = startWebDriverHub(); + const preparation = startProviderPreparation(); + const runtime = makeRuntime({ prepareSession: preparation.prepareSession }); + const lease = makeLease(); + + try { + globalThis.fetch = hub.fetch; + const allocating = settle(() => runtime.leaseLifecycle.allocate!(lease)); + await preparation.waitFor(1); + // Shutdown takes every lease in its first synchronous step, while the create is held. + const shuttingDown = settle(() => runtime.shutdown()); + await preparation.answerHeld(); + + assert.deepEqual(await shuttingDown, { status: 'resolved', value: undefined }); + assert.deepEqual(hub.deleted, ['session-1']); + assert.equal(preparation.cleanups, 1); + assertRejectedWith(await allocating, 'provider_session_shutdown_during_allocation', { + leaseId: lease.leaseId, + providerSessionId: 'session-1', + cleanup: 'confirmed', + }); + } finally { + await preparation.answerHeld(); + await runtime.shutdown(); + } +}); + +/** + * Taking a lease out of the map to dispose it opened the lease up again: an allocation that arrived + * while that disposal still waited on the hub saw nothing holding the lease and rented a second session + * the release could never see, and a shutdown that arrived in the same window answered with the first + * session live. Until the disposal settles, the lease belongs to the route that is ending it. + */ +test('an allocation that arrives during a release still disposing creates no second session', async () => { + // The hub keeps the release's DELETE pending, which is the window this test needs. + const hub = startWebDriverHub({ heldDeletes: 1 }); + const preparation = startProviderPreparation(); + const runtime = makeRuntime({ prepareSession: preparation.prepareSession }); + const lease = makeLease(); + + try { + globalThis.fetch = hub.fetch; + const allocating = settle(() => runtime.leaseLifecycle.allocate!(lease)); + await preparation.waitFor(1); + await preparation.answerHeld(); + assertResolvedWith(await allocating, 'session-1'); + + const releasing = settle(() => runtime.leaseLifecycle.release!(lease)); + const racing = settle(() => runtime.leaseLifecycle.allocate!(lease)); + await schedulerYield(); + assert.equal( + hub.created.length, + 1, + 'an allocation that arrives mid-release must not bill a second session', + ); + assert.equal(preparation.started, 1, 'an allocation refused mid-release prepares nothing'); + + const shuttingDown = settle(() => runtime.shutdown()); + await schedulerYield(); + // The release is still waiting on the hub, so neither it nor the shutdown may have answered. + assert.deepEqual(hub.deleted, []); + assert.equal(await alreadySettled(shuttingDown), false, 'shutdown must await the release'); + + hub.answerHeldDeletes(); + assertRejectedWith(await racing, 'provider_session_release_in_progress', { + leaseId: lease.leaseId, + providerSessionId: 'session-1', + }); + // Shutdown joined the disposal that was already running, so no session of this provider outlives it. + assert.deepEqual(await shuttingDown, { status: 'resolved', value: undefined }); + assert.deepEqual(hub.deleted, ['session-1']); + assert.equal(preparation.cleanups, 1); + assert.deepEqual(await releasing, { + status: 'resolved', + value: { provider: 'webdriver-test', providerSessionId: 'session-1' }, + }); + } finally { + hub.answerHeldDeletes(); + await preparation.answerHeld(); + await runtime.shutdown(); + } +}); + +/** + * A repeated allocation used to answer for the live session and then await, which gave a release room to + * end that session before the answer reached the caller, who was told the lease was alive. Its flags are + * validated first and the answer that follows the lookup carries no await, so a release that gets there + * first is the outcome the caller gets instead. + */ +test('a repeated allocation cannot answer that a released lease is alive', async () => { + // The hub holds the release's DELETE, so the release is certainly still ending the session at the + // moment the repeat allocation answers. + const hub = startWebDriverHub({ heldDeletes: 1 }); + globalThis.fetch = hub.fetch; + const runtime = makeRuntime(); + const lease = makeLease(); + + try { + await runtime.leaseLifecycle.allocate!(lease); + assert.deepEqual(hub.created, ['session-1']); + // The healthy case first: a repeat allocation of a live lease reports that lease as held. + assert.deepEqual(await runtime.leaseLifecycle.allocate!(lease), { + provider: 'webdriver-test', + }); + + const releasing = settle(() => runtime.leaseLifecycle.release!(lease)); + const repeated = settle(() => runtime.leaseLifecycle.allocate!(lease)); + assertRejectedWith(await repeated, 'provider_session_release_in_progress', { + leaseId: lease.leaseId, + providerSessionId: 'session-1', + cleanup: 'pending', + }); + assert.equal(hub.created.length, 1, 'a refused repeat allocation bills nothing'); + + hub.answerHeldDeletes(); + assert.deepEqual(await releasing, { + status: 'resolved', + value: { provider: 'webdriver-test', providerSessionId: 'session-1' }, + }); + assert.deepEqual(hub.deleted, ['session-1']); + } finally { + hub.answerHeldDeletes(); + await runtime.shutdown(); + } +}); + +/** + * The same window on the route that ends every session: a shutdown that lands first is the outcome a + * repeat allocation reports, rather than a success with no session behind it. + */ +test('a repeated allocation cannot answer that a shut down lease is alive', async () => { + const hub = startWebDriverHub({ heldDeletes: 1 }); + globalThis.fetch = hub.fetch; + const runtime = makeRuntime(); + const lease = makeLease(); + + try { + await runtime.leaseLifecycle.allocate!(lease); + assert.deepEqual(hub.created, ['session-1']); + + const shuttingDown = settle(() => runtime.shutdown()); + const repeated = settle(() => runtime.leaseLifecycle.allocate!(lease)); + assertRejectedWith(await repeated, 'provider_session_shutdown_during_allocation', { + leaseId: lease.leaseId, + providerSessionId: 'session-1', + cleanup: 'pending', + }); + assert.equal(hub.created.length, 1, 'a refused repeat allocation bills nothing'); + + hub.answerHeldDeletes(); + assert.deepEqual(await shuttingDown, { status: 'resolved', value: undefined }); + assert.deepEqual(hub.deleted, ['session-1']); + } finally { + hub.answerHeldDeletes(); + await runtime.shutdown(); + } +}); + +/** + * An allocation reaches its first await before provider preparation begins, and a release issued in + * that moment used to find nothing holding the lease and answer that it had released it. The create that + * went on to run then billed a session no route was left to stop. The lease has to belong to the + * allocation from the moment it is asked for, so the release ends up taking it and disposing what the + * create rents. + */ +test('a release issued before provider preparation begins disposes the session the allocation rents', async () => { + const hub = startWebDriverHub(); + const preparation = startProviderPreparation(); + const runtime = makeRuntime({ prepareSession: preparation.prepareSession }); + const lease = makeLease(); + + try { + globalThis.fetch = hub.fetch; + // Both calls are issued in one synchronous step, so the release reaches the manager while the + // allocation is at its first await and provider preparation has not answered. + const allocating = settle(() => runtime.leaseLifecycle.allocate!(lease)); + const releasing = settle(() => runtime.leaseLifecycle.release!(lease)); + assert.deepEqual(hub.created, [], 'nothing may be billed before preparation answers'); + // The allocation has reached provider preparation, which is still held. + await preparation.waitFor(1); + await preparation.answerHeld(); + + assert.deepEqual(await releasing, { + status: 'resolved', + value: { provider: 'webdriver-test', providerSessionId: 'session-1' }, + }); + assertRejectedWith(await allocating, 'provider_session_released_during_allocation', { + leaseId: lease.leaseId, + providerSessionId: 'session-1', + cleanup: 'confirmed', + }); + // The session the create rented is gone: a release that reported success leaves no billed session. + assert.deepEqual(hub.deleted, ['session-1']); + } finally { + await preparation.answerHeld(); + await runtime.shutdown(); + } +}); + +/** + * Ending a session ends the billed resource and hands back what the provider prepared; the artifact + * lookup that follows is only what the release envelope reports. A plugin's own artifact callback has + * no bound of its own, so it may not be what keeps a lease from being allocated again, what keeps a + * second release from answering, or what a daemon close waits through. + */ +test('an artifact callback that never answers holds neither the lease nor shutdown', async () => { + const hub = startWebDriverHub(); + globalThis.fetch = hub.fetch; + let answerArtifacts: (() => void) | undefined; + const lookupStarted = startSignal(); + const runtime = makeRuntime({ + providerCallbackTimeoutMs: 40, + listArtifacts: async () => { + lookupStarted.report(); + await new Promise((ready) => { + answerArtifacts = ready; + }); + return { provider: 'webdriver-test', status: 'ready', cloudArtifacts: [] }; + }, + }); + const lease = makeLease(); + + try { + await runtime.leaseLifecycle.allocate!(lease); + const releasing = settle(() => runtime.leaseLifecycle.release!(lease)); + // The lookup starts only once the hub has ended the session and the provider's cleanup has answered, + // so its having started is the moment the billed work is finished. + await awaitSignal(lookupStarted.arrived, 'the artifact lookup to start'); + assert.deepEqual(hub.deleted, ['session-1']); + await schedulerYield(); + + // A re-allocation during the hanging lookup is the lease's own business, not the lookup's. + const reallocated = settle(() => runtime.leaseLifecycle.allocate!(lease)); + assertResolvedWith(await reallocated, 'session-2', describe(await reallocated)); + const shuttingDown = settle(() => runtime.shutdown()); + assert.deepEqual( + await shuttingDown, + { status: 'resolved', value: undefined }, + 'shutdown may not wait on an optional artifact lookup', + ); + + const result = await releasing; + assert.equal(result.status, 'resolved', describe(result)); + const value = (result as Readonly<{ value: Record | undefined }>).value; + assert.equal(value?.providerSessionId, 'session-1'); + // What the lookup could not answer for becomes an unavailable report, the way its failures do. + assert.equal( + (value?.cloudArtifacts as Record | undefined)?.status, + 'unavailable', + ); + assert.deepEqual(hub.created, ['session-1', 'session-2']); + assert.deepEqual(hub.deleted, ['session-1', 'session-2']); + } finally { + answerArtifacts?.(); + await runtime.shutdown(); + } +}); + +/** + * A provider's own cleanup is the half of disposal the hub's `DELETE` does not cover, and a custom one + * has no bound either. Once its budget runs out this manager cannot learn whether the resource was + * handed back, so it says so and lets the lease be used again rather than holding it forever. + */ +test('a provider cleanup that outlives its budget ends the disposal as uncertain', async () => { + const hub = startWebDriverHub(); + globalThis.fetch = hub.fetch; + let answerCleanup: (() => void) | undefined; + let cleanupCalls = 0; + const runtime = makeRuntime({ + providerCallbackTimeoutMs: 40, + prepareSession: async ({ base }) => ({ + ...base, + cleanup: async () => { + cleanupCalls += 1; + if (cleanupCalls > 1) return undefined; + await new Promise((ready) => { + answerCleanup = ready; + }); + return undefined; + }, + }), + }); + const lease = makeLease(); + + try { + await runtime.leaseLifecycle.allocate!(lease); + const releasing = settle(() => runtime.leaseLifecycle.release!(lease)); + await hub.deletedArrived('session-1'); + + const result = await releasing; + assert.equal(result.status, 'resolved', describe(result)); + const value = (result as Readonly<{ value: Record | undefined }>).value; + assert.deepEqual(warningCodes(value?.warnings), ['PROVIDER_CLEANUP_FAILED']); + assert.match(String(warningMessages(value?.warnings)[0]), /40ms/); + + // The lease is the next caller's again, and a later session this manager can end still ends cleanly. + assertResolvedWith( + await settle(() => runtime.leaseLifecycle.allocate!(lease)), + 'session-2', + 'a cleanup whose outcome is unknown may not keep the lease', + ); + assert.deepEqual( + await settle(() => runtime.shutdown()), + { status: 'resolved', value: undefined }, + 'only a session the provider would not end may fail a close', + ); + assert.deepEqual(hub.created, ['session-1', 'session-2']); + assert.deepEqual(hub.deleted, ['session-1', 'session-2']); + } finally { + answerCleanup?.(); + await runtime.shutdown(); + } +}); + +/** + * A hub that will not end a session leaves it billing, so the release that was told to end it names the + * session and the provider's refusal, exactly as it does for a session it was handed ready-made. + */ +test('a release the provider refused reports the session left running', async () => { + const hub = startWebDriverHub({ deleteFailure: 'provider rejected the delete' }); + globalThis.fetch = hub.fetch; + const runtime = makeRuntime(); + const lease = makeLease(); + + try { + await runtime.leaseLifecycle.allocate!(lease); + const releasing = settle(() => runtime.leaseLifecycle.release!(lease)); + + const result = await releasing; + assert.equal(result.status, 'resolved', describe(result)); + const value = (result as Readonly<{ value: Record | undefined }>).value; + assert.equal(value?.providerSessionId, 'session-1'); + assert.deepEqual(warningCodes(value?.warnings), ['WEBDRIVER_SESSION_DELETE_FAILED']); + assert.match(String(warningMessages(value?.warnings)[0]), /provider rejected the delete/); + assert.deepEqual(hub.deleted, []); + } finally { + hub.allowDeletes(); + await runtime.shutdown(); + } +}); + +/** + * A create the hub refused still hands back what the provider prepared, and that hand-back is the + * plugin's own code too. A `cleanup` that never answers there would keep the allocation from answering, + * with the lease held behind it and nothing on the lease left to end, so it gets the same budget the + * disposal's cleanup gets. + */ +test('a create the hub refused still answers when its provider cleanup hangs', async () => { + const hub = startWebDriverHub({ createFailure: 'hub refused the create' }); + globalThis.fetch = hub.fetch; + let answerCleanup: (() => void) | undefined; + const runtime = makeRuntime({ + providerCallbackTimeoutMs: 40, + prepareSession: async ({ base }) => ({ + ...base, + cleanup: async () => { + await new Promise((ready) => { + answerCleanup = ready; + }); + return undefined; + }, + }), + }); + const lease = makeLease(); + + try { + const allocating = settle(() => runtime.leaseLifecycle.allocate!(lease)); + // The create failed and its cleanup is the step that will not answer, so the allocation cannot be + // left waiting on it. + const outcome = await allocating; + assert.equal( + outcome.status, + 'rejected', + `the refused create still reports the hub's own refusal, got ${describe(outcome)}`, + ); + const error = (outcome as Readonly<{ error: AppError }>).error; + const details = (error.details ?? {}) as Record; + assert.equal(details.status, 500, 'the hub refusal is what the caller is told'); + assert.match(String(details.cleanupError), /did not answer within 40ms/); + + // The lease is the next caller's again: an allocation issued after the hung cleanup reaches the hub + // and is refused for the hub's own refusal, not for a teardown that never finishes. And a close does + // not wait on a callback it gave up on. + const nextAllocation = await settle(() => runtime.leaseLifecycle.allocate!(lease)); + assert.equal(nextAllocation.status, 'rejected', describe(nextAllocation)); + const nextDetails = ((nextAllocation as Readonly<{ error: AppError }>).error.details ?? + {}) as Record; + assert.equal( + nextDetails.status, + 500, + 'the lease is free, so this caller hears the hub again rather than a contention refusal', + ); + assert.deepEqual( + await settle(() => runtime.shutdown()), + { status: 'resolved', value: undefined }, + 'a create cleanup this manager gave up on may not hold a close', + ); + assert.deepEqual(hub.created, []); + } finally { + answerCleanup?.(); + await runtime.shutdown(); + } +}); + +/** + * A close reports every session it left billing, and each session keeps its own reason in that report: + * two sessions the provider refused to end carry two refusals, and collapsing them by warning code would + * hand the operator one message for both. + */ +test('a shutdown that could not stop two sessions names both refusals', async () => { + const hub = startWebDriverHub({ deleteFailure: 'provider rejected the delete' }); + globalThis.fetch = hub.fetch; + const runtime = makeRuntime(); + const first = { ...makeLease(), leaseId: 'lease-a' }; + const second = { ...makeLease(), leaseId: 'lease-b' }; + + try { + await runtime.leaseLifecycle.allocate!(first); + await runtime.leaseLifecycle.allocate!(second); + assert.deepEqual(hub.created, ['session-1', 'session-2']); + + const shuttingDown = await settle(() => runtime.shutdown()); + assert.equal( + shuttingDown.status, + 'rejected', + 'a close that left two sessions billing is not a clean close', + ); + const details = ((shuttingDown as Readonly<{ error: AppError }>).error.details ?? {}) as Record< + string, + unknown + >; + assert.equal(details.reason, 'provider_session_cleanup_incomplete'); + assert.deepEqual(details.providerSessionIds, ['session-1', 'session-2']); + // Each named session carries the reason this provider gave for it, so an operator can tell the two + // refusals apart instead of being handed one message for both sessions. + const warnings = details.cleanupWarnings as Array>; + assert.deepEqual( + warnings.map((warning) => [warning.providerSessionId, warning.code]), + [ + ['session-1', 'WEBDRIVER_SESSION_DELETE_FAILED'], + ['session-2', 'WEBDRIVER_SESSION_DELETE_FAILED'], + ], + ); + assert.deepEqual( + warnings.map((warning) => /session-\d/.exec(String(warning.message))?.[0]), + ['session-1', 'session-2'], + ); + } finally { + hub.allowDeletes(); + await runtime.shutdown(); + } +}); + +/** + * A shutdown that could not stop a session may not report a clean close, and the allocation whose + * session it was is told the outcome is uncertain rather than that nothing needs cleaning up. + */ +test('a session shutdown could not stop fails the shutdown and the refused allocation', async () => { + const hub = startWebDriverHub({ deleteFailure: 'provider rejected the delete' }); + const preparation = startProviderPreparation(); + const runtime = makeRuntime({ prepareSession: preparation.prepareSession }); + const lease = makeLease(); + + try { + globalThis.fetch = hub.fetch; + const allocating = settle(() => runtime.leaseLifecycle.allocate!(lease)); + await preparation.waitFor(1); + const shuttingDown = settle(() => runtime.shutdown()); + await preparation.answerHeld(); + + assertRejectedWith(await allocating, 'provider_session_shutdown_during_allocation', { + leaseId: lease.leaseId, + providerSessionId: 'session-1', + cleanup: 'uncertain', + cleanupWarning: 'WEBDRIVER_SESSION_DELETE_FAILED', + }); + // A daemon close reports a plugin that left work behind, so the session and the provider's refusal + // have to travel on the error rather than be dropped. + assertRejectedWith(await shuttingDown, 'provider_session_cleanup_incomplete', { + providerSessionIds: ['session-1'], + cleanup: 'uncertain', + cleanupWarning: 'WEBDRIVER_SESSION_DELETE_FAILED', + }); + // The create-retry policy stays: nothing here deletes or creates a second session on its own. + assert.deepEqual(hub.created, ['session-1']); + assert.deepEqual(hub.deleted, []); + assert.equal(preparation.cleanups, 1); + } finally { + hub.allowDeletes(); + await preparation.answerHeld(); + await runtime.shutdown(); + } +}); + +/** + * The hub answers every WebDriver request and records the session ids it was asked to create and + * delete, so a test's levers over an in-flight create and an in-flight disposal are + * {@link startProviderPreparation} and the `heldDeletes` option. + */ +function startWebDriverHub( + options: Readonly<{ + deleteFailure?: string; + heldDeletes?: number; + createFailure?: string; + }> = {}, +) { + const created: string[] = []; + const deleted: string[] = []; + let deletesToHold = options.heldDeletes ?? 0; + let deleteFailure = options.deleteFailure; + const held: (() => void)[] = []; + /** Per-session arrival so a test can await the hub ending one rather than watching the clock. */ + const deleteSignals = new Map>(); + const signalForDelete = (sessionId: string) => { + const existing = deleteSignals.get(sessionId) ?? startSignal(); + deleteSignals.set(sessionId, existing); + return existing; + }; + return { + created, + deleted, + /** Resolves once the hub has answered a `DELETE` for `sessionId`. */ + deletedArrived: async (sessionId: string) => { + await awaitSignal(signalForDelete(sessionId).arrived, `the hub to end session ${sessionId}`); + }, + allowDeletes: () => { + deleteFailure = undefined; + }, + /** Lets every held `DELETE` finish, which is how a release that is still disposing resumes. */ + answerHeldDeletes: () => { + for (const ready of held.splice(0)) ready(); + }, + fetch: (async (input: unknown, init?: RequestInit) => { + const request = new Request(input as RequestInfo, init); + const path = new URL(request.url).pathname; + if (request.method === 'POST' && path.endsWith('/session')) { + if (options.createFailure) { + return jsonResponse( + { value: { error: 'unknown error', message: `${options.createFailure}` } }, + 500, + ); + } + const sessionId = `session-${String(created.length + 1)}`; + created.push(sessionId); + return jsonResponse({ sessionId, capabilities: {} }); + } + if (request.method === 'DELETE') { + const sessionId = decodeURIComponent(path.split('/').pop()!); + if (deletesToHold > 0) { + deletesToHold -= 1; + await new Promise((ready) => { + held.push(ready); + }); + } + if (deleteFailure) { + return jsonResponse( + { value: { error: 'unknown error', message: `${deleteFailure}: ${sessionId}` } }, + 500, + ); + } + deleted.push(sessionId); + signalForDelete(sessionId).report(); + } + return jsonResponse({ value: null }); + }) as typeof fetch, + }; +} + +type PrepareSession = NonNullable; + +/** + * How long an arrival wait gives a step that has not happened at all. It is a watchdog, not a poll + * budget: the step that arrives resolves its own signal in that same step, however slow the worker is, + * so this only ever elapses on a manager or a seam that never reached the step. + */ +const ARRIVAL_BUDGET_MS = 2_000; + +/** + * Waits for a step to be reported — a create that reached the provider, the hub asked to delete a + * session, a plugin callback that was entered — rather than polling the clock for it. The timer only + * names a step that never came, so a slow worker cannot fail a test whose allocation is progressing. + */ +async function awaitSignal(signal: Promise, step: string): Promise { + let timer: ReturnType | undefined; + try { + const arrived = await Promise.race([ + signal.then(() => true), + new Promise((giveUp) => { + timer = setTimeout(() => giveUp(false), ARRIVAL_BUDGET_MS); + timer.unref?.(); + }), + ]); + assert.ok(arrived, `the manager never reached ${step}`); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +/** + * A step a test waits to be reported, reported once by the seam that reaches it. + */ +function startSignal() { + let reportArrival!: () => void; + const arrived = new Promise((ready) => { + reportArrival = ready; + }); + let reported = false; + return { + arrived, + report: () => { + if (reported) return; + reported = true; + reportArrival(); + }, + }; +} + +/** + * `prepareSession` is the step that runs before this manager sends `POST /session`, so holding it per + * call keeps a create in flight across a release or a shutdown without racing one with a sleep. It + * counts the creates that reached the provider — the difference between one billed session and two — + * and the provider cleanup a prepared session owes, which is the half of disposal the hub's `DELETE` + * does not cover. + */ +function startProviderPreparation() { + const state = { started: 0, cleanups: 0 }; + const calls: { readonly released: boolean; readonly release: () => void }[] = []; + /** Who is waiting to hear that a create has reached the provider. */ + const waiting: (() => void)[] = []; + const prepareSession: PrepareSession = async ({ base }) => { + state.started += 1; + const call = { released: false, release: () => {} }; + const held = new Promise((ready) => { + call.release = () => { + if (call.released) return; + call.released = true; + ready(); + }; + }); + calls.push(call); + // This create has arrived, so anyone waiting for one hears it in this same step rather than on the + // next round of a poll. + for (const announce of waiting.splice(0)) announce(); + await held; + return { + ...base, + cleanup: async () => { + state.cleanups += 1; + return undefined; + }, + }; + }; + return Object.assign(state, { + prepareSession, + /** + * Resolves once `expected` allocations have reached the provider and are held there. Each wait is + * ended by the arrival itself, so a test neither polls the scheduler nor races the create; the one + * timer only reports a create that never arrived, and never elapses while the seam works. + */ + waitFor: async (expected: number) => { + while (calls.length < expected) { + await awaitSignal( + new Promise((announce) => { + waiting.push(announce); + }), + `${String(expected)} allocation(s) to reach the provider (got ${String(calls.length)})`, + ); + } + }, + /** Lets every create that is held reach the hub. */ + answerHeld: async () => { + for (const call of calls) call.release(); + await schedulerYield(); + }, + }); +} + +/** How one lease-lifecycle call ended, so a test can assert disposal before the caller's outcome. */ +type Settled = + | Readonly<{ status: 'resolved'; value: T }> + | Readonly<{ status: 'rejected'; error: unknown }>; + +/** Runs a call and turns its outcome into a value, so no rejection goes unobserved. */ +async function settle(run: () => Promise): Promise> { + try { + return { status: 'resolved', value: await run() }; + } catch (error) { + return { status: 'rejected', error }; + } +} + +/** Whether a call has answered, so a test can name work that must still be pending. */ +async function alreadySettled(outcome: Promise>): Promise { + return await Promise.race([ + outcome.then(() => true), + new Promise((pending) => { + setTimeout(() => pending(false), 0); + }), + ]); +} + +/** Yields one macrotask, so every promise the test has started runs the microtasks it already queued. */ +async function schedulerYield(): Promise { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); +} + +/** The allocation envelope names the hub session the lease now holds. */ +function assertResolvedWith( + outcome: Settled | undefined>, + sessionId: string, + expectation?: string, +): void { + assert.equal( + outcome.status, + 'resolved', + `${expectation ?? `expected ${sessionId}`}, got ${describe(outcome)}`, + ); + assert.equal( + (outcome as Readonly<{ value: Record | undefined }>).value?.sessionId, + sessionId, + ); +} + +/** + * What a refused call reports: its typed reason or error code, the lease and session it was refused + * for, and what the teardown that took the session came to. + */ +function assertRejectedWith( + outcome: Settled, + reason: string, + expected: Readonly<{ + leaseId?: string; + providerSessionId?: string; + providerSessionIds?: string[]; + cleanup?: string; + cleanupWarning?: string; + flags?: string[]; + }> = {}, +): void { + assert.equal(outcome.status, 'rejected', `expected ${reason}, got ${describe(outcome)}`); + const error = (outcome as Readonly<{ error: unknown }>).error; + assert.ok(error instanceof AppError, `expected an AppError, got ${String(error)}`); + const details = error.details as Record; + assert.equal(details.reason ?? error.code, reason); + if (expected.leaseId !== undefined) assert.equal(details.leaseId, expected.leaseId); + if (expected.providerSessionId !== undefined) { + assert.equal(details.providerSessionId, expected.providerSessionId); + } + if (expected.providerSessionIds !== undefined) { + assert.deepEqual(details.providerSessionIds, expected.providerSessionIds); + } + if (expected.cleanup !== undefined) assert.equal(details.cleanup, expected.cleanup); + if (expected.cleanupWarning !== undefined) { + // A refusal for one session reports its warnings keyed by code, which loses nothing because one + // session's warnings carry distinct codes; a shutdown reports every session it left billing as one + // entry per warning. Either way the caller is told which refusal it got. + const warnings = details.cleanupWarnings; + const codes = Array.isArray(warnings) + ? (warnings as Array>).map((warning) => warning.code) + : Object.keys((warnings ?? {}) as Record); + assert.deepEqual(codes, [expected.cleanupWarning]); + } + if (expected.flags !== undefined) assert.deepEqual(details.flags, expected.flags); +} + +function warningCodes(warnings: unknown): string[] { + return ((warnings as Array> | undefined) ?? []).map((warning) => + String(warning.code), + ); +} + +function warningMessages(warnings: unknown): unknown[] { + return ((warnings as Array> | undefined) ?? []).map( + (warning) => warning.message, + ); +} + +function describe(outcome: Settled): string { + return outcome.status === 'rejected' + ? `rejection: ${String(outcome.error)}` + : `resolution: ${JSON.stringify(outcome.value)}`; +} + +function makeRuntime(overrides: Partial = {}) { + return createCloudWebDriverRuntime({ + clientVersion: 'test', + provider: 'webdriver-test', + endpoint: 'https://webdriver.test/wd/hub/', + platform: 'android', + deviceName: 'Test device', + profileFields: consumeAllProfileFields('webdriver-test'), + requestPolicy: { retryAttempts: 0 }, + ...overrides, + }); +} + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function makeLease(): DeviceLease { + return { + leaseId: 'lease-1', + tenantId: 'team-a', + runId: 'run-a', + clientId: 'client-a', + leaseProvider: 'webdriver-test', + backend: 'android-instance', + deviceKey: 'webdriver-test:device-a', + createdAt: 1, + expiresAt: 2, + heartbeatAt: 1, + }; +} diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index 4425f761b6..8e91b83f9e 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -38,21 +38,86 @@ type CloudWebDriverReleaseWarning = Readonly<{ code: 'WEBDRIVER_SESSION_DELETE_FAILED' | 'PROVIDER_CLEANUP_FAILED'; message: string; }>; -type CloudWebDriverCloseResult = Readonly<{ - cleanup?: Record; +type LeaseResult = Record | undefined; + +/** + * What one lease has in this manager: the create the hub has not answered, the session that create + * rented, or the disposal that is ending it. This is the only place a billed session is recorded, so + * every route that can end one — release, shutdown, and the allocation still waiting for the create — + * reads it here. Entries never leave this module, so their fields stay mutable. + * + * The entry in the map owns the lease's session, and it keeps owning it while it is being ended: + * `taken` is set in the same synchronous step that starts the disposal, and the entry leaves the map + * only once that disposal settles. So a lease is never between owners while its session may be live — + * an allocation that arrives mid-release is refused, a second release joins the disposal in flight, and + * shutdown awaits it instead of starting another. + * + * An allocation reads its own refusal from the `taken` of the entry it joined, so the reason names the + * teardown that actually happened rather than whatever state the manager is in by the time the create + * answers, and it carries what that disposal came to. + */ +type WebDriverLeaseEntry = { + lease: DeviceLease; + /** Settles with the session the hub rented, or rejects when the create billed nothing. */ + creation: Promise; + /** What the create rented. Missing while the create is still running, and never set when it failed. */ + session: WebDriverProviderSession | undefined; + /** Which route took this lease from its allocation, and the one disposal that owns its session. */ + taken: WebDriverLeaseTake | undefined; +}; + +/** + * Whichever route took a lease away from its allocation, and the single disposal of what that entry + * stood for. `settled` is what that disposal reported and stays missing while it is still running, so a + * caller can tell a session that is gone from one whose outcome it cannot see yet. `artifacts` is the + * release envelope's report and is deliberately separate: releasing the lease slot and `shutdown()` + * await only disposal. A release also awaits artifacts, including when it joins a shutdown's take. + */ +type WebDriverLeaseTake = { + by: 'release' | 'shutdown'; + /** The session being ended, known as soon as the create named it. */ + providerSessionId: string | undefined; + disposal: Promise; + settled: WebDriverDisposalOutcome | undefined; + /** The artifact report, started once the billed resources are settled, and joined by the release. */ + artifacts: Promise | undefined; +}; + +/** + * What ending one lease's session came to. `cleanup` answers whether a billed session is certainly + * gone: `confirmed` once the hub ended it, `uncertain` when ending it failed, and `none` when the + * create rented nothing. + */ +type WebDriverDisposalOutcome = Readonly<{ + cleanup: 'confirmed' | 'uncertain' | 'none'; + session: WebDriverProviderSession | undefined; warnings: CloudWebDriverReleaseWarning[]; + /** What the provider's own cleanup reported, for the release envelope. */ + cleanupPayload?: Record; }>; -type LeaseResult = Record | undefined; + +/** The fields of one warning, keyed as `details.cleanupWarnings` carries them on a refusal. */ +function warningMessagesByCode(warnings: CloudWebDriverReleaseWarning[]): Record { + return Object.fromEntries(warnings.map((warning) => [warning.code, warning.message])); +} + +/** + * Budget for prepared-session cleanup on disposal and failed creation, and for teardown artifact lookup. + * The preparation callback itself is not bounded by this budget. + */ +const DEFAULT_PROVIDER_CALLBACK_TIMEOUT_MS = 30_000; /** * Owns WebDriver session lifecycle and stale-owner retention. Deployment decisions stay in the * sibling runtime-deployment module so neither concern widens the public runtime façade. */ export class WebDriverSessionManager { - private readonly sessionsByLeaseId = new Map(); + private readonly entriesByLeaseId = new Map(); private readonly ownedDeviceIds = new Set(); private readonly releasedProviderSessionIdsByLeaseId = new Map(); private readonly options: CloudWebDriverRuntimeOptions; + /** Whether this manager has shut down; it can then hold nothing, so it must create nothing. */ + private shutDown = false; constructor(options: CloudWebDriverRuntimeOptions) { this.options = options; @@ -63,11 +128,11 @@ export class WebDriverSessionManager { } findSessionForDevice(device: DeviceInfo): WebDriverProviderSession | undefined { - return [...this.sessionsByLeaseId.values()].find((session) => session.device.id === device.id); + return this.liveSessions().find((session) => session.device.id === device.id); } findSessionForLease(leaseId: string): WebDriverProviderSession | undefined { - return this.sessionsByLeaseId.get(leaseId); + return this.liveSession(this.entriesByLeaseId.get(leaseId)); } getInteractor(device: DeviceInfo): Interactor | undefined { @@ -76,83 +141,68 @@ export class WebDriverSessionManager { heartbeat(lease: DeviceLease): Record | undefined { if (lease.leaseProvider !== this.options.provider) return undefined; - if (!this.sessionsByLeaseId.has(lease.leaseId)) return undefined; - return { provider: this.options.provider }; + if (!this.liveSession(this.entriesByLeaseId.get(lease.leaseId))) return undefined; + return this.liveLeaseEnvelope(); } async allocate(lease: DeviceLease, req?: LeaseLifecycleContext): Promise { if (lease.leaseProvider !== this.options.provider) return undefined; - // Before the reuse below: a repeat allocation of a live lease carries flags of its own. - // Loaded here rather than eagerly so the package entry's closure stays unchanged. - const { rejectRefusedProviderProfileFields } = - await import('@agent-device/contracts/provider-profile-fields'); - rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); - if (this.sessionsByLeaseId.has(lease.leaseId)) return this.heartbeat(lease); - const prepared = await this.prepareSession(lease, req); - const client = new WebDriverClient({ - clientVersion: this.options.clientVersion, - endpoint: prepared.endpoint, - auth: prepared.auth, - headers: prepared.headers, - requestPolicy: this.options.requestPolicy, - }); - const session = await this.createSessionWithPreparedCleanup(client, prepared, lease, req); - const device = this.deviceForLease(lease, prepared); - const providerSessionId = prepared.providerSessionId ?? session.sessionId; - const capabilities = createCloudWebDriverCapabilities({ - provider: this.options.provider, - platform: prepared.platform, - overrides: this.options.capabilityOverrides, - }); - this.sessionsByLeaseId.set( - lease.leaseId, - Object.freeze({ - lease, - device, - client, - prepared, - capabilities, - webDriverSessionId: session.sessionId, - providerSessionId, - interactor: createWebDriverInteractor({ - client, - backend: snapshotBackendForPlatform(prepared.platform), - capabilities, - targetId: device.id, - }), - }), - ); - this.ownedDeviceIds.add(device.id); - return { - provider: this.options.provider, - deviceId: device.id, - sessionId: session.sessionId, - providerSessionId, - capabilities, - ...prepared.providerData, - }; + const held = this.entriesByLeaseId.get(lease.leaseId); + // Joining someone else's entry, so this call answers for its own flags, and it settles them before it + // reads any state. Nothing else can await on this route: the lookup below is the last one, and the + // answer is read from it with no await after, so a release or a shutdown that lands while the flags + // are being checked is the outcome this call reports rather than a session already ended. + if (held !== undefined) await this.rejectRefusedProfileFields(req); + // Re-read after that await, and nothing awaits after this lookup except joining the create below, so + // a release or a shutdown that landed in the meantime is what this call reports. + let entry = this.entriesByLeaseId.get(lease.leaseId); + if (entry === undefined) { + if (this.shutDown) throw allocationRefusedAfterShutdown(this.options.provider, lease); + // Published before this call awaits anything, so a lease is never unowned while its allocation is at + // its first await: a release or a shutdown that arrives there finds this entry and takes it, and the + // create the entry stands on checks this call's own profile flags before it bills anything. + entry = this.beginAllocation(lease, req); + } + // A lease whose disposal is running belongs to the route that is ending it, so this call rented + // nothing and is refused for that disposal rather than for a session of its own. + if (entry.taken) throw refusalForTeardownInProgress(this.options.provider, lease, entry.taken); + // Reported without awaiting anything after the lookup, so the session is live at the moment the + // caller is told about it: a release or a shutdown cannot end it inside this call. + if (entry.session) return this.liveLeaseEnvelope(); + const session = await entry.creation; + // The session this call rented is its own only while nobody took the lease. Otherwise a release or + // shutdown ended what the create rented, and this caller is refused rather than handed a session id + // nothing will release. + if (entry.taken) throw await this.refusalForTakenSession(lease, entry.taken); + return this.allocationResult(session); } async release(lease: DeviceLease): Promise { if (lease.leaseProvider !== this.options.provider) return undefined; - const session = this.sessionsByLeaseId.get(lease.leaseId); - if (!session) return undefined; - this.sessionsByLeaseId.delete(lease.leaseId); - this.releasedProviderSessionIdsByLeaseId.set(lease.leaseId, session.providerSessionId); - const close = await this.closeSession(session); - const artifacts = await this.safeListArtifacts(session); + const entry = this.entriesByLeaseId.get(lease.leaseId); + // Nothing holds the lease: it was never allocated here, or its release finished and gave the lease + // back. A release that arrives while another one disposes joins that disposal. + if (entry === undefined) return undefined; + const take = this.take(entry, 'release'); + const outcome = await take.disposal; + // A create that failed billed nothing, and gave back what its preparation had already rented. + if (outcome.session === undefined) return undefined; + // The artifact report is joined here, after the lease has been given back and the billed session is + // known to be ended: a slow plugin lookup may delay this answer, and nothing that runs against the + // lease. + const artifacts = await take.artifacts; return { provider: this.options.provider, - providerSessionId: session.providerSessionId, - ...close.cleanup, - ...(close.warnings.length > 0 ? { warnings: close.warnings } : {}), + providerSessionId: outcome.session.providerSessionId, + ...outcome.cleanupPayload, + ...(outcome.warnings.length > 0 ? { warnings: outcome.warnings } : {}), ...(artifacts ? { cloudArtifacts: artifacts } : {}), }; } async listCloudArtifacts(query: CloudArtifactsQuery): Promise { if (query.provider !== this.options.provider) return undefined; - const session = this.sessionsByLeaseId.get(query.leaseId ?? ''); + const session = this.findSessionForLease(query.leaseId ?? ''); if (session) return await this.listActiveCloudArtifacts(session, query.providerSessionId); return await this.listReleasedCloudArtifacts(query); } @@ -177,12 +227,260 @@ export class WebDriverSessionManager { }); } + /** + * Ends every session this manager holds or is on its way to holding, and says so when it could not. + * Each lease is taken, or joined, in one synchronous step, so a create that answers afterwards is + * disposed by the one disposal of that entry and by no other route, and a release already running is + * awaited rather than raced. + */ async shutdown(): Promise { - await Promise.allSettled( - [...this.sessionsByLeaseId.values()].map(async (session) => await this.closeSession(session)), - ); - this.sessionsByLeaseId.clear(); + this.shutDown = true; + // Snapshot before anything is awaited: `allocate` publishes an entry only when none exists and + // refuses once this manager is shut down, and nothing between that flag and this loop awaits. + const takes = [...this.entriesByLeaseId.values()].map((entry) => this.take(entry, 'shutdown')); + const outcomes = await Promise.all(takes.map((take) => take.disposal)); this.ownedDeviceIds.clear(); + // A session the provider would not give back is still billing, and this process will not stop it. + // The plugin's close is where an operator learns what a daemon close left running, so the session + // and the provider's reason travel on the error rather than being dropped. + const unended = outcomes.filter((outcome) => outcome.cleanup === 'uncertain'); + if (unended.length > 0) throw shutdownLeftSessionsRunning(this.options.provider, unended); + } + + /** + * Publishes the entry one lease's create is standing on before that create can answer, so a duplicate + * allocation joins it and a release or shutdown that arrives mid-create has somewhere to find the + * session the create will rent. The entry records that session when the hub answers, and leaves the + * map when the create failed and billed nothing. + */ + private beginAllocation(lease: DeviceLease, req: LeaseLifecycleContext | undefined) { + let rentSession!: (session: WebDriverProviderSession) => void; + let refuseSession!: (error: unknown) => void; + const creation = new Promise((rent, refuse) => { + rentSession = rent; + refuseSession = refuse; + }); + const entry: WebDriverLeaseEntry = { lease, creation, session: undefined, taken: undefined }; + this.entriesByLeaseId.set(lease.leaseId, entry); + // Started after the publication above, so the create's answer can never arrive without an entry + // beside it. Whichever route took the entry in the meantime disposes the session this answers. + void this.createSessionForLease(lease, req).then( + (session) => { + entry.session = session; + this.ownedDeviceIds.add(session.device.id); + rentSession(session); + }, + (error: unknown) => { + // The create billed nothing and gave back what its preparation rented, so the lease is free + // again as soon as nobody is left to dispose anything for it. + if (entry.taken === undefined && this.entriesByLeaseId.get(lease.leaseId) === entry) { + this.entriesByLeaseId.delete(lease.leaseId); + } + refuseSession(error); + }, + ); + return entry; + } + + /** + * Prepares the lease and rents the hub session, and hands back the session this manager can dispose. + * The caller's own profile flags are refused here, first, so an allocation that published its entry + * before it could await anything still bills nothing for flags this provider reads no field for, and a + * release that took the entry meanwhile has nothing to end. + */ + private async createSessionForLease( + lease: DeviceLease, + req: LeaseLifecycleContext | undefined, + ): Promise { + await this.rejectRefusedProfileFields(req); + const prepared = await this.prepareSession(lease, req); + const client = new WebDriverClient({ + clientVersion: this.options.clientVersion, + endpoint: prepared.endpoint, + auth: prepared.auth, + headers: prepared.headers, + requestPolicy: this.options.requestPolicy, + }); + const session = await this.createSessionWithPreparedCleanup(client, prepared, lease, req); + const device = this.deviceForLease(lease, prepared); + const capabilities = createCloudWebDriverCapabilities({ + provider: this.options.provider, + platform: prepared.platform, + overrides: this.options.capabilityOverrides, + }); + return Object.freeze({ + lease, + device, + client, + prepared, + capabilities, + webDriverSessionId: session.sessionId, + providerSessionId: prepared.providerSessionId ?? session.sessionId, + interactor: createWebDriverInteractor({ + client, + backend: snapshotBackendForPlatform(prepared.platform), + capabilities, + targetId: device.id, + }), + }); + } + + /** + * Hands one lease's session to the one route that may end it. The take and the disposal it starts are + * recorded before this step returns, so no route can reach the lease in between and find it free, and + * the entry leaves the map once that disposal has settled — which is once the billed resources have + * been ended, not once every optional report about them has been collected. + */ + private take(entry: WebDriverLeaseEntry, by: 'release' | 'shutdown'): WebDriverLeaseTake { + const taken = entry.taken; + if (taken) return taken; + let reportDisposal!: (outcome: WebDriverDisposalOutcome) => void; + const disposal = new Promise((report) => { + reportDisposal = report; + }); + const take: WebDriverLeaseTake = { + by, + providerSessionId: entry.session?.providerSessionId, + disposal, + settled: undefined, + artifacts: undefined, + }; + entry.taken = take; + const settle = (outcome: WebDriverDisposalOutcome): void => { + take.settled = outcome; + take.providerSessionId = outcome.session?.providerSessionId; + // The artifact report starts once session deletion and provider cleanup settle. The lease slot is + // freed and `shutdown()` answers without awaiting it. A release still joins the report, even when + // shutdown started this take. + take.artifacts = + outcome.session === undefined + ? undefined + : this.safeListArtifacts(outcome.session, true).catch(() => undefined); + // Whatever the create rented is now ended, or was never rented, so this entry owns nothing and the + // lease is free again. + if (this.entriesByLeaseId.get(entry.lease.leaseId) === entry) { + this.entriesByLeaseId.delete(entry.lease.leaseId); + } + reportDisposal(outcome); + }; + // `disposeTakenEntry` reports its own failures, so the rejection branch is only for what it could not + // report at all. It still gives the lease back, because a disposal this manager lost sight of is + // reported, not waited on forever. + void this.disposeTakenEntry(entry).then(settle, (error: unknown) => { + settle({ + cleanup: 'uncertain', + session: entry.session, + warnings: [{ code: 'PROVIDER_CLEANUP_FAILED', message: errorMessage(error) }], + }); + }); + return take; + } + + /** + * Ends what one taken entry stands for: the session its create rented, once that create answers, or + * nothing when the create billed nothing. This is the only place a session is ended, and one entry is + * taken by exactly one route, so a session is ended once. After creation answers, session deletion and + * provider cleanup are bounded. Creation still awaits `prepareSession` without a callback budget. + */ + private async disposeTakenEntry(entry: WebDriverLeaseEntry): Promise { + const session = entry.session ?? (await entry.creation.catch(() => undefined)); + entry.session = session; + if (session === undefined) return { cleanup: 'none', session: undefined, warnings: [] }; + this.releasedProviderSessionIdsByLeaseId.set(entry.lease.leaseId, session.providerSessionId); + const warnings: CloudWebDriverReleaseWarning[] = []; + let cleanupPayload: Record | undefined; + try { + await session.client.deleteSession(); + } catch (error) { + warnings.push({ + code: 'WEBDRIVER_SESSION_DELETE_FAILED', + message: errorMessage(error), + }); + } + try { + cleanupPayload = await this.providerCallback( + `${this.options.provider} cleanup for lease ${entry.lease.leaseId}`, + () => session.prepared.cleanup?.(), + ); + } catch (error) { + warnings.push({ code: 'PROVIDER_CLEANUP_FAILED', message: errorMessage(error) }); + } + // Only a hub that answered can say the session is gone. A delete the provider refused leaves the + // session billing, and this manager has no way left to learn otherwise. + const cleanup = warnings.length > 0 ? 'uncertain' : 'confirmed'; + return { cleanup, session, warnings, cleanupPayload }; + } + + /** + * Awaits one provider callback the manager was handed, within the budget this package gives a provider + * round trip. Past it, the outcome is a refusal: the callback keeps running and this manager reports + * what it could not learn rather than holding the lease open for it. + */ + private async providerCallback( + label: string, + run: () => Promise | undefined, + ): Promise { + const timeoutMs = + this.options.providerCallbackTimeoutMs ?? DEFAULT_PROVIDER_CALLBACK_TIMEOUT_MS; + const pending = Promise.resolve(run()); + // A callback that was too slow is no longer awaited, so its late outcome must not surface as an + // unhandled rejection. + void pending.catch(() => undefined); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + pending, + new Promise((_resolve, refuse) => { + timer = setTimeout(() => { + refuse( + new AppError( + 'COMMAND_FAILED', + `${label} did not answer within ${String(timeoutMs)}ms.`, + ), + ); + }, timeoutMs); + timer.unref?.(); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + } + + /** + * The refusal for the allocation whose session a teardown took. The disposal's own answer is the only + * thing that can confirm a billed session is gone, so the refused caller waits for it rather than + * claiming an end it cannot see. + */ + private async refusalForTakenSession( + lease: DeviceLease, + take: WebDriverLeaseTake, + ): Promise { + const outcome = await take.disposal; + return refusalForTeardown(this.options.provider, lease, take.by, { + providerSessionId: outcome.session?.providerSessionId, + cleanup: outcome.cleanup, + warnings: outcome.warnings, + }); + } + + /** + * What this manager says about a lease it holds a live session for. `heartbeat` and the reuse answer of + * `allocate` share it, so the daemon's `recordProviderSession` sees one envelope from both routes. + */ + private liveLeaseEnvelope(): Record { + return { provider: this.options.provider }; + } + + private allocationResult(session: WebDriverProviderSession): Record { + return { + provider: this.options.provider, + deviceId: session.device.id, + sessionId: session.webDriverSessionId, + providerSessionId: session.providerSessionId, + capabilities: session.capabilities, + ...session.prepared.providerData, + }; } private async createSessionWithPreparedCleanup( @@ -199,11 +497,29 @@ export class WebDriverSessionManager { const failure = isWebDriverRequestTimeout(error) ? sessionCreateTimeoutError(error, this.options.provider, lease, prepared) : error; - await releaseOnFailure(failure, () => prepared.cleanup?.()); + // The hand-back is the plugin's own code, and no session exists yet for a release or a shutdown to + // end, so this is the one cleanup the allocation itself must not wait on forever. + await releaseOnFailure(failure, () => + this.providerCallback(`${this.options.provider} cleanup for lease ${lease.leaseId}`, () => + prepared.cleanup?.(), + ), + ); throw failure; } } + /** + * Refuses the flags this provider reads no profile field for, so a field that would be ignored is + * reported instead of dropped. Each allocation answers for its own flags: the one that published the + * entry is refused by its own create, before it can bill anything, and one that joined an entry is + * refused here. The module is loaded on first use to keep the package entry's own closure unchanged. + */ + private async rejectRefusedProfileFields(req: LeaseLifecycleContext | undefined): Promise { + const { rejectRefusedProviderProfileFields } = + await import('@agent-device/contracts/provider-profile-fields'); + rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); + } + private async prepareSession( lease: DeviceLease, req: LeaseLifecycleContext | undefined, @@ -249,40 +565,43 @@ export class WebDriverSessionManager { }; } - private async closeSession( - session: WebDriverProviderSession, - ): Promise { - const warnings: CloudWebDriverReleaseWarning[] = []; - let cleanup: Record | undefined; - try { - await session.client.deleteSession(); - } catch (error) { - warnings.push({ - code: 'WEBDRIVER_SESSION_DELETE_FAILED', - message: errorMessage(error), - }); - } - try { - cleanup = await session.prepared.cleanup?.(); - } catch (error) { - warnings.push({ code: 'PROVIDER_CLEANUP_FAILED', message: errorMessage(error) }); - } - return { cleanup, warnings }; + /** The session a lease holds and can still drive: rented, and not on its way to being ended. */ + private liveSession(entry: WebDriverLeaseEntry | undefined) { + return entry?.taken === undefined ? entry?.session : undefined; } + private liveSessions(): WebDriverProviderSession[] { + return [...this.entriesByLeaseId.values()].flatMap((entry) => { + const session = this.liveSession(entry); + return session ? [session] : []; + }); + } + + /** + * Collects one session's artifacts, reporting anything the provider could not answer for. The + * disposal path asks for the callback budget, because it must answer for a lease; `listCloudArtifacts` + * is a caller's own request and is left to the provider's own bound. + */ private async safeListArtifacts( session: WebDriverProviderSession, + withinCallbackBudget = false, ): Promise { const listArtifacts = session.prepared.listArtifacts ?? this.options.listArtifacts; if (!listArtifacts) return undefined; + const params = { + provider: this.options.provider, + lease: session.lease, + device: session.device, + webDriverSessionId: session.webDriverSessionId, + providerSessionId: session.providerSessionId, + }; try { - return await listArtifacts({ - provider: this.options.provider, - lease: session.lease, - device: session.device, - webDriverSessionId: session.webDriverSessionId, - providerSessionId: session.providerSessionId, - }); + return withinCallbackBudget + ? await this.providerCallback( + `${this.options.provider} artifact lookup for session ${session.providerSessionId}`, + () => listArtifacts(params), + ) + : await listArtifacts(params); } catch (error) { return unavailableCloudArtifactsResult({ provider: this.options.provider, @@ -321,3 +640,154 @@ function sessionCreateTimeoutError( timeout, ); } + +/** What a route that ended one lease's session can say about it, `pending` while it still cannot. */ +type WebDriverTeardownReport = Readonly<{ + providerSessionId: string | undefined; + cleanup: WebDriverDisposalOutcome['cleanup'] | 'pending'; + warnings: CloudWebDriverReleaseWarning[]; +}>; + +/** + * The refusal an allocation gets for the teardown that ended the session it stood for: which route ended + * it, and whether that end is confirmed or known to have failed. + */ +function refusalForTeardown( + provider: string, + lease: DeviceLease, + by: 'release' | 'shutdown', + report: WebDriverTeardownReport, +): AppError { + return new AppError( + 'COMMAND_FAILED', + `${provider} ended the WebDriver session allocated for lease ${lease.leaseId} before the allocation returned.`, + { + reason: + by === 'release' + ? 'provider_session_released_during_allocation' + : 'provider_session_shutdown_during_allocation', + provider, + leaseId: lease.leaseId, + runId: lease.runId, + ...(report.providerSessionId ? { providerSessionId: report.providerSessionId } : {}), + cleanup: report.cleanup, + ...(report.warnings.length > 0 + ? { cleanupWarnings: warningMessagesByCode(report.warnings) } + : {}), + hint: teardownHint(provider, by, report.cleanup), + }, + ); +} + +/** + * The refusal for an allocation that arrived while a release was still ending the lease's previous + * session. It rented nothing, so it is told the lease is being ended rather than that a session of its + * own died, and it may not allocate again until that disposal finishes. + */ +function refusalForTeardownInProgress( + provider: string, + lease: DeviceLease, + take: WebDriverLeaseTake, +): AppError { + // Both answers describe the same disposal, so both read the same two facts from it. The hint stays + // separate: this caller created nothing, so it is told about the lease, not about a session of its own. + const cleanup = take.settled?.cleanup ?? 'pending'; + const providerSessionId = take.providerSessionId ?? take.settled?.session?.providerSessionId; + if (take.by === 'shutdown') { + return refusalForTeardown(provider, lease, 'shutdown', { + providerSessionId, + cleanup, + warnings: take.settled?.warnings ?? [], + }); + } + return new AppError( + // ADR 0010: the lease slot is held by the release in flight, so this is contention, and the caller + // can expect the same allocate to succeed once that release finishes. + 'DEVICE_IN_USE', + `${provider} is releasing lease ${lease.leaseId} and allocated no WebDriver session for it.`, + { + reason: 'provider_session_release_in_progress', + provider, + leaseId: lease.leaseId, + runId: lease.runId, + ...(providerSessionId ? { providerSessionId } : {}), + cleanup, + hint: + cleanup === 'uncertain' + ? `agent-device could not stop the previous session of this lease. Check ${provider} for it before allocating again.` + : 'Wait for this lease to finish releasing, then allocate it again.', + }, + ); +} + +/** + * This manager was already shut down when the allocation arrived, so it created nothing and owes nothing. + */ +function allocationRefusedAfterShutdown(provider: string, lease: DeviceLease): AppError { + return new AppError( + 'COMMAND_FAILED', + `${provider} is shut down and created no WebDriver session for lease ${lease.leaseId}.`, + { + reason: 'provider_session_shutdown_during_allocation', + provider, + leaseId: lease.leaseId, + runId: lease.runId, + cleanup: 'none', + hint: 'Start the daemon again to allocate a provider session.', + }, + ); +} + +/** + * What the refused allocation should do next: nothing once the hub confirmed the session is gone, and a + * manual stop on the provider when the hub did not confirm it. + */ +function teardownHint( + provider: string, + by: 'release' | 'shutdown', + cleanup: WebDriverDisposalOutcome['cleanup'] | 'pending', +): string { + if (cleanup === 'uncertain') { + return `${provider} did not confirm it stopped the session this allocation created, and that session may keep billing. Check ${provider} for it before allocating the lease again.`; + } + if (cleanup === 'pending') { + return `agent-device is still stopping the session this allocation created. Wait for that ${by === 'release' ? 'release' : 'shutdown'} to finish before allocating the lease again.`; + } + if (cleanup === 'none') { + return 'Nothing was billed for this allocation. Allocate the lease again.'; + } + return 'Allocate the lease again; agent-device stopped the session this allocation created.'; +} + +/** + * Sessions this manager could not end at close. A daemon close reports what it left running, so the + * error names every session still billing and why it is still there, and the create-retry policy stays + * untouched: nothing here deletes or creates a session again on its own. + */ +function shutdownLeftSessionsRunning( + provider: string, + outcomes: WebDriverDisposalOutcome[], +): AppError { + const providerSessionIds = outcomes.map((outcome) => outcome.session?.providerSessionId); + return new AppError( + 'COMMAND_FAILED', + `${provider} left ${String(outcomes.length)} WebDriver session(s) running after shutdown: ${providerSessionIds.join(', ')}.`, + { + reason: 'provider_session_cleanup_incomplete', + provider, + providerSessionIds, + cleanup: 'uncertain', + // One entry per warning, each naming the session it is about: several sessions the provider would + // not give back carry several refusals, and keying them by code alone would report one reason for + // all of them. + cleanupWarnings: outcomes.flatMap((outcome) => + outcome.warnings.map((warning) => ({ + providerSessionId: outcome.session?.providerSessionId, + code: warning.code, + message: warning.message, + })), + ), + hint: `${provider} did not stop these sessions and they keep billing. Stop the session ids above on ${provider}.`, + }, + ); +} diff --git a/packages/provider-webdriver/src/runtime.ts b/packages/provider-webdriver/src/runtime.ts index 4f400bf28c..601d54f1af 100644 --- a/packages/provider-webdriver/src/runtime.ts +++ b/packages/provider-webdriver/src/runtime.ts @@ -103,6 +103,12 @@ export type CloudWebDriverRuntimeOptions = { capabilityOverrides?: CloudWebDriverCapabilityOverrides; /** Profile fields this provider reads; any field it refuses fails lease allocation. */ profileFields: ProviderProfileFieldDeclaration; + /** + * Budget for a prepared session's `cleanup` on disposal and on a failed create, and for `listArtifacts` + * during teardown. Defaults to 30,000 ms. This budget does not bound `prepareSession`; release and + * shutdown still wait for preparation to finish. + */ + providerCallbackTimeoutMs?: number; }; export function createCloudWebDriverRuntime(