From 14fb9a9549144cdce570723258ba57fd3095b9cb Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 19:37:38 +0200 Subject: [PATCH 01/14] test: slow lane: a target boots a real simulator and emulator, and off shuts both down (#372) --- e2e/README.md | 2 +- e2e/slow-warm-pool.test.ts | 432 +++++++++++++++++++++++++++++++++++++ 2 files changed, 433 insertions(+), 1 deletion(-) create mode 100644 e2e/slow-warm-pool.test.ts diff --git a/e2e/README.md b/e2e/README.md index 77fc7813..94dcf484 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -65,4 +65,4 @@ running test body silently never fires — see the comment in `helpers/env.ts`. - `daemon status` ignores `--json` and always prints raw JSON; documented by an `it.fails` in `daemon-lifecycle.test.ts`. - Android is exercised through the fake driver in the fast lane; only - `slow-android-smoke.test.ts` touches a real emulator. + `slow-android-smoke.test.ts` and `slow-warm-pool.test.ts` touch a real emulator. diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts new file mode 100644 index 00000000..de558277 --- /dev/null +++ b/e2e/slow-warm-pool.test.ts @@ -0,0 +1,432 @@ +import { execFile } from "node:child_process"; +import { existsSync, readdirSync } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; + +import { withDaemon, waitFor, type TestEnv } from "./helpers/index.js"; +import { iosDeviceSet, setDevices, sweepStaleDeviceSets } from "./helpers/ios-device-set.js"; + +const execFileAsync = promisify(execFile); + +/** + * The warm pool on real devices: a target boots a real simulator and a real emulator ahead of any + * lease, and with the pool off a released device of each platform ends shut down. The fast lane + * proves what the pool decides against fake drivers; only a real `simctl` and a real emulator can + * prove that the create-and-boot path works end to end and that the Android snapshot restore + * followed by the pool's shutdown leaves an emulator that boots cleanly next time. + * + * Run only through `scripts/slow-e2e.sh e2e/slow-warm-pool.test.ts`, never in CI. + */ + +const ANDROID_HOME = + process.env.ANDROID_HOME ?? + process.env.ANDROID_SDK_ROOT ?? + join(homedir(), "Library/Android/sdk"); +/** The ABI an emulator runs natively on this host; an image of the other ABI is listed but cannot boot. */ +const HOST_ABI = process.arch === "arm64" ? "arm64-v8a" : "x86_64"; + +type Platform = "ios" | "android"; + +interface Target { + readonly count: 1; + readonly model: string; + readonly osVersion: string; + readonly platform: Platform; +} + +interface Row { + readonly id: string; + readonly driverDeviceId: string; + readonly state: string; + readonly spec: { readonly platform: Platform; readonly model: string }; +} + +interface Grant { + readonly device: Row; + readonly lease: { readonly id: string }; +} + +interface CatalogPlatform { + readonly platform: string; + readonly models: readonly string[]; + readonly runtimes: readonly string[]; + readonly modelRuntimes: Readonly>; + readonly classDefaults: Readonly>; + readonly images?: readonly { runtime: string; tag: string; abi: string }[]; +} + +const SECOND = 1000; +const MINUTE = 60 * SECOND; +/** Two real boots (one of them an emulator), one lease and release of each, and a second round. */ +const TEST_TIMEOUT = 30 * MINUTE; +const LEASE_TIMEOUT = 10 * MINUTE; +const READY_TIMEOUT = 15 * MINUTE; + +async function installedIosRuntimes(): Promise { + const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "runtimes", "-j"]); + const parsed = JSON.parse(stdout) as { + runtimes: { version: string; isAvailable: boolean; platform?: string }[]; + }; + return parsed.runtimes + .filter((runtime) => runtime.isAvailable && (runtime.platform ?? "iOS") === "iOS") + .map((runtime) => runtime.version); +} + +function installedAndroidImages(): boolean { + const root = join(ANDROID_HOME, "system-images"); + return ( + existsSync(join(ANDROID_HOME, "emulator", "emulator")) && + existsSync(join(ANDROID_HOME, "platform-tools", "adb")) && + existsSync(root) && + readdirSync(root).length > 0 + ); +} + +/** The reason this machine cannot run the lane, naming the missing platform; undefined when it can. */ +async function missingPlatform(): Promise { + if (process.platform !== "darwin" || (await installedIosRuntimes()).length === 0) { + return "iOS is missing: no macOS host with an installed iOS runtime"; + } + if (!installedAndroidImages()) { + return "Android is missing: no emulator, adb and installed system image"; + } + return undefined; +} + +/** Simlock's emulators live on Simlock's own adb server, so that is the one to ask. */ +async function onlineEmulators(adbServerPort: number): Promise { + const adb = join(ANDROID_HOME, "platform-tools", "adb"); + const { stdout } = await execFileAsync(adb, ["-P", String(adbServerPort), "devices"]).catch( + () => ({ stdout: "" }), + ); + return stdout + .split("\n") + .slice(1) + .map((line) => line.trim()) + .filter((line) => line.endsWith("device")) + .map((line) => line.split(/\s+/)[0] as string); +} + +function newest(versions: readonly string[]): string | undefined { + return [...versions].sort((a, b) => b.localeCompare(a, undefined, { numeric: true }))[0]; +} + +/** The newest installed runtime that some model pairs with, and a phone model that pairs with it. */ +function targetFor(catalog: CatalogPlatform, platform: Platform): Target { + const runnable = + platform === "android" + ? (catalog.images ?? []).filter((image) => image.abi === HOST_ABI).map((i) => i.runtime) + : catalog.runtimes; + const pairing = runnable.filter((runtime) => + catalog.models.some((model) => catalog.modelRuntimes[model]?.includes(runtime)), + ); + const osVersion = newest(pairing); + if (osVersion === undefined) { + throw new Error( + `no installed ${platform} runtime pairs with a model: ${JSON.stringify(catalog)}`, + ); + } + const phone = catalog.classDefaults["phone"]; + const model = + phone !== undefined && catalog.modelRuntimes[phone]?.includes(osVersion) + ? phone + : (catalog.models.find((name) => catalog.modelRuntimes[name]?.includes(osVersion)) as string); + return { count: 1, model, osVersion, platform }; +} + +async function writeWarmPool(env: TestEnv, warmPool: Record): Promise { + const current = JSON.parse(await readFile(env.configPath, "utf8")) as { + warmPool?: Record; + }; + current.warmPool = { ...current.warmPool, ...warmPool }; + await writeFile(env.configPath, `${JSON.stringify(current, null, 2)}\n`, "utf8"); + await env.restartDaemon(); +} + +interface Lane { + readonly env: TestEnv; + readonly adbServerPort: number; + readonly deviceSet: string; + readonly targets: Record; +} + +/** A real-driver daemon whose pool targets one device of the newest runtime of each platform. */ +async function warmDaemon(): Promise { + await sweepStaleDeviceSets(); + const env = await withDaemon({ driver: "real" }); + const adbServerPort = env.adbServerPort; + if (adbServerPort === undefined) throw new Error("the real-SDK lane must allocate an adb port"); + const reported = await env.cli(["catalog", "--json"]); + expect(reported.code, `catalog failed: ${reported.stderr}`).toBe(0); + const platforms = (reported.json as { platforms: CatalogPlatform[] }).platforms; + const catalogOf = (platform: Platform): CatalogPlatform => { + const found = platforms.find((entry) => entry.platform === platform); + if (found === undefined) throw new Error(`simlock catalog reported no ${platform} platform`); + return found; + }; + const targets = { + android: targetFor(catalogOf("android"), "android"), + ios: targetFor(catalogOf("ios"), "ios"), + }; + await writeWarmPool(env, { + maxConcurrentBoots: 2, + targets: [targets.ios, targets.android], + }); + return { adbServerPort, deviceSet: iosDeviceSet(env.home), env, targets }; +} + +async function devices(env: TestEnv): Promise { + const listed = await env.cli(["list", "--devices"]); + expect(listed.code, listed.stderr).toBe(0); + return listed.json as Row[]; +} + +/** Waits until the device with this driver id is in `state`, naming the last rows seen on timeout. */ +async function untilState( + env: TestEnv, + driverDeviceId: string, + state: string, + timeout = READY_TIMEOUT, +): Promise { + let last: Row[] = []; + await waitFor( + async () => { + last = await devices(env); + return last.some((row) => row.driverDeviceId === driverDeviceId && row.state === state); + }, + { + interval: 1000, + label: () => + `${driverDeviceId} is ${state}; last saw ${JSON.stringify(last.map((r) => [r.driverDeviceId, r.state]))}`, + timeout, + }, + ); +} + +/** Waits for one ready device per platform, with no lease asked for yet. */ +async function untilBothReady(env: TestEnv): Promise { + let last: Row[] = []; + await waitFor( + async () => { + last = await devices(env); + return (["ios", "android"] as const).every((platform) => + last.some((row) => row.spec.platform === platform && row.state === "ready"), + ); + }, + { + interval: 2000, + label: () => `a ready simulator and a ready emulator; last saw ${JSON.stringify(last)}`, + timeout: READY_TIMEOUT, + }, + ); + return last; +} + +async function lease( + env: TestEnv, + target: Target, + agent: string, +): Promise<{ readonly grant: Grant; readonly stderr: string }> { + const result = await env.cli( + [ + "lease", + "--platform", + target.platform, + "--device", + target.model, + "--os", + target.osVersion, + "--agent-id", + agent, + "--detach", + ], + { timeout: LEASE_TIMEOUT }, + ); + expect(result.code, `${target.platform} lease failed: ${result.stderr}`).toBe(0); + return { grant: result.json as Grant, stderr: result.stderr }; +} + +async function release(env: TestEnv, grant: Grant): Promise { + const released = await env.cli(["release", grant.lease.id], { timeout: LEASE_TIMEOUT }); + expect(released.code, `release failed: ${released.stderr}`).toBe(0); +} + +/** The `booting` progress lines a lease wrote to stderr; one per boot the lease waited on. */ +function bootingLines(stderr: string): string[] { + return stderr.split("\n").filter((line) => { + try { + const parsed = JSON.parse(line) as { push?: string; stage?: string }; + return parsed.push === "progress" && parsed.stage === "booting"; + } catch { + return false; + } + }); +} + +async function nuke(env: TestEnv): Promise { + await env + .cli(["nuke", "--delete-devices", "--yes"], { timeout: 5 * MINUTE }) + .catch(() => undefined); +} + +describe( + "warm pool targets (real simctl and emulator)", + { tags: ["slow", "ios", "android"] }, + () => { + it( + "with a target of one iPhone and one Android phone, ends with one ready simulator and one ready emulator before any lease", + { timeout: TEST_TIMEOUT }, + async (context) => { + const missing = await missingPlatform(); + if (missing !== undefined) context.skip(missing); + const { env } = await warmDaemon(); + try { + const rows = await untilBothReady(env); + + for (const platform of ["ios", "android"] as const) { + const ofPlatform = rows.filter((row) => row.spec.platform === platform); + expect( + ofPlatform.map((row) => row.state), + `${platform} devices`, + ).toEqual(["ready"]); + } + const recorded = await env.events(); + expect( + recorded.filter((entry) => entry.event.startsWith("lease.")), + "no lease was asked for before the devices were ready", + ).toEqual([]); + const ready = recorded.filter((entry) => entry.event === "device.ready"); + // The two lines the verifier reads in the log: both device.ready events, before any lease. + for (const entry of ready) { + console.info( + `device.ready ${JSON.stringify(entry.payload)} at ${new Date(entry.timestamp).toISOString()}`, + ); + } + expect( + ready.map((entry) => (entry.payload as { deviceId: string }).deviceId).sort(), + ).toEqual(rows.map((row) => row.id).sort()); + } finally { + await nuke(env); + } + }, + ); + + it( + "grants the first lease of each targeted model warm with no booting progress, and after release each device is ready again", + { timeout: TEST_TIMEOUT }, + async (context) => { + const missing = await missingPlatform(); + if (missing !== undefined) context.skip(missing); + const { env, targets } = await warmDaemon(); + try { + await untilBothReady(env); + const before = await env.events(); + const readyEvents = before.filter((entry) => entry.event === "device.ready"); + expect(readyEvents.length, "device.ready events before the first lease").toBeGreaterThan( + 1, + ); + + const granted: Grant[] = []; + for (const platform of ["ios", "android"] as const) { + const { grant, stderr } = await lease(env, targets[platform], `warm-${platform}`); + expect( + bootingLines(stderr), + `${platform}: the first lease must not wait on a boot`, + ).toEqual([]); + granted.push(grant); + } + + const grantedEvents = (await env.events()).filter( + (entry) => entry.event === "lease.granted", + ); + for (const grant of granted) { + const entry = grantedEvents.find( + (candidate) => (candidate.payload as { leaseId?: string }).leaseId === grant.lease.id, + ); + expect( + entry?.payload, + `${grant.device.spec.platform}: lease.granted names the warm pool as its source`, + ).toMatchObject({ source: "warm" }); + expect(entry?.timestamp ?? 0).toBeGreaterThan( + Math.max(...readyEvents.map((ready) => ready.timestamp)), + ); + } + + for (const grant of granted) await release(env, grant); + // iOS is erased, shut down and booted again by the pool; Android restores its snapshot. + for (const grant of granted) { + await untilState(env, grant.device.driverDeviceId, "ready"); + } + } finally { + await nuke(env); + } + }, + ); + + it( + "with warmPool.enabled false, a released simulator and a released emulator are both shut down, and each boots on the next lease", + { timeout: TEST_TIMEOUT }, + async (context) => { + const missing = await missingPlatform(); + if (missing !== undefined) context.skip(missing); + const { adbServerPort, deviceSet, env, targets } = await warmDaemon(); + try { + await untilBothReady(env); + await writeWarmPool(env, { enabled: false }); + + const first = { + android: await lease(env, targets.android, "cold-android"), + ios: await lease(env, targets.ios, "cold-ios"), + }; + await release(env, first.ios.grant); + await release(env, first.android.grant); + + const iosUdid = first.ios.grant.device.driverDeviceId; + await untilState(env, iosUdid, "shutdown", 5 * MINUTE); + await untilState(env, first.android.grant.device.driverDeviceId, "shutdown", 5 * MINUTE); + await waitFor( + async () => + (await setDevices(deviceSet)).find((device) => device.udid === iosUdid)?.state === + "Shutdown", + { interval: 1000, label: "the simulator is Shutdown in simctl", timeout: MINUTE }, + ); + await waitFor(async () => (await onlineEmulators(adbServerPort)).length === 0, { + interval: 1000, + label: "no emulator is online on Simlock's adb server", + timeout: MINUTE, + }); + + // The next boot is clean: each lease is granted and its device really is running. + const second = { + android: await lease(env, targets.android, "again-android"), + ios: await lease(env, targets.ios, "again-ios"), + }; + const rows = await devices(env); + for (const { grant } of [second.ios, second.android]) { + expect( + rows.find((row) => row.id === grant.device.id)?.state, + `${grant.device.spec.platform} device is leased after its second boot`, + ).toBe("leased"); + } + expect( + (await setDevices(deviceSet)).find( + (device) => device.udid === second.ios.grant.device.driverDeviceId, + )?.state, + ).toBe("Booted"); + expect( + (await onlineEmulators(adbServerPort)).length, + "the emulator booted again and is online", + ).toBeGreaterThan(0); + + await release(env, second.ios.grant); + await release(env, second.android.grant); + } finally { + await nuke(env); + } + }, + ); + }, +); From c1156a290dc1da201f1d4d4bdcd9e6daa45233b6 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 20:04:01 +0200 Subject: [PATCH 02/14] test: slow warm pool: a held end state, a booting lease, a probe that skips, and true docs (#372) Fixes review round 1 findings. --- e2e/README.md | 10 ++-- e2e/slow-warm-pool.test.ts | 117 ++++++++++++++++++++++++++++--------- 2 files changed, 93 insertions(+), 34 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 94dcf484..4a25f4f2 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -17,8 +17,9 @@ recovery. Anything provable in-process belongs in a unit test instead. | slow | `pnpm run test:e2e:slow` | real `simctl` / `adb` / emulators | manual, pre-release | The slow flows are tagged `slow` plus `ios`/`android`, so the fast lane excludes -them with `--tags-filter='!slow'`. Each also gates itself at runtime with -`describe.skipIf` on platform and SDK availability. Filter further with, for +them with `--tags-filter='!slow'`. Each also gates itself at runtime on platform and SDK +availability: with `describe.skipIf`, or, in `slow-warm-pool.test.ts`, with +`context.skip(reason)` inside each test. Filter further with, for example, `pnpm exec vitest run --project e2e --tags-filter='ios && !android'`. The fast lane runs four files at once: each flow has its own `SIMLOCK_HOME`, so @@ -57,12 +58,9 @@ running test body silently never fires — see the comment in `helpers/env.ts`. - **The slow lane does not pass reliably as a whole run.** A daemon occasionally outlives `daemon stop` and trips teardown's stray-process - assertion; not yet root-caused, and not yet distinguished from the reclaim - stall recorded in [../docs/internal/KNOWN-PITFALLS.md](../docs/internal/KNOWN-PITFALLS.md). + assertion; not yet root-caused. The slim lane also fails while a 50-label disable batch can outrun its fixed 60s limit on a cold boot ([#206](https://github.com/callstackincubator/simlock/issues/206)). Treat the slow lane as a manual, one-flow-at-a-time tool until both are fixed. -- `daemon status` ignores `--json` and always prints raw JSON; documented by an - `it.fails` in `daemon-lifecycle.test.ts`. - Android is exercised through the fake driver in the fast lane; only `slow-android-smoke.test.ts` and `slow-warm-pool.test.ts` touch a real emulator. diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index de558277..ceb88a2c 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -64,25 +64,41 @@ const MINUTE = 60 * SECOND; const TEST_TIMEOUT = 30 * MINUTE; const LEASE_TIMEOUT = 10 * MINUTE; const READY_TIMEOUT = 15 * MINUTE; +/** How long a shut-down device is watched: several pool ticks, so a pool still on would have booted it. */ +const HOLD = 45 * SECOND; +/** The installed iOS runtimes; empty when `xcrun simctl` is missing or unusable, so the lane skips. */ async function installedIosRuntimes(): Promise { - const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "runtimes", "-j"]); - const parsed = JSON.parse(stdout) as { - runtimes: { version: string; isAvailable: boolean; platform?: string }[]; - }; - return parsed.runtimes - .filter((runtime) => runtime.isAvailable && (runtime.platform ?? "iOS") === "iOS") - .map((runtime) => runtime.version); + try { + const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "runtimes", "-j"]); + const parsed = JSON.parse(stdout) as { + runtimes: { version: string; isAvailable: boolean; platform?: string }[]; + }; + return parsed.runtimes + .filter((runtime) => runtime.isAvailable && (runtime.platform ?? "iOS") === "iOS") + .map((runtime) => runtime.version); + } catch { + return []; + } } +/** Whether an emulator, adb and a system image of the host's ABI (`system-images///`) exist. */ function installedAndroidImages(): boolean { const root = join(ANDROID_HOME, "system-images"); - return ( - existsSync(join(ANDROID_HOME, "emulator", "emulator")) && - existsSync(join(ANDROID_HOME, "platform-tools", "adb")) && - existsSync(root) && - readdirSync(root).length > 0 - ); + if ( + !existsSync(join(ANDROID_HOME, "emulator", "emulator")) || + !existsSync(join(ANDROID_HOME, "platform-tools", "adb")) || + !existsSync(root) + ) { + return false; + } + try { + return readdirSync(root).some((api) => + readdirSync(join(root, api)).some((tag) => existsSync(join(root, api, tag, HOST_ABI))), + ); + } catch { + return false; + } } /** The reason this machine cannot run the lane, naming the missing platform; undefined when it can. */ @@ -91,7 +107,7 @@ async function missingPlatform(): Promise { return "iOS is missing: no macOS host with an installed iOS runtime"; } if (!installedAndroidImages()) { - return "Android is missing: no emulator, adb and installed system image"; + return `Android is missing: no emulator, adb and installed ${HOST_ABI} system image`; } return undefined; } @@ -114,27 +130,24 @@ function newest(versions: readonly string[]): string | undefined { return [...versions].sort((a, b) => b.localeCompare(a, undefined, { numeric: true }))[0]; } -/** The newest installed runtime that some model pairs with, and a phone model that pairs with it. */ +/** The newest installed runtime the phone-class default model pairs with, and that model. */ function targetFor(catalog: CatalogPlatform, platform: Platform): Target { const runnable = platform === "android" ? (catalog.images ?? []).filter((image) => image.abi === HOST_ABI).map((i) => i.runtime) : catalog.runtimes; - const pairing = runnable.filter((runtime) => - catalog.models.some((model) => catalog.modelRuntimes[model]?.includes(runtime)), + const phone = catalog.classDefaults["phone"]; + const osVersion = newest( + runnable.filter( + (runtime) => phone !== undefined && catalog.modelRuntimes[phone]?.includes(runtime), + ), ); - const osVersion = newest(pairing); - if (osVersion === undefined) { + if (phone === undefined || osVersion === undefined) { throw new Error( - `no installed ${platform} runtime pairs with a model: ${JSON.stringify(catalog)}`, + `no installed ${platform} runtime pairs with the phone-class model: ${JSON.stringify(catalog)}`, ); } - const phone = catalog.classDefaults["phone"]; - const model = - phone !== undefined && catalog.modelRuntimes[phone]?.includes(osVersion) - ? phone - : (catalog.models.find((name) => catalog.modelRuntimes[name]?.includes(osVersion)) as string); - return { count: 1, model, osVersion, platform }; + return { count: 1, model: phone, osVersion, platform }; } async function writeWarmPool(env: TestEnv, warmPool: Record): Promise { @@ -206,6 +219,24 @@ async function untilState( ); } +/** Fails if the device leaves `state` at any poll within `duration`: an end state must hold, not be seen once. */ +async function holdsState( + env: TestEnv, + driverDeviceId: string, + state: string, + duration: number, +): Promise { + const until = Date.now() + duration; + await waitFor( + async () => { + const row = (await devices(env)).find((entry) => entry.driverDeviceId === driverDeviceId); + expect(row?.state, `${driverDeviceId} stays ${state}`).toBe(state); + return Date.now() >= until; + }, + { interval: 2000, label: `${driverDeviceId} stays ${state}`, timeout: duration + MINUTE }, + ); +} + /** Waits for one ready device per platform, with no lease asked for yet. */ async function untilBothReady(env: TestEnv): Promise { let last: Row[] = []; @@ -328,6 +359,11 @@ describe( expect(readyEvents.length, "device.ready events before the first lease").toBeGreaterThan( 1, ); + for (const entry of readyEvents) { + console.info( + `device.ready ${JSON.stringify(entry.payload)} at ${new Date(entry.timestamp).toISOString()}`, + ); + } const granted: Grant[] = []; for (const platform of ["ios", "android"] as const) { @@ -356,7 +392,7 @@ describe( } for (const grant of granted) await release(env, grant); - // iOS is erased, shut down and booted again by the pool; Android restores its snapshot. + // iOS is shut down, erased and booted again by the pool; Android restores its snapshot. for (const grant of granted) { await untilState(env, grant.device.driverDeviceId, "ready"); } @@ -399,11 +435,36 @@ describe( timeout: MINUTE, }); - // The next boot is clean: each lease is granted and its device really is running. + // Shut down is an end state: the pool, were it still on, would boot both again within this window. + const androidId = first.android.grant.device.driverDeviceId; + await holdsState(env, iosUdid, "shutdown", HOLD); + await holdsState(env, androidId, "shutdown", HOLD); + expect( + (await setDevices(deviceSet)).find((device) => device.udid === iosUdid)?.state, + "the simulator is still Shutdown in simctl", + ).toBe("Shutdown"); + expect(await onlineEmulators(adbServerPort), "no emulator came back online").toEqual([]); + + // The next boot is clean: each lease waits on a boot, is granted, and its device really is running. const second = { android: await lease(env, targets.android, "again-android"), ios: await lease(env, targets.ios, "again-ios"), }; + for (const platform of ["ios", "android"] as const) { + expect( + bootingLines(second[platform].stderr), + `${platform}: the lease waited on a boot`, + ).not.toEqual([]); + const granted = (await env.events()).find( + (entry) => + entry.event === "lease.granted" && + (entry.payload as { leaseId?: string }).leaseId === second[platform].grant.lease.id, + ); + expect( + granted?.payload, + `${platform}: the grant did not come from a warm device`, + ).not.toMatchObject({ source: "warm" }); + } const rows = await devices(env); for (const { grant } of [second.ios, second.android]) { expect( From bdb4177498f38ed88aed5279b26a7b5dbb3e2463 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 20:13:53 +0200 Subject: [PATCH 03/14] test: slow warm pool: split the warm-grant check out to pass Fallow complexity (#372) --- e2e/slow-warm-pool.test.ts | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index ceb88a2c..13cd768b 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -297,6 +297,26 @@ function bootingLines(stderr: string): string[] { }); } +async function expectGrantedWarm( + env: TestEnv, + granted: readonly Grant[], + readyEvents: readonly { timestamp: number }[], +): Promise { + const grantedEvents = (await env.events()).filter((entry) => entry.event === "lease.granted"); + for (const grant of granted) { + const entry = grantedEvents.find( + (candidate) => (candidate.payload as { leaseId?: string }).leaseId === grant.lease.id, + ); + expect( + entry?.payload, + `${grant.device.spec.platform}: lease.granted names the warm pool as its source`, + ).toMatchObject({ source: "warm" }); + expect(entry?.timestamp ?? 0).toBeGreaterThan( + Math.max(...readyEvents.map((ready) => ready.timestamp)), + ); + } +} + async function nuke(env: TestEnv): Promise { await env .cli(["nuke", "--delete-devices", "--yes"], { timeout: 5 * MINUTE }) @@ -375,21 +395,7 @@ describe( granted.push(grant); } - const grantedEvents = (await env.events()).filter( - (entry) => entry.event === "lease.granted", - ); - for (const grant of granted) { - const entry = grantedEvents.find( - (candidate) => (candidate.payload as { leaseId?: string }).leaseId === grant.lease.id, - ); - expect( - entry?.payload, - `${grant.device.spec.platform}: lease.granted names the warm pool as its source`, - ).toMatchObject({ source: "warm" }); - expect(entry?.timestamp ?? 0).toBeGreaterThan( - Math.max(...readyEvents.map((ready) => ready.timestamp)), - ); - } + await expectGrantedWarm(env, granted, readyEvents); for (const grant of granted) await release(env, grant); // iOS is shut down, erased and booted again by the pool; Android restores its snapshot. From fb0720d2bdbcee74c2edb38e96858f5866c67791 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 20:26:30 +0200 Subject: [PATCH 04/14] test: slow warm pool: hold end states for real, name the released device, true comments (#372) Review round 2 fixes: holdsState fails on any poll that leaves the state; test 1 holds one ready device per platform for a window; test 3 asserts the second lease gets the first round's device and that its lease.granted event exists; the HOLD comment names one pool tick. 4 findings -> 0 --- e2e/slow-warm-pool.test.ts | 70 ++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index 13cd768b..470d00ad 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -64,7 +64,7 @@ const MINUTE = 60 * SECOND; const TEST_TIMEOUT = 30 * MINUTE; const LEASE_TIMEOUT = 10 * MINUTE; const READY_TIMEOUT = 15 * MINUTE; -/** How long a shut-down device is watched: several pool ticks, so a pool still on would have booted it. */ +/** How long a shut-down device is watched: longer than one pool tick (30s, WARM_POOL_TICK_MS), so a pool still on would have booted it. */ const HOLD = 45 * SECOND; /** The installed iOS runtimes; empty when `xcrun simctl` is missing or unusable, so the lane skips. */ @@ -219,22 +219,50 @@ async function untilState( ); } -/** Fails if the device leaves `state` at any poll within `duration`: an end state must hold, not be seen once. */ -async function holdsState( - env: TestEnv, - driverDeviceId: string, - state: string, - duration: number, -): Promise { +/** + * Runs `check` (which throws on a violation) at every poll for `duration`, and fails on the first + * poll that throws: waitFor swallows a throwing predicate and keeps polling, so the violation is + * kept here and rethrown after the window, and the loop stops at once. + */ +async function holds(label: string, duration: number, check: () => Promise): Promise { const until = Date.now() + duration; + let violation: unknown; await waitFor( async () => { - const row = (await devices(env)).find((entry) => entry.driverDeviceId === driverDeviceId); - expect(row?.state, `${driverDeviceId} stays ${state}`).toBe(state); + try { + await check(); + } catch (error: unknown) { + violation = error; + return true; + } return Date.now() >= until; }, - { interval: 2000, label: `${driverDeviceId} stays ${state}`, timeout: duration + MINUTE }, + { interval: 2000, label, timeout: duration + MINUTE }, ); + if (violation !== undefined) throw violation; +} + +/** Fails if the device is not in `state` at any poll within `duration`: an end state must hold, not be seen once. */ +async function holdsState( + env: TestEnv, + driverDeviceId: string, + state: string, + duration: number, +): Promise { + await holds(`${driverDeviceId} stays ${state}`, duration, async () => { + const row = (await devices(env)).find((entry) => entry.driverDeviceId === driverDeviceId); + expect(row?.state, `${driverDeviceId} stays ${state}`).toBe(state); + }); +} + +/** Per platform, exactly one device, and it is ready. */ +function expectOneReadyPerPlatform(rows: readonly Row[]): void { + for (const platform of ["ios", "android"] as const) { + expect( + rows.filter((row) => row.spec.platform === platform).map((row) => row.state), + `${platform} devices`, + ).toEqual(["ready"]); + } } /** Waits for one ready device per platform, with no lease asked for yet. */ @@ -337,13 +365,11 @@ describe( try { const rows = await untilBothReady(env); - for (const platform of ["ios", "android"] as const) { - const ofPlatform = rows.filter((row) => row.spec.platform === platform); - expect( - ofPlatform.map((row) => row.state), - `${platform} devices`, - ).toEqual(["ready"]); - } + expectOneReadyPerPlatform(rows); + // The end state holds: no extra device starts, and none of the two leaves ready, across a pool tick. + await holds("one ready device per platform", HOLD, async () => { + expectOneReadyPerPlatform(await devices(env)); + }); const recorded = await env.events(); expect( recorded.filter((entry) => entry.event.startsWith("lease.")), @@ -466,10 +492,18 @@ describe( entry.event === "lease.granted" && (entry.payload as { leaseId?: string }).leaseId === second[platform].grant.lease.id, ); + expect( + granted, + `${platform}: a lease.granted event names the second lease`, + ).toBeDefined(); expect( granted?.payload, `${platform}: the grant did not come from a warm device`, ).not.toMatchObject({ source: "warm" }); + expect( + second[platform].grant.device.id, + `${platform}: the second lease boots the device the first one released`, + ).toBe(first[platform].grant.device.id); } const rows = await devices(env); for (const { grant } of [second.ios, second.android]) { From f6e06de0180e3f6b0beac3d4d3f2f6f852c1aaef Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 20:32:23 +0200 Subject: [PATCH 05/14] docs: holds comment states that a violation is rethrown at once (#372) Comment-only; no test count changes (0 failing -> 0 failing) --- e2e/slow-warm-pool.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index 470d00ad..b7daf13f 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -222,7 +222,8 @@ async function untilState( /** * Runs `check` (which throws on a violation) at every poll for `duration`, and fails on the first * poll that throws: waitFor swallows a throwing predicate and keeps polling, so the violation is - * kept here and rethrown after the window, and the loop stops at once. + * kept here, the predicate returns true so the loop stops at once, and the violation is rethrown + * straight after, without waiting out the window. */ async function holds(label: string, duration: number, check: () => Promise): Promise { const until = Date.now() + duration; From c32b279c1b135e72ba8dda43b0ef5690da3a658f Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 20:51:51 +0200 Subject: [PATCH 06/14] test: slow warm pool: the second lease boots any shut-down device of its kind (#372) The planner boots the first shutdown device that serves the request, not the one just released. 1 failing -> 1 failing (slow lane, not run here) --- e2e/slow-warm-pool.test.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index b7daf13f..71020751 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -479,6 +479,14 @@ describe( expect(await onlineEmulators(adbServerPort), "no emulator came back online").toEqual([]); // The next boot is clean: each lease waits on a boot, is granted, and its device really is running. + // Which shut-down device of the kind a lease boots is the daemon's choice: any of them serves. + const shutdownBefore = (await devices(env)).filter((row) => row.state === "shutdown"); + expect( + shutdownBefore.map((row) => row.id), + "the released devices are shut down before the second leases", + ).toEqual( + expect.arrayContaining([first.ios.grant.device.id, first.android.grant.device.id]), + ); const second = { android: await lease(env, targets.android, "again-android"), ios: await lease(env, targets.ios, "again-ios"), @@ -503,8 +511,10 @@ describe( ).not.toMatchObject({ source: "warm" }); expect( second[platform].grant.device.id, - `${platform}: the second lease boots the device the first one released`, - ).toBe(first[platform].grant.device.id); + `${platform}: the second lease boots a device that was shut down`, + ).toBeOneOf( + shutdownBefore.filter((row) => row.spec.platform === platform).map((row) => row.id), + ); } const rows = await devices(env); for (const { grant } of [second.ios, second.android]) { From 69b7eac45c648cd1e532082204077b40f07fbf73 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:02:00 +0200 Subject: [PATCH 07/14] test: slow warm pool: the pool is off from the start, so the released device is the only one the next lease can boot (#372) The second lease asserts the released device's id. 1 failing -> 1 failing (slow lane, not run here) --- e2e/slow-warm-pool.test.ts | 45 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index 71020751..447c5f46 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -15,8 +15,9 @@ const execFileAsync = promisify(execFile); * The warm pool on real devices: a target boots a real simulator and a real emulator ahead of any * lease, and with the pool off a released device of each platform ends shut down. The fast lane * proves what the pool decides against fake drivers; only a real `simctl` and a real emulator can - * prove that the create-and-boot path works end to end and that the Android snapshot restore - * followed by the pool's shutdown leaves an emulator that boots cleanly next time. + * prove that the create-and-boot path works end to end, that a released Android emulator comes + * back ready from its snapshot, and that with the pool off a released device of each platform ends + * shut down and is the device the next lease boots. * * Run only through `scripts/slow-e2e.sh e2e/slow-warm-pool.test.ts`, never in CI. */ @@ -166,8 +167,11 @@ interface Lane { readonly targets: Record; } -/** A real-driver daemon whose pool targets one device of the newest runtime of each platform. */ -async function warmDaemon(): Promise { +/** + * A real-driver daemon whose pool targets one device of the newest runtime of each platform; with + * `enabled: false` the pool is off from the first start, so no device exists until a lease asks. + */ +async function warmDaemon(options: { readonly enabled?: boolean } = {}): Promise { await sweepStaleDeviceSets(); const env = await withDaemon({ driver: "real" }); const adbServerPort = env.adbServerPort; @@ -187,6 +191,7 @@ async function warmDaemon(): Promise { await writeWarmPool(env, { maxConcurrentBoots: 2, targets: [targets.ios, targets.android], + ...(options.enabled === undefined ? {} : { enabled: options.enabled }), }); return { adbServerPort, deviceSet: iosDeviceSet(env.home), env, targets }; } @@ -441,11 +446,11 @@ describe( async (context) => { const missing = await missingPlatform(); if (missing !== undefined) context.skip(missing); - const { adbServerPort, deviceSet, env, targets } = await warmDaemon(); + // The pool is off from the start, so the only device of each kind is the one the first lease + // creates: once released it is the only shut-down device of its kind, and the planner has no + // other to boot on the next lease. + const { adbServerPort, deviceSet, env, targets } = await warmDaemon({ enabled: false }); try { - await untilBothReady(env); - await writeWarmPool(env, { enabled: false }); - const first = { android: await lease(env, targets.android, "cold-android"), ios: await lease(env, targets.ios, "cold-ios"), @@ -479,14 +484,16 @@ describe( expect(await onlineEmulators(adbServerPort), "no emulator came back online").toEqual([]); // The next boot is clean: each lease waits on a boot, is granted, and its device really is running. - // Which shut-down device of the kind a lease boots is the daemon's choice: any of them serves. - const shutdownBefore = (await devices(env)).filter((row) => row.state === "shutdown"); - expect( - shutdownBefore.map((row) => row.id), - "the released devices are shut down before the second leases", - ).toEqual( - expect.arrayContaining([first.ios.grant.device.id, first.android.grant.device.id]), - ); + // Each platform has exactly one device, the released one, so the next lease can only boot it. + const before = await devices(env); + for (const platform of ["ios", "android"] as const) { + expect( + before + .filter((row) => row.spec.platform === platform) + .map((row) => [row.id, row.state]), + `${platform}: the released device is the only device and it is shut down`, + ).toEqual([[first[platform].grant.device.id, "shutdown"]]); + } const second = { android: await lease(env, targets.android, "again-android"), ios: await lease(env, targets.ios, "again-ios"), @@ -511,10 +518,8 @@ describe( ).not.toMatchObject({ source: "warm" }); expect( second[platform].grant.device.id, - `${platform}: the second lease boots a device that was shut down`, - ).toBeOneOf( - shutdownBefore.filter((row) => row.spec.platform === platform).map((row) => row.id), - ); + `${platform}: the second lease boots the device the first one released`, + ).toBe(first[platform].grant.device.id); } const rows = await devices(env); for (const { grant } of [second.ios, second.android]) { From f1088de8524773d004b403c265464e31d78e2a54 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 21:12:19 +0200 Subject: [PATCH 08/14] test: the warm-pool lane reads the released emulator's reclaim strategy (#372) Round 6 finding: the lane's snapshot-restore claim could stay green on a wipe fallback. Test 2 now asserts device.reclaimed strategy is snapshot; two comments reworded. --- e2e/slow-warm-pool.test.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index 447c5f46..db62cab8 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -16,7 +16,7 @@ const execFileAsync = promisify(execFile); * lease, and with the pool off a released device of each platform ends shut down. The fast lane * proves what the pool decides against fake drivers; only a real `simctl` and a real emulator can * prove that the create-and-boot path works end to end, that a released Android emulator comes - * back ready from its snapshot, and that with the pool off a released device of each platform ends + * back ready from its snapshot (its `device.reclaimed` strategy is `snapshot`), and that with the pool off a released device of each platform ends * shut down and is the device the next lease boots. * * Run only through `scripts/slow-e2e.sh e2e/slow-warm-pool.test.ts`, never in CI. @@ -169,7 +169,8 @@ interface Lane { /** * A real-driver daemon whose pool targets one device of the newest runtime of each platform; with - * `enabled: false` the pool is off from the first start, so no device exists until a lease asks. + * `enabled: false` the config is rewritten and the daemon restarted with the pool off, so no + * device exists until a lease asks. */ async function warmDaemon(options: { readonly enabled?: boolean } = {}): Promise { await sweepStaleDeviceSets(); @@ -434,6 +435,18 @@ describe( for (const grant of granted) { await untilState(env, grant.device.driverDeviceId, "ready"); } + // A failed snapshot load falls back to a wipe and the pool boots the device back to ready, + // so the state alone cannot tell the two apart: the reclaim's own strategy does. + const android = granted.find((grant) => grant.device.spec.platform === "android"); + const reclaimed = (await env.events()).filter( + (entry) => + entry.event === "device.reclaimed" && + (entry.payload as { deviceId?: string }).deviceId === android?.device.id, + ); + expect( + reclaimed.map((entry) => (entry.payload as { strategy?: string }).strategy), + "device.reclaimed strategy of the released emulator", + ).toEqual(["snapshot"]); } finally { await nuke(env); } @@ -473,7 +486,7 @@ describe( timeout: MINUTE, }); - // Shut down is an end state: the pool, were it still on, would boot both again within this window. + // Shut down is an end state: the pool, if it were on, would boot both again within this window. const androidId = first.android.grant.device.driverDeviceId; await holdsState(env, iosUdid, "shutdown", HOLD); await holdsState(env, androidId, "shutdown", HOLD); From 41c444f04b1f9a4b5b004f1c2fdec693b08802b1 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 22:08:00 +0200 Subject: [PATCH 09/14] test: slow warm pool: every step of the pool-off test has its own budget and names itself (#372) The 30 min timeout named no step: the log showed no progress and unbounded calls (list, simctl, adb) could not fail alone. Each step is bounded, logged, and a failure dumps the daemon log tail and the device rows. --- e2e/slow-warm-pool.test.ts | 155 +++++++++++++++++++++++++++++-------- 1 file changed, 124 insertions(+), 31 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index db62cab8..a777b7d8 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -64,6 +64,16 @@ const MINUTE = 60 * SECOND; /** Two real boots (one of them an emulator), one lease and release of each, and a second round. */ const TEST_TIMEOUT = 30 * MINUTE; const LEASE_TIMEOUT = 10 * MINUTE; +/** A release returns once the lease is gone; the device's reclaim is awaited separately. */ +const RELEASE_TIMEOUT = 3 * MINUTE; +/** Every call that asks the machine or the daemon a question, so one that never answers names itself. */ +const QUERY_TIMEOUT = MINUTE; +/** + * The pool-off test creates each device from nothing (an Android image's first boot is a cold + * boot), so it is two cold leases, two warm ones, two releases and two end-state holds. The + * budgets of its steps (see `step`) sum to under this; each fires, naming its step, before it. + */ +const COLD_TEST_TIMEOUT = 80 * MINUTE; const READY_TIMEOUT = 15 * MINUTE; /** How long a shut-down device is watched: longer than one pool tick (30s, WARM_POOL_TICK_MS), so a pool still on would have booted it. */ const HOLD = 45 * SECOND; @@ -116,9 +126,13 @@ async function missingPlatform(): Promise { /** Simlock's emulators live on Simlock's own adb server, so that is the one to ask. */ async function onlineEmulators(adbServerPort: number): Promise { const adb = join(ANDROID_HOME, "platform-tools", "adb"); - const { stdout } = await execFileAsync(adb, ["-P", String(adbServerPort), "devices"]).catch( - () => ({ stdout: "" }), - ); + const { stdout } = await execFileAsync(adb, ["-P", String(adbServerPort), "devices"], { + timeout: QUERY_TIMEOUT, + }).catch((error: unknown) => { + // A timed-out adb is a hang to report, not "no emulator online". + if ((error as { killed?: boolean }).killed === true) throw error; + return { stdout: "" }; + }); return stdout .split("\n") .slice(1) @@ -198,7 +212,7 @@ async function warmDaemon(options: { readonly enabled?: boolean } = {}): Promise } async function devices(env: TestEnv): Promise { - const listed = await env.cli(["list", "--devices"]); + const listed = await env.cli(["list", "--devices"], { timeout: QUERY_TIMEOUT }); expect(listed.code, listed.stderr).toBe(0); return listed.json as Row[]; } @@ -316,7 +330,7 @@ async function lease( } async function release(env: TestEnv, grant: Grant): Promise { - const released = await env.cli(["release", grant.lease.id], { timeout: LEASE_TIMEOUT }); + const released = await env.cli(["release", grant.lease.id], { timeout: RELEASE_TIMEOUT }); expect(released.code, `release failed: ${released.stderr}`).toBe(0); } @@ -352,6 +366,46 @@ async function expectGrantedWarm( } } +/** + * Runs one named step of a long test with its own budget. It logs when the step starts and ends, + * so the slow-lane log shows where a run is, and it fails with the step's name when the budget + * runs out: a hang names itself long before the test's own timeout, which names nothing. + */ +async function step(name: string, budget: number, run: () => Promise): Promise { + const started = Date.now(); + console.info(`step start: ${name} (budget ${Math.round(budget / SECOND)}s)`); + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + run(), + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`step "${name}" did not finish within ${budget}ms`)), + budget, + ); + }), + ]); + } finally { + clearTimeout(timer); + console.info(`step end: ${name} after ${Math.round((Date.now() - started) / SECOND)}s`); + } +} + +/** On a failure, what the daemon last said and what it held: the log is all that is left of a torn-down home. */ +async function diagnose(env: TestEnv): Promise { + try { + const log = await readFile(env.logPath, "utf8"); + console.info(`daemon.log tail:\n${log.split("\n").slice(-60).join("\n")}`); + } catch (error: unknown) { + console.info(`daemon.log unreadable: ${String(error)}`); + } + try { + console.info(`devices at failure: ${JSON.stringify(await devices(env))}`); + } catch (error: unknown) { + console.info(`devices at failure unreadable: ${String(error)}`); + } +} + async function nuke(env: TestEnv): Promise { await env .cli(["nuke", "--delete-devices", "--yes"], { timeout: 5 * MINUTE }) @@ -455,7 +509,7 @@ describe( it( "with warmPool.enabled false, a released simulator and a released emulator are both shut down, and each boots on the next lease", - { timeout: TEST_TIMEOUT }, + { timeout: COLD_TEST_TIMEOUT }, async (context) => { const missing = await missingPlatform(); if (missing !== undefined) context.skip(missing); @@ -464,34 +518,62 @@ describe( // other to boot on the next lease. const { adbServerPort, deviceSet, env, targets } = await warmDaemon({ enabled: false }); try { + // Each first lease creates its device and boots it from nothing. const first = { - android: await lease(env, targets.android, "cold-android"), - ios: await lease(env, targets.ios, "cold-ios"), + android: await step( + "first lease: create and boot the emulator", + LEASE_TIMEOUT + MINUTE, + () => lease(env, targets.android, "cold-android"), + ), + ios: await step( + "first lease: create and boot the simulator", + LEASE_TIMEOUT + MINUTE, + () => lease(env, targets.ios, "cold-ios"), + ), }; - await release(env, first.ios.grant); - await release(env, first.android.grant); + await step("release the simulator", RELEASE_TIMEOUT + MINUTE, () => + release(env, first.ios.grant), + ); + await step("release the emulator", RELEASE_TIMEOUT + MINUTE, () => + release(env, first.android.grant), + ); const iosUdid = first.ios.grant.device.driverDeviceId; - await untilState(env, iosUdid, "shutdown", 5 * MINUTE); - await untilState(env, first.android.grant.device.driverDeviceId, "shutdown", 5 * MINUTE); - await waitFor( - async () => - (await setDevices(deviceSet)).find((device) => device.udid === iosUdid)?.state === - "Shutdown", - { interval: 1000, label: "the simulator is Shutdown in simctl", timeout: MINUTE }, + const androidId = first.android.grant.device.driverDeviceId; + await step("the simulator reaches shutdown", 6 * MINUTE, () => + untilState(env, iosUdid, "shutdown", 5 * MINUTE), + ); + await step("the emulator reaches shutdown", 6 * MINUTE, () => + untilState(env, androidId, "shutdown", 5 * MINUTE), + ); + await step("simctl reports the simulator Shutdown", 2 * MINUTE, () => + waitFor( + async () => + (await setDevices(deviceSet, { timeoutMs: QUERY_TIMEOUT })).find( + (device) => device.udid === iosUdid, + )?.state === "Shutdown", + { interval: 1000, label: "the simulator is Shutdown in simctl", timeout: MINUTE }, + ), + ); + await step("no emulator is online", 2 * MINUTE, () => + waitFor(async () => (await onlineEmulators(adbServerPort)).length === 0, { + interval: 1000, + label: "no emulator is online on Simlock's adb server", + timeout: MINUTE, + }), ); - await waitFor(async () => (await onlineEmulators(adbServerPort)).length === 0, { - interval: 1000, - label: "no emulator is online on Simlock's adb server", - timeout: MINUTE, - }); // Shut down is an end state: the pool, if it were on, would boot both again within this window. - const androidId = first.android.grant.device.driverDeviceId; - await holdsState(env, iosUdid, "shutdown", HOLD); - await holdsState(env, androidId, "shutdown", HOLD); + await step("the simulator stays shutdown", HOLD + 2 * MINUTE, () => + holdsState(env, iosUdid, "shutdown", HOLD), + ); + await step("the emulator stays shutdown", HOLD + 2 * MINUTE, () => + holdsState(env, androidId, "shutdown", HOLD), + ); expect( - (await setDevices(deviceSet)).find((device) => device.udid === iosUdid)?.state, + (await setDevices(deviceSet, { timeoutMs: QUERY_TIMEOUT })).find( + (device) => device.udid === iosUdid, + )?.state, "the simulator is still Shutdown in simctl", ).toBe("Shutdown"); expect(await onlineEmulators(adbServerPort), "no emulator came back online").toEqual([]); @@ -508,8 +590,12 @@ describe( ).toEqual([[first[platform].grant.device.id, "shutdown"]]); } const second = { - android: await lease(env, targets.android, "again-android"), - ios: await lease(env, targets.ios, "again-ios"), + android: await step("second lease: boot the emulator", LEASE_TIMEOUT, () => + lease(env, targets.android, "again-android"), + ), + ios: await step("second lease: boot the simulator", LEASE_TIMEOUT, () => + lease(env, targets.ios, "again-ios"), + ), }; for (const platform of ["ios", "android"] as const) { expect( @@ -542,7 +628,7 @@ describe( ).toBe("leased"); } expect( - (await setDevices(deviceSet)).find( + (await setDevices(deviceSet, { timeoutMs: QUERY_TIMEOUT })).find( (device) => device.udid === second.ios.grant.device.driverDeviceId, )?.state, ).toBe("Booted"); @@ -551,8 +637,15 @@ describe( "the emulator booted again and is online", ).toBeGreaterThan(0); - await release(env, second.ios.grant); - await release(env, second.android.grant); + await step("release the second simulator lease", RELEASE_TIMEOUT + MINUTE, () => + release(env, second.ios.grant), + ); + await step("release the second emulator lease", RELEASE_TIMEOUT + MINUTE, () => + release(env, second.android.grant), + ); + } catch (error: unknown) { + await diagnose(env); + throw error; } finally { await nuke(env); } From 7485dcfd94c0cc2010722d41ea45aba1a1ac3340 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 22:08:20 +0200 Subject: [PATCH 10/14] test: slow warm pool: the second-lease checks are one helper (#372) --- e2e/slow-warm-pool.test.ts | 46 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index a777b7d8..bc075dd1 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -406,6 +406,30 @@ async function diagnose(env: TestEnv): Promise { } } +/** The second lease waited on a boot, was not served warm, and booted the device the first one released. */ +async function expectBootedFromShutdown( + env: TestEnv, + platform: Platform, + second: { readonly grant: Grant; readonly stderr: string }, + released: Grant, +): Promise { + expect(bootingLines(second.stderr), `${platform}: the lease waited on a boot`).not.toEqual([]); + const granted = (await env.events()).find( + (entry) => + entry.event === "lease.granted" && + (entry.payload as { leaseId?: string }).leaseId === second.grant.lease.id, + ); + expect(granted, `${platform}: a lease.granted event names the second lease`).toBeDefined(); + expect( + granted?.payload, + `${platform}: the grant did not come from a warm device`, + ).not.toMatchObject({ source: "warm" }); + expect( + second.grant.device.id, + `${platform}: the second lease boots the device the first one released`, + ).toBe(released.device.id); +} + async function nuke(env: TestEnv): Promise { await env .cli(["nuke", "--delete-devices", "--yes"], { timeout: 5 * MINUTE }) @@ -598,27 +622,7 @@ describe( ), }; for (const platform of ["ios", "android"] as const) { - expect( - bootingLines(second[platform].stderr), - `${platform}: the lease waited on a boot`, - ).not.toEqual([]); - const granted = (await env.events()).find( - (entry) => - entry.event === "lease.granted" && - (entry.payload as { leaseId?: string }).leaseId === second[platform].grant.lease.id, - ); - expect( - granted, - `${platform}: a lease.granted event names the second lease`, - ).toBeDefined(); - expect( - granted?.payload, - `${platform}: the grant did not come from a warm device`, - ).not.toMatchObject({ source: "warm" }); - expect( - second[platform].grant.device.id, - `${platform}: the second lease boots the device the first one released`, - ).toBe(first[platform].grant.device.id); + await expectBootedFromShutdown(env, platform, second[platform], first[platform].grant); } const rows = await devices(env); for (const { grant } of [second.ios, second.android]) { From 32c8acf88471426c2fcd4d674d81b5f1cc9a7d4d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 22:18:29 +0200 Subject: [PATCH 11/14] test: slow warm pool: bounded queries, true timing doc, device.ready lines reach the log (#372) Fix round 8: every question put to the machine or daemon has a timeout, the test timeouts exceed budgets plus unbudgeted work, test 3's daemon start is a named step inside the try, and the slow lane runs the default reporter so a passing test's console lines reach the log. 4 findings -> 0 open --- e2e/README.md | 4 ++ e2e/helpers/env.ts | 4 +- e2e/helpers/events.ts | 8 +++- e2e/helpers/index.ts | 1 + e2e/slow-warm-pool.test.ts | 93 +++++++++++++++++++++++++------------- package.json | 2 +- 6 files changed, 75 insertions(+), 37 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 4a25f4f2..466d39ab 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -16,6 +16,10 @@ recovery. Anything provable in-process belongs in a unit test instead. | fast | `pnpm run test:e2e` | scriptable fake driver, no SDKs | `pnpm run test`, CI | | slow | `pnpm run test:e2e:slow` | real `simctl` / `adb` / emulators | manual, pre-release | +The slow lane runs with vitest's `default` reporter: under an agent vitest's minimal reporter drops +the console output of a passing test, and a slow flow's step and `device.ready` lines are what its log is +read for. + The slow flows are tagged `slow` plus `ios`/`android`, so the fast lane excludes them with `--tags-filter='!slow'`. Each also gates itself at runtime on platform and SDK availability: with `describe.skipIf`, or, in `slow-warm-pool.test.ts`, with diff --git a/e2e/helpers/env.ts b/e2e/helpers/env.ts index 7bf9b76f..b29ed8a1 100644 --- a/e2e/helpers/env.ts +++ b/e2e/helpers/env.ts @@ -149,7 +149,7 @@ export interface TestEnv { cli(args: readonly string[], options?: CliOptions): Promise; cliBackground(args: readonly string[], options?: CliOptions): CliBackgroundHandle; mcpClient(options?: McpClientOptions): Promise; - events(since?: string): Promise; + events(since?: string, timeout?: number): Promise; expectEvents( names: readonly string[], options?: { readonly since?: string; readonly timeout?: number }, @@ -235,7 +235,7 @@ export async function withDaemon(options: WithDaemonOptions = {}): Promise events(env, since), + events: (since, timeout) => events(env, since, timeout), expectEvents: (names, expectOptions) => expectEvents(env, names, expectOptions), async startDaemon() { return cli(["daemon", "start"], env); diff --git a/e2e/helpers/events.ts b/e2e/helpers/events.ts index c1ebc783..3dbb13b7 100644 --- a/e2e/helpers/events.ts +++ b/e2e/helpers/events.ts @@ -10,8 +10,12 @@ export interface RecordedEvent { } /** Replays the business-event history via `simlock events --since`, parsed. */ -export async function events(env: NodeJS.ProcessEnv, since = "1h"): Promise { - const result = await cli(["events", "--since", since], env); +export async function events( + env: NodeJS.ProcessEnv, + since = "1h", + timeout?: number, +): Promise { + const result = await cli(["events", "--since", since], env, { timeout }); if (result.code !== 0) { throw new Error(`simlock events failed (exit ${String(result.code)}): ${result.stderr}`); } diff --git a/e2e/helpers/index.ts b/e2e/helpers/index.ts index 35036dc2..d4f367a3 100644 --- a/e2e/helpers/index.ts +++ b/e2e/helpers/index.ts @@ -9,5 +9,6 @@ export { freeLoopbackPort } from "./port.js"; export type { TestEnv } from "./env.js"; export { waitFor } from "./wait.js"; export { events } from "./events.js"; +export type { RecordedEvent } from "./events.js"; export { waitForDeviceState, waitForLeaseCount } from "./status.js"; export type { CliResult } from "./cli.js"; diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index bc075dd1..797d8c40 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -6,7 +6,7 @@ import { join } from "node:path"; import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; -import { withDaemon, waitFor, type TestEnv } from "./helpers/index.js"; +import { withDaemon, waitFor, type RecordedEvent, type TestEnv } from "./helpers/index.js"; import { iosDeviceSet, setDevices, sweepStaleDeviceSets } from "./helpers/ios-device-set.js"; const execFileAsync = promisify(execFile); @@ -61,19 +61,31 @@ interface CatalogPlatform { const SECOND = 1000; const MINUTE = 60 * SECOND; -/** Two real boots (one of them an emulator), one lease and release of each, and a second round. */ +/** Two real boots (one of them an emulator) and, before any lease, a hold of one pool tick: well inside it. */ const TEST_TIMEOUT = 30 * MINUTE; +/** + * Two real boots, a lease and a release of each, and the wait for each to be ready again. The + * waits alone (a 15 min readiness wait, two 10 min leases, two 3 min releases, two 15 min waits + * for ready again) add up to 71 min, plus at most 9 one-minute queries and a 5 min nuke: a wait + * that gives up names itself before this fires. + */ +const WARM_ROUND_TIMEOUT = 90 * MINUTE; const LEASE_TIMEOUT = 10 * MINUTE; /** A release returns once the lease is gone; the device's reclaim is awaited separately. */ const RELEASE_TIMEOUT = 3 * MINUTE; -/** Every call that asks the machine or the daemon a question, so one that never answers names itself. */ +/** Every call that asks the machine or the daemon a question (`simctl`, `adb`, the CLI), so one that never answers names itself. */ const QUERY_TIMEOUT = MINUTE; /** * The pool-off test creates each device from nothing (an Android image's first boot is a cold - * boot), so it is two cold leases, two warm ones, two releases and two end-state holds. The - * budgets of its steps (see `step`) sum to under this; each fires, naming its step, before it. + * boot), so it is two cold leases, two boots from shutdown, four releases and two end-state + * holds. The budgets of its steps (see `step`) sum to 84.5 min. Outside any step it makes at most + * eight one-minute queries, and on a failure `diagnose` asks for the devices (1 min) and `nuke` + * runs for up to 5 min: 84.5 + 8 + 1 + 5 = 98.5 min, under this, so each step budget fires, + * naming its step, before the test's own timeout, which names nothing. */ -const COLD_TEST_TIMEOUT = 80 * MINUTE; +const COLD_TEST_TIMEOUT = 100 * MINUTE; +/** The pool-off test's first step, which brings the daemon up and reads its catalog. */ +const DAEMON_START_BUDGET = 5 * MINUTE; const READY_TIMEOUT = 15 * MINUTE; /** How long a shut-down device is watched: longer than one pool tick (30s, WARM_POOL_TICK_MS), so a pool still on would have booted it. */ const HOLD = 45 * SECOND; @@ -81,14 +93,18 @@ const HOLD = 45 * SECOND; /** The installed iOS runtimes; empty when `xcrun simctl` is missing or unusable, so the lane skips. */ async function installedIosRuntimes(): Promise { try { - const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "runtimes", "-j"]); + const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "runtimes", "-j"], { + timeout: QUERY_TIMEOUT, + }); const parsed = JSON.parse(stdout) as { runtimes: { version: string; isAvailable: boolean; platform?: string }[]; }; return parsed.runtimes .filter((runtime) => runtime.isAvailable && (runtime.platform ?? "iOS") === "iOS") .map((runtime) => runtime.version); - } catch { + } catch (error: unknown) { + // A timed-out simctl is a hang to report, not "no iOS runtime installed". + if ((error as { killed?: boolean }).killed === true) throw error; return []; } } @@ -191,7 +207,7 @@ async function warmDaemon(options: { readonly enabled?: boolean } = {}): Promise const env = await withDaemon({ driver: "real" }); const adbServerPort = env.adbServerPort; if (adbServerPort === undefined) throw new Error("the real-SDK lane must allocate an adb port"); - const reported = await env.cli(["catalog", "--json"]); + const reported = await env.cli(["catalog", "--json"], { timeout: QUERY_TIMEOUT }); expect(reported.code, `catalog failed: ${reported.stderr}`).toBe(0); const platforms = (reported.json as { platforms: CatalogPlatform[] }).platforms; const catalogOf = (platform: Platform): CatalogPlatform => { @@ -211,6 +227,20 @@ async function warmDaemon(options: { readonly enabled?: boolean } = {}): Promise return { adbServerPort, deviceSet: iosDeviceSet(env.home), env, targets }; } +/** The business-event history, bounded like every other question put to the daemon. */ +function recordedEvents(env: TestEnv): Promise { + return env.events(undefined, QUERY_TIMEOUT); +} + +/** The two lines the verifier reads in the log: each device.ready event, with its time. */ +function logDeviceReady(ready: readonly RecordedEvent[]): void { + for (const entry of ready) { + console.info( + `device.ready ${JSON.stringify(entry.payload)} at ${new Date(entry.timestamp).toISOString()}`, + ); + } +} + async function devices(env: TestEnv): Promise { const listed = await env.cli(["list", "--devices"], { timeout: QUERY_TIMEOUT }); expect(listed.code, listed.stderr).toBe(0); @@ -351,7 +381,9 @@ async function expectGrantedWarm( granted: readonly Grant[], readyEvents: readonly { timestamp: number }[], ): Promise { - const grantedEvents = (await env.events()).filter((entry) => entry.event === "lease.granted"); + const grantedEvents = (await recordedEvents(env)).filter( + (entry) => entry.event === "lease.granted", + ); for (const grant of granted) { const entry = grantedEvents.find( (candidate) => (candidate.payload as { leaseId?: string }).leaseId === grant.lease.id, @@ -414,7 +446,7 @@ async function expectBootedFromShutdown( released: Grant, ): Promise { expect(bootingLines(second.stderr), `${platform}: the lease waited on a boot`).not.toEqual([]); - const granted = (await env.events()).find( + const granted = (await recordedEvents(env)).find( (entry) => entry.event === "lease.granted" && (entry.payload as { leaseId?: string }).leaseId === second.grant.lease.id, @@ -455,18 +487,13 @@ describe( await holds("one ready device per platform", HOLD, async () => { expectOneReadyPerPlatform(await devices(env)); }); - const recorded = await env.events(); + const recorded = await recordedEvents(env); expect( recorded.filter((entry) => entry.event.startsWith("lease.")), "no lease was asked for before the devices were ready", ).toEqual([]); const ready = recorded.filter((entry) => entry.event === "device.ready"); - // The two lines the verifier reads in the log: both device.ready events, before any lease. - for (const entry of ready) { - console.info( - `device.ready ${JSON.stringify(entry.payload)} at ${new Date(entry.timestamp).toISOString()}`, - ); - } + logDeviceReady(ready); expect( ready.map((entry) => (entry.payload as { deviceId: string }).deviceId).sort(), ).toEqual(rows.map((row) => row.id).sort()); @@ -478,23 +505,19 @@ describe( it( "grants the first lease of each targeted model warm with no booting progress, and after release each device is ready again", - { timeout: TEST_TIMEOUT }, + { timeout: WARM_ROUND_TIMEOUT }, async (context) => { const missing = await missingPlatform(); if (missing !== undefined) context.skip(missing); const { env, targets } = await warmDaemon(); try { await untilBothReady(env); - const before = await env.events(); + const before = await recordedEvents(env); const readyEvents = before.filter((entry) => entry.event === "device.ready"); expect(readyEvents.length, "device.ready events before the first lease").toBeGreaterThan( 1, ); - for (const entry of readyEvents) { - console.info( - `device.ready ${JSON.stringify(entry.payload)} at ${new Date(entry.timestamp).toISOString()}`, - ); - } + logDeviceReady(readyEvents); const granted: Grant[] = []; for (const platform of ["ios", "android"] as const) { @@ -516,7 +539,7 @@ describe( // A failed snapshot load falls back to a wipe and the pool boots the device back to ready, // so the state alone cannot tell the two apart: the reclaim's own strategy does. const android = granted.find((grant) => grant.device.spec.platform === "android"); - const reclaimed = (await env.events()).filter( + const reclaimed = (await recordedEvents(env)).filter( (entry) => entry.event === "device.reclaimed" && (entry.payload as { deviceId?: string }).deviceId === android?.device.id, @@ -537,11 +560,17 @@ describe( async (context) => { const missing = await missingPlatform(); if (missing !== undefined) context.skip(missing); - // The pool is off from the start, so the only device of each kind is the one the first lease - // creates: once released it is the only shut-down device of its kind, and the planner has no - // other to boot on the next lease. - const { adbServerPort, deviceSet, env, targets } = await warmDaemon({ enabled: false }); + // The daemon is restarted with the targets and enabled false in one config write, so the + // pool never has a target to boot: the only device of each kind is the one the first lease + // creates, and once released it is the only shut-down device of its kind, so the planner + // has no other to boot on the next lease. + let lane: Lane | undefined; try { + // Inside the try and a step, so a hung setup names itself and `diagnose` and `nuke` run. + lane = await step("start the daemon with the pool off", DAEMON_START_BUDGET, () => + warmDaemon({ enabled: false }), + ); + const { adbServerPort, deviceSet, env, targets } = lane; // Each first lease creates its device and boots it from nothing. const first = { android: await step( @@ -648,10 +677,10 @@ describe( release(env, second.android.grant), ); } catch (error: unknown) { - await diagnose(env); + if (lane !== undefined) await diagnose(lane.env); throw error; } finally { - await nuke(env); + if (lane !== undefined) await nuke(lane.env); } }, ); diff --git a/package.json b/package.json index 27e66f68..36285db9 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "test:unit": "vitest run --project unit", "test:changed": "vitest run --project unit --changed origin/main", "test:e2e": "pnpm run build && pnpm run build:e2e && vitest run --project e2e --tags-filter='!slow'", - "test:e2e:slow": "pnpm run build && pnpm run build:e2e && vitest run --project e2e --no-file-parallelism", + "test:e2e:slow": "pnpm run build && pnpm run build:e2e && vitest run --project e2e --no-file-parallelism --reporter=default", "test:console": "pnpm run build && pnpm run build:e2e && playwright test", "typecheck": "tsc --noEmit && tsc -p ui/tsconfig.json", "typecheck:e2e": "pnpm run build && tsc -p e2e/tsconfig.typecheck.json", From d5a82bd57402eab650cb91cee0b0e673e0d90ec3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 22:19:17 +0200 Subject: [PATCH 12/14] test: slow warm pool: the pool-off test's end-state checks are helpers (#372) Keeps the test body under the Fallow complexity threshold. 1 failing -> 0 failing --- e2e/slow-warm-pool.test.ts | 65 +++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index 797d8c40..b9ff29c6 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -462,6 +462,44 @@ async function expectBootedFromShutdown( ).toBe(released.device.id); } +/** Each platform has exactly one device, the one its first lease released, and it is shut down. */ +function expectOnlyReleasedDevices( + rows: readonly Row[], + first: Readonly>, +): void { + for (const platform of ["ios", "android"] as const) { + expect( + rows.filter((row) => row.spec.platform === platform).map((row) => [row.id, row.state]), + `${platform}: the released device is the only device and it is shut down`, + ).toEqual([[first[platform].grant.device.id, "shutdown"]]); + } +} + +/** Both second leases hold their device, and the simulator and the emulator really are running. */ +async function expectSecondBootsRunning( + env: TestEnv, + deviceSet: string, + adbServerPort: number, + second: Readonly>, +): Promise { + const rows = await devices(env); + for (const { grant } of [second.ios, second.android]) { + expect( + rows.find((row) => row.id === grant.device.id)?.state, + `${grant.device.spec.platform} device is leased after its second boot`, + ).toBe("leased"); + } + expect( + (await setDevices(deviceSet, { timeoutMs: QUERY_TIMEOUT })).find( + (device) => device.udid === second.ios.grant.device.driverDeviceId, + )?.state, + ).toBe("Booted"); + expect( + (await onlineEmulators(adbServerPort)).length, + "the emulator booted again and is online", + ).toBeGreaterThan(0); +} + async function nuke(env: TestEnv): Promise { await env .cli(["nuke", "--delete-devices", "--yes"], { timeout: 5 * MINUTE }) @@ -633,15 +671,7 @@ describe( // The next boot is clean: each lease waits on a boot, is granted, and its device really is running. // Each platform has exactly one device, the released one, so the next lease can only boot it. - const before = await devices(env); - for (const platform of ["ios", "android"] as const) { - expect( - before - .filter((row) => row.spec.platform === platform) - .map((row) => [row.id, row.state]), - `${platform}: the released device is the only device and it is shut down`, - ).toEqual([[first[platform].grant.device.id, "shutdown"]]); - } + expectOnlyReleasedDevices(await devices(env), first); const second = { android: await step("second lease: boot the emulator", LEASE_TIMEOUT, () => lease(env, targets.android, "again-android"), @@ -653,22 +683,7 @@ describe( for (const platform of ["ios", "android"] as const) { await expectBootedFromShutdown(env, platform, second[platform], first[platform].grant); } - const rows = await devices(env); - for (const { grant } of [second.ios, second.android]) { - expect( - rows.find((row) => row.id === grant.device.id)?.state, - `${grant.device.spec.platform} device is leased after its second boot`, - ).toBe("leased"); - } - expect( - (await setDevices(deviceSet, { timeoutMs: QUERY_TIMEOUT })).find( - (device) => device.udid === second.ios.grant.device.driverDeviceId, - )?.state, - ).toBe("Booted"); - expect( - (await onlineEmulators(adbServerPort)).length, - "the emulator booted again and is online", - ).toBeGreaterThan(0); + await expectSecondBootsRunning(env, deviceSet, adbServerPort, second); await step("release the second simulator lease", RELEASE_TIMEOUT + MINUTE, () => release(env, second.ios.grant), From cc87232746a739ef88147c7d46cd7e29effba745 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 22:19:57 +0200 Subject: [PATCH 13/14] test: e2e events helper passes a timeout only when given (#372) --- e2e/helpers/events.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e/helpers/events.ts b/e2e/helpers/events.ts index 3dbb13b7..176ae367 100644 --- a/e2e/helpers/events.ts +++ b/e2e/helpers/events.ts @@ -15,7 +15,11 @@ export async function events( since = "1h", timeout?: number, ): Promise { - const result = await cli(["events", "--since", since], env, { timeout }); + const result = await cli( + ["events", "--since", since], + env, + timeout === undefined ? {} : { timeout }, + ); if (result.code !== 0) { throw new Error(`simlock events failed (exit ${String(result.code)}): ${result.stderr}`); } From 8d62bbc0d83b1d6e9358173fc4d5382a1ab59fd2 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Tue, 6 Oct 2026 22:26:00 +0200 Subject: [PATCH 14/14] docs(e2e): the warm-pool timeout comments state the bounds the code has (#372) claims-only: 0 failing -> 0 failing --- e2e/slow-warm-pool.test.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/e2e/slow-warm-pool.test.ts b/e2e/slow-warm-pool.test.ts index b9ff29c6..f10ab75e 100644 --- a/e2e/slow-warm-pool.test.ts +++ b/e2e/slow-warm-pool.test.ts @@ -66,8 +66,9 @@ const TEST_TIMEOUT = 30 * MINUTE; /** * Two real boots, a lease and a release of each, and the wait for each to be ready again. The * waits alone (a 15 min readiness wait, two 10 min leases, two 3 min releases, two 15 min waits - * for ready again) add up to 71 min, plus at most 9 one-minute queries and a 5 min nuke: a wait - * that gives up names itself before this fires. + * for ready again) add up to 71 min, plus at most 9 one-minute queries and a 5 min nuke: 85 min, + * so each of those waits names itself before this fires. The setup (`warmDaemon`: the stale + * device-set sweep and the daemon start and stop calls) has no bound of its own; this timeout ends it. */ const WARM_ROUND_TIMEOUT = 90 * MINUTE; const LEASE_TIMEOUT = 10 * MINUTE; @@ -79,11 +80,11 @@ const QUERY_TIMEOUT = MINUTE; * The pool-off test creates each device from nothing (an Android image's first boot is a cold * boot), so it is two cold leases, two boots from shutdown, four releases and two end-state * holds. The budgets of its steps (see `step`) sum to 84.5 min. Outside any step it makes at most - * eight one-minute queries, and on a failure `diagnose` asks for the devices (1 min) and `nuke` - * runs for up to 5 min: 84.5 + 8 + 1 + 5 = 98.5 min, under this, so each step budget fires, + * nine one-minute queries, and on a failure `diagnose` asks for the devices (1 min) and `nuke` + * runs for up to 5 min: 84.5 + 9 + 1 + 5 = 99.5 min, under this, so each step budget fires, * naming its step, before the test's own timeout, which names nothing. */ -const COLD_TEST_TIMEOUT = 100 * MINUTE; +const COLD_TEST_TIMEOUT = 110 * MINUTE; /** The pool-off test's first step, which brings the daemon up and reads its catalog. */ const DAEMON_START_BUDGET = 5 * MINUTE; const READY_TIMEOUT = 15 * MINUTE; @@ -604,7 +605,8 @@ describe( // has no other to boot on the next lease. let lane: Lane | undefined; try { - // Inside the try and a step, so a hung setup names itself and `diagnose` and `nuke` run. + // Inside the try and a step, so a hung setup names itself. `lane` is unset then, so + // `diagnose` and `nuke` are skipped; they run once `warmDaemon` has returned. lane = await step("start the daemon with the pool off", DAEMON_START_BUDGET, () => warmDaemon({ enabled: false }), );