diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerListenerPeer.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerListenerPeer.swift index 3c014701f1..efe9197a2f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerListenerPeer.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerListenerPeer.swift @@ -3,8 +3,7 @@ import Network /// Who reached the runner listener, decided once per connection from its path. The host reaches the /// runner only over loopback (a simulator or macOS host, or usbmux on a device) or through the /// device's CoreDevice tunnel, a `utun` interface of type `.other`. A peer on any other interface -/// reached the device directly over a network it shares with it, and is closed before it can hold a -/// connection open. +/// reached the device directly over a shared network and requires explicit forwarding support. enum RunnerListenerPeer: Equatable { /// May hold an idle connection: the simulator listener watch (`runner-retention.ts`) does, to /// learn when the runner dies. @@ -12,6 +11,10 @@ enum RunnerListenerPeer: Equatable { case tunnel case directNetwork + func isAdmitted(acceptsForwardedConnections: Bool) -> Bool { + self != .directNetwork || acceptsForwardedConnections + } + static func of(_ connection: NWConnection) -> RunnerListenerPeer { classify( isLoopbackAddress: isLoopback(connection.endpoint), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift index f133004245..00587d95fb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift @@ -9,6 +9,7 @@ import Network struct RunnerListenerConfiguration { let port: NWEndpoint.Port let token: RunnerListenerToken + let bindAddress: IPv4Address? static func fromEnvironment( _ environment: [String: String] = ProcessInfo.processInfo.environment @@ -20,6 +21,11 @@ struct RunnerListenerConfiguration { else { return nil } - return RunnerListenerConfiguration(port: port, token: token) + var bindAddress: IPv4Address? + if let rawAddress = environment["AGENT_DEVICE_RUNNER_BIND_IP"] { + guard let address = IPv4Address(rawAddress), address != .any else { return nil } + bindAddress = address + } + return RunnerListenerConfiguration(port: port, token: token, bindAddress: bindAddress) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index f6e4d6d9bb..96144e3f26 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -124,8 +124,17 @@ extension RunnerTests { ) } let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png" - let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName) +#if os(iOS) + // A managed XCTest host can override TMPDIR outside the app container. + // The returned tmp/ path must resolve inside the runner's data container. + let temporaryDirectory = URL(fileURLWithPath: NSHomeDirectory(), isDirectory: true) + .appendingPathComponent("tmp", isDirectory: true) +#else + let temporaryDirectory = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) +#endif + let filePath = temporaryDirectory.appendingPathComponent(fileName).path do { + try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true) try pngData.write(to: URL(fileURLWithPath: filePath)) } catch { return Response( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift index 18dd1c10ca..9990e181b4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift @@ -36,12 +36,12 @@ extension RunnerTests { private func admit(_ connection: NWConnection, token: RunnerListenerToken) { let peer = RunnerListenerPeer.of(connection) - guard peer != .directNetwork else { + guard peer.isAdmitted(acceptsForwardedConnections: acceptsForwardedConnections) else { connection.cancel() return } let inbound = RunnerInboundRequest(token: token) - if peer == .tunnel { + if peer != .loopback { armDeadline(for: inbound, over: connection) } receiveRequest(inbound, over: connection) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 143ac75555..0918a48077 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -81,6 +81,7 @@ final class RunnerTests: XCTestCase { static let defaultRecordingFps: Int32 = 15 var listener: NWListener? var listenerToken: RunnerListenerToken? + var acceptsForwardedConnections = false var doneExpectation: XCTestExpectation? var transportConnections: [ObjectIdentifier: NWConnection] = [:] let shutdownResponseWrites = DispatchGroup() @@ -342,13 +343,14 @@ final class RunnerTests: XCTestCase { NSLog("AGENT_DEVICE_RUNNER_HEADLESS_STARTUP=1") guard let configuration = RunnerListenerConfiguration.fromEnvironment() else { NSLog("AGENT_DEVICE_RUNNER_LISTENER_CONFIGURATION_MISSING") - XCTFail("runner listener port or token is missing; refusing to listen") + XCTFail("runner listener port, token, or bind address is missing or invalid; refusing to listen") return } listenerToken = configuration.token + acceptsForwardedConnections = configuration.bindAddress != nil NSLog("AGENT_DEVICE_RUNNER_AUTH=required") NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", configuration.port.rawValue) - listener = try Self.makeRunnerListener(port: configuration.port) + listener = try Self.makeRunnerListener(port: configuration.port, bindAddress: configuration.bindAddress) // Reads the OS prompt strings off the alert path, so the first accept/dismiss deadline does not pay for it. DispatchQueue.global(qos: .utility).async { _ = RunnerAlertLabelVocabulary.system } DispatchQueue.global(qos: .utility).async { _ = RunnerLaunchConfirmationPrompt.system } @@ -397,9 +399,14 @@ final class RunnerTests: XCTestCase { } } - /// Simulators and macOS bind loopback: the host reaches them only through 127.0.0.1. A physical - /// device listens on every interface because the CoreDevice tunnel arrives on a network address. - static func makeRunnerListener(port: NWEndpoint.Port) throws -> NWListener { + /// Cloud simulators bind their assigned address; local simulators keep the loopback listener. + /// Physical devices accept CoreDevice tunnel connections on their network interface. + static func makeRunnerListener(port: NWEndpoint.Port, bindAddress: IPv4Address? = nil) throws -> NWListener { + if let bindAddress { + let parameters = NWParameters.tcp + parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(bindAddress), port: port) + return try NWListener(using: parameters) + } #if targetEnvironment(simulator) || os(macOS) let parameters = NWParameters.tcp parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(.loopback), port: port) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index f16705845a..ac426e2382 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -114,10 +114,8 @@ } } -// One window read. Both answers are required: an app with no window answers `frame` with -// `(0,0 0x0)` without raising, so only `exists` says whether the window is there. The window -// object is handed back because resolving this instance is what its `screen` will answer from. -// The frame is reported even when refused, so a refusal names what it looked at. +// Check existence before geometry: reading a missing window's frame records an XCTest failure, +// even when the exception is caught. Keep the resolved instance for its screen lookup. static RunnerApplicationScreenFailure RunnerResolveUsableWindow( id application, id _Nullable *window, @@ -125,23 +123,24 @@ static RunnerApplicationScreenFailure RunnerResolveUsableWindow( ) { __block id readWindow = nil; __block CGRect frame = CGRectNull; + __block BOOL frameRead = NO; __block BOOL resolved = NO; NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ id candidate = [[application valueForKey:@"windows"] valueForKey:@"firstMatch"]; NSNumber *present = [candidate valueForKey:@"exists"]; resolved = [present isKindOfClass:NSNumber.class] && present.boolValue; + if (!resolved) return; NSValue *frameValue = [candidate valueForKey:@"frame"]; [frameValue getValue:&frame size:sizeof(frame)]; + frameRead = YES; readWindow = candidate; }]; *observedFrame = frame; if (exception != nil || !resolved || !AgentDeviceForegroundFrameIsMeasurable(frame)) { NSLog( - @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) raised=%@ resolved=%@", - frame.origin.x, - frame.origin.y, - frame.size.width, - frame.size.height, + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=%@ raised=%@ resolved=%@", + frameRead ? [NSString stringWithFormat:@"(%g,%g %gx%g)", + frame.origin.x, frame.origin.y, frame.size.width, frame.size.height] : @"unread", exception != nil ? @"yes" : @"no", resolved ? @"yes" : @"no" ); diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+EnvironmentTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+EnvironmentTests.swift index 70df4e515a..1fd965f9cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+EnvironmentTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+EnvironmentTests.swift @@ -31,5 +31,15 @@ extension RunnerTests { ) } } + func testListenerConfigurationAcceptsOnlyConcreteBindAddresses() { + let base = ["AGENT_DEVICE_RUNNER_PORT": "8100", "AGENT_DEVICE_RUNNER_TOKEN": String(repeating: "t", count: 32)] + XCTAssertNil(RunnerListenerConfiguration.fromEnvironment(base)?.bindAddress) + let remote = base.merging(["AGENT_DEVICE_RUNNER_BIND_IP": "10.244.2.3"]) { $1 } + XCTAssertEqual(RunnerListenerConfiguration.fromEnvironment(remote)?.bindAddress?.debugDescription, "10.244.2.3") + for invalid in ["", "0.0.0.0", "localhost", "10.244.2.999"] { + XCTAssertNil(RunnerListenerConfiguration.fromEnvironment(base.merging(["AGENT_DEVICE_RUNNER_BIND_IP": invalid]) { $1 })) + } + } + } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift index 1bc8c37b91..ede54c72ac 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift @@ -101,7 +101,7 @@ extension RunnerTests { let message = try XCTUnwrap(response.data?.message) // The answer names the file rather than carrying bytes, and the name is a host-resolvable form: // absolute on macOS, container-relative `tmp/…` on iOS. Both name the one file just written - // into the runner's temporary directory. + // into the runner's data container on iOS, regardless of the host's TMPDIR. let fileName = URL(fileURLWithPath: message).lastPathComponent XCTAssertTrue(message.hasSuffix(fileName), message) #if os(iOS) @@ -109,7 +109,11 @@ extension RunnerTests { #elseif os(macOS) XCTAssertTrue(message.hasPrefix("/"), message) #endif +#if os(iOS) + let storedPath = (NSHomeDirectory() as NSString).appendingPathComponent(message) +#else let storedPath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName) +#endif XCTAssertEqual(try Data(contentsOf: URL(fileURLWithPath: storedPath)), pngData) XCTAssertEqual(response.data?.screenshotMetadata?.pixelsPerPoint, 3) try? FileManager.default.removeItem(atPath: storedPath) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ListenerPeerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ListenerPeerTests.swift index 280fabd4f1..825e8f946b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ListenerPeerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ListenerPeerTests.swift @@ -22,6 +22,15 @@ extension RunnerTests { } } + func testListenerPeerRequiresExplicitForwardingForDirectNetworkAccess() { + XCTAssertFalse(RunnerListenerPeer.directNetwork.isAdmitted(acceptsForwardedConnections: false)) + XCTAssertTrue(RunnerListenerPeer.directNetwork.isAdmitted(acceptsForwardedConnections: true)) + for peer: RunnerListenerPeer in [.loopback, .tunnel] { + XCTAssertTrue(peer.isAdmitted(acceptsForwardedConnections: false)) + XCTAssertTrue(peer.isAdmitted(acceptsForwardedConnections: true)) + } + } + func testListenerPeerRecognizesEveryLoopbackAddressForm() { let peer = { (host: String) in NWEndpoint.hostPort(host: NWEndpoint.Host(host), port: 8100) } XCTAssertTrue(RunnerListenerPeer.isLoopback(peer("127.0.0.1"))) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift index eeabd71dd5..2d1aaa86b9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift @@ -151,6 +151,16 @@ extension RunnerTests { #endif } + func testRunnerListenerRequiresTheConfiguredBindAddressAndPort() throws { + let address = try XCTUnwrap(IPv4Address("10.23.45.67")) + let port = try XCTUnwrap(NWEndpoint.Port(rawValue: 8100)) + let listener = try Self.makeRunnerListener(port: port, bindAddress: address) + XCTAssertEqual( + listener.parameters.requiredLocalEndpoint, + .hostPort(host: .ipv4(address), port: port) + ) + } + func testListenerRefusesRequestsWithoutTheTokenBeforeDecoding() throws { let listener = try startTransportListener() defer { listener.cancel() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 289012ccf7..cbf7d70a39 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -202,6 +202,7 @@ extension RunnerTests { XCTAssertEqual(failure, .unresolvedWindow) XCTAssertNil(screen) XCTAssertEqual(displayID, 0) + XCTAssertFalse(window.resolved, "a missing window's frame lookup records an XCTest failure") } /// The sole-screen rule's decision, at the seam the production probe feeds: the values the /// AXRuntime probe reads in, never a device list. The windowed/unknown rows below fail without diff --git a/packages/apple-runner/src/__tests__/fake-runner-server.ts b/packages/apple-runner/src/__tests__/fake-runner-server.ts index ddca93b480..2bea64b953 100644 --- a/packages/apple-runner/src/__tests__/fake-runner-server.ts +++ b/packages/apple-runner/src/__tests__/fake-runner-server.ts @@ -16,6 +16,7 @@ export type FakeRunnerResponse = | { kind: 'hangUp' } /** Hangs up on this request and every later one for the command: the entry is never consumed. */ | { kind: 'hangUpAlways' } + | { kind: 'httpErrorAlways'; status: number } /** Hangs up and stops listening, as a runner process that died mid-command. */ | { kind: 'exit' } /** Never answers, as a runner still executing the command when its caller gives up. */ @@ -104,7 +105,9 @@ function recordFakeRunnerRequest(req: http.IncomingMessage, raw: string): FakeRu function takeScriptedResponse( queue: FakeRunnerResponse[] | undefined, ): FakeRunnerResponse | undefined { - return queue?.[0]?.kind === 'hangUpAlways' ? queue[0] : queue?.shift(); + return queue?.[0]?.kind === 'hangUpAlways' || queue?.[0]?.kind === 'httpErrorAlways' + ? queue[0] + : queue?.shift(); } function writeFakeRunnerResponse( @@ -117,6 +120,10 @@ function writeFakeRunnerResponse( return; } if (next.kind === 'hold') return; + if (next.kind === 'httpErrorAlways') { + res.writeHead(next.status).end('upstream unavailable'); + return; + } if (next.kind === 'hangUp' || next.kind === 'hangUpAlways') { res.destroy(); return; diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index 3fcb985970..ce504f563b 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -1,3 +1,4 @@ +import { makeRunnerSession as makeLocalRunnerSession } from './runner-session-fixtures.ts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { afterEach, test, vi } from 'vitest'; @@ -40,7 +41,7 @@ afterEach(async () => { }); function runnerSession(port: number): RunnerSession { - return { + return makeLocalRunnerSession({ sessionId: `fake:${port}`, device: IOS_SIMULATOR, deviceId: IOS_SIMULATOR.id, @@ -51,7 +52,7 @@ function runnerSession(port: number): RunnerSession { child: { pid: process.pid, exitCode: null }, state: 'ready', commandCharges: new RunnerCommandAccounting(), - }; + }); } const TAP: RunnerCommand = { command: 'tap', x: 10, y: 10, commandId: 'cmd-1' }; diff --git a/packages/apple-runner/src/__tests__/index-recovery-wiring.test.ts b/packages/apple-runner/src/__tests__/index-recovery-wiring.test.ts index f528cda479..db11df3d2f 100644 --- a/packages/apple-runner/src/__tests__/index-recovery-wiring.test.ts +++ b/packages/apple-runner/src/__tests__/index-recovery-wiring.test.ts @@ -11,7 +11,10 @@ import { withAppleRunnerProvider } from '../runner-provider.ts'; import { classifyRunnerReportedError, type RunnerCommand } from '../runner-contract.ts'; import { RUNNER_REPLY_LOST_REASON } from '../runner-error-classification.ts'; import { createRunnerPhaseBudget, requireRunnerPhaseRemainingMs } from '../runner-phase-budget.ts'; -import { captureDiagnostics } from './runner-session-fixtures.ts'; +import { + captureDiagnostics, + makeRunnerSession as makeLocalRunnerSession, +} from './runner-session-fixtures.ts'; import { startFakeRunnerServer, type FakeRunnerResponse, @@ -104,7 +107,7 @@ afterEach(async () => { }); function makeRunnerSession(port: number, sessionId = `wiring:${port}`): RunnerSession { - const session: RunnerSession = { + const session: RunnerSession = makeLocalRunnerSession({ sessionId, device: IOS_SIMULATOR, deviceId: IOS_SIMULATOR.id, @@ -115,7 +118,7 @@ function makeRunnerSession(port: number, sessionId = `wiring:${port}`): RunnerSe child: { pid: process.pid, exitCode: null }, state: 'ready', commandCharges: new RunnerCommandAccounting(), - }; + }); return session; } diff --git a/packages/apple-runner/src/__tests__/index-runner-requests.test.ts b/packages/apple-runner/src/__tests__/index-runner-requests.test.ts index 8575589e20..a11ada6d05 100644 --- a/packages/apple-runner/src/__tests__/index-runner-requests.test.ts +++ b/packages/apple-runner/src/__tests__/index-runner-requests.test.ts @@ -73,6 +73,7 @@ test('runner-internal request sites build exactly their runner-requests.json ent }); const session = makeRunnerSession({ port: server.port, + testPromise: new Promise(() => {}), xctestrunPath: path.join(scratch, 'runner.xctestrun'), jsonPath: path.join(scratch, 'runner.json'), }); @@ -91,6 +92,7 @@ test('runner-internal request sites build exactly their runner-requests.json ent async (request) => await executeRunnerCommand(IOS_SIMULATOR, request, {}), ); await notifyIosRunnerAppRelaunched(IOS_SIMULATOR); + vi.spyOn(session.launch, 'waitForExit').mockResolvedValue(true); await disposeRunnerSession(session); appleRunnerTestHost.update({ diff --git a/packages/apple-runner/src/__tests__/runner-adoption.test.ts b/packages/apple-runner/src/__tests__/runner-adoption.test.ts index 0261940ca4..1914e6b8a9 100644 --- a/packages/apple-runner/src/__tests__/runner-adoption.test.ts +++ b/packages/apple-runner/src/__tests__/runner-adoption.test.ts @@ -191,11 +191,11 @@ test('adoption succeeds for a live, matching, probe-healthy runner', async () => expect(session).not.toBeNull(); expect(session?.endpoint.port).toBe(lease.port); expect(session?.state).toBe('ready'); - expect(session?.child.pid).toBe(424242); + expect(session?.launch.local?.child?.pid).toBe(424242); expect(session?.sessionId).toBe(lease.sessionId); expect(session?.xctestrunArtifact?.reason).toBe('adopted_from_lease'); expect(session?.xctestrunArtifact?.cacheKey).toBe(lease.cacheKey); - expect(session?.lease?.cacheKey).toBe(lease.cacheKey); + expect(session?.launch.local?.lease?.cacheKey).toBe(lease.cacheKey); // Adoption transfers ownership: the lease on disk now belongs to us. expect(readStaleRunnerLease(simulator.id)).toBeNull(); }); @@ -213,7 +213,7 @@ test('the probe and the adopted session carry the token the runner was launched }); expect(session).toMatchObject({ endpoint: { port: lease.port, token: 'runner-token-fixture' }, - lease: { runnerToken: 'runner-token-fixture' }, + launch: { local: { lease: { runnerToken: 'runner-token-fixture' } } }, }); }); @@ -365,7 +365,7 @@ test('adoption accepts a legacy lease whose live pid is runner-shaped', async () const session = await tryAdoptRunnerSessionFromLease(simulator, {}); expect(session?.state).toBe('ready'); - expect(session?.child.pid).toBe(424242); + expect(session?.launch.local?.child?.pid).toBe(424242); }); test('a runner for a simulator in a custom simulator set is adopted', async () => { @@ -492,7 +492,7 @@ test('physical coredevice lane adopts the detached runner and names its lane', a const session = await tryAdoptRunnerSessionFromLease(physicalCoreDevice, {}); expect(session?.state).toBe('ready'); - expect(session?.child.pid).toBe(lease.runnerPid); + expect(session?.launch.local?.child?.pid).toBe(lease.runnerPid); expect(session?.endpoint.port).toBe(lease.port); expect( emittedDiagnostics.find((event) => event.phase === 'ios_runner_lease_adopted')?.data, diff --git a/packages/apple-runner/src/__tests__/runner-command-recovery.test.ts b/packages/apple-runner/src/__tests__/runner-command-recovery.test.ts index 51afdd5128..7cd0bc85e8 100644 --- a/packages/apple-runner/src/__tests__/runner-command-recovery.test.ts +++ b/packages/apple-runner/src/__tests__/runner-command-recovery.test.ts @@ -1,3 +1,4 @@ +import { makeRunnerSession as makeLocalRunnerSession } from './runner-session-fixtures.ts'; import assert from 'node:assert/strict'; import { RunnerCommandAccounting } from '../runner-session-types.ts'; import { afterEach, test, vi } from 'vitest'; @@ -27,7 +28,7 @@ afterEach(async () => { }); function makeRunnerSession(port: number): RunnerSession { - return { + return makeLocalRunnerSession({ sessionId: `fake:${port}`, device: IOS_SIMULATOR, deviceId: IOS_SIMULATOR.id, @@ -38,7 +39,7 @@ function makeRunnerSession(port: number): RunnerSession { child: { pid: process.pid, exitCode: null }, state: 'ready', commandCharges: new RunnerCommandAccounting(), - }; + }); } function tapCommand(commandId = 'cmd-1'): RunnerCommand { @@ -290,3 +291,29 @@ test('a truncated retained response is not recovered', async () => { }); assert.equal(invalidate.mock.calls.length, 0); }); + +for (const command of [ + tapCommand(), + { command: 'snapshot', commandId: 'snapshot-1' } as RunnerCommand, +]) { + test(`a forwarding outage after ${command.command} keeps the generation without a status request`, async () => { + const { result, invalidate } = await runRecovery({ + command, + script: [{ kind: 'ok', data: { lifecycleState: 'completed' } }], + transportError: new AppError('COMMAND_FAILED', 'gateway unavailable', { + reason: 'runner_forwarding_unavailable', + }), + }); + await assert.rejects(result, (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.dispatched, 'unknown'); + assert.equal( + error.details?.reason, + command.command === 'tap' ? 'runner_reply_lost' : 'runner_forwarding_unavailable', + ); + return true; + }); + assert.equal(server?.requests.length, 0); + assert.equal(invalidate.mock.calls.length, 0); + }); +} diff --git a/packages/apple-runner/src/__tests__/runner-disposal.test.ts b/packages/apple-runner/src/__tests__/runner-disposal.test.ts index 4f3d81614d..1c48fa87f3 100644 --- a/packages/apple-runner/src/__tests__/runner-disposal.test.ts +++ b/packages/apple-runner/src/__tests__/runner-disposal.test.ts @@ -3,7 +3,10 @@ import { IOS_SIMULATOR, MACOS_DEVICE, TVOS_SIMULATOR } from './device-fixtures.t import type { ExecResult } from '@agent-device/host-kit/command'; import { RunnerCommandAccounting, type RunnerSession } from '../runner-session-types.ts'; import { appleRunnerTestHost } from '../test-host.ts'; -import { makeRunnerLease } from './runner-session-fixtures.ts'; +import { + makeRunnerLease, + makeRunnerSession as makeLocalRunnerSession, +} from './runner-session-fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; const { mockCleanupTempFile } = vi.hoisted(() => ({ @@ -75,8 +78,8 @@ test('macOS runner abort waits for XCTest teardown after SIGINT', async () => { await abort; expect(runnerSignals(session)).toEqual(['SIGINT']); - expect(mockCleanupTempFile).toHaveBeenCalledWith(session.xctestrunPath); - expect(mockCleanupTempFile).toHaveBeenCalledWith(session.jsonPath); + expect(mockCleanupTempFile).toHaveBeenCalledWith(session.launch.local!.xctestrunPath); + expect(mockCleanupTempFile).toHaveBeenCalledWith(session.launch.local!.jsonPath); }); test('macOS runner abort stages TERM after the interrupt grace period', async () => { @@ -168,7 +171,7 @@ test('simulator disposal skips container-app termination after a foreign takeove await disposeRunnerSession(session, { graceful: false, waitTimeoutMs: 1 }); expect(simulatorTerminateCalls()).toEqual([]); - expect(mockCleanupTempFile).toHaveBeenCalledWith(session.xctestrunPath); + expect(mockCleanupTempFile).toHaveBeenCalledWith(session.launch.local!.xctestrunPath); }); test('disposal serializes behind a successor reclaim window and never terminates its runner', async () => { @@ -256,9 +259,9 @@ function simulatorTerminateCalls(): unknown[] { function makeRunnerSession( device: RunnerSession['device'], testPromise: Promise, - overrides: Partial = {}, + overrides: Parameters[0] = {}, ): RunnerSession { - return { + return makeLocalRunnerSession({ sessionId: `${device.id}:8123:test`, device, deviceId: device.id, @@ -270,12 +273,12 @@ function makeRunnerSession( state: 'ready', commandCharges: new RunnerCommandAccounting(), ...overrides, - }; + }); } function runnerSignals(session: RunnerSession): NodeJS.Signals[] { return mockSignalProcessGroupBestEffort.mock.calls - .filter(([pid]) => pid === session.child.pid) + .filter(([pid]) => pid === session.launch.local!.child?.pid) .map(([, signal]) => signal as NodeJS.Signals); } diff --git a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts index 7c881cc936..521450a91d 100644 --- a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts +++ b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts @@ -540,3 +540,24 @@ test('a runner that does not require the token is session-fatal and never retrie assert.equal(shouldRebuildCachedRunnerArtifact(refusal), false); } }); + +test('forwarding recovery requires the typed gateway failure, never matching prose', () => { + const message = 'Runner HTTP forwarding is unavailable'; + const typed = new AppError('COMMAND_FAILED', message, { + reason: 'runner_forwarding_unavailable', + status: 502, + }); + assert.equal(isRetryableRunnerError(typed), true); + assert.equal(shouldRestartRunnerBeforeCommandSend(typed, { command: 'tap' }), false); + assert.equal(shouldRebuildCachedRunnerArtifact(typed), false); + assert.equal( + shouldRestartRunnerAfterReadinessPreflight( + new AppError('COMMAND_FAILED', message, { + ...typed.details, + runnerReadinessPreflightFailed: true, + }), + ), + false, + ); + assert.equal(isRetryableRunnerError(new AppError('COMMAND_FAILED', message)), false); +}); diff --git a/packages/apple-runner/src/__tests__/runner-exchange-deadline-journal.test.ts b/packages/apple-runner/src/__tests__/runner-exchange-deadline-journal.test.ts index ffbfcfe787..563e6dc6b1 100644 --- a/packages/apple-runner/src/__tests__/runner-exchange-deadline-journal.test.ts +++ b/packages/apple-runner/src/__tests__/runner-exchange-deadline-journal.test.ts @@ -1,3 +1,4 @@ +import { makeRunnerSession as makeLocalRunnerSession } from './runner-session-fixtures.ts'; import assert from 'node:assert/strict'; import type { ExecResult } from '@agent-device/host-kit/command'; import { withRequestDeadline } from '@agent-device/test-kit'; @@ -31,7 +32,7 @@ test('a daemon deadline follows the ordinary canceled Apple runner journal drain return realSetTimeout(callback, delay, ...args); }); try { - const session: RunnerSession = { + const session: RunnerSession = makeLocalRunnerSession({ sessionId: `deadline-journal:${runner.port}`, device: IOS_SIMULATOR, deviceId: IOS_SIMULATOR.id, @@ -42,7 +43,7 @@ test('a daemon deadline follows the ordinary canceled Apple runner journal drain child: { pid: process.pid, exitCode: null }, state: 'ready', commandCharges: new RunnerCommandAccounting(), - }; + }); const requestId = `deadline-journal:${runner.port}`; const deadlineResult = withRequestDeadline(requestId, 25, async () => { diff --git a/packages/apple-runner/src/__tests__/runner-exchange.test.ts b/packages/apple-runner/src/__tests__/runner-exchange.test.ts index 81c1688996..2105a5a448 100644 --- a/packages/apple-runner/src/__tests__/runner-exchange.test.ts +++ b/packages/apple-runner/src/__tests__/runner-exchange.test.ts @@ -16,7 +16,10 @@ vi.mock('../runner-transport.ts', async (importOriginal) => { import { executeRunnerExchange } from '../runner-exchange.ts'; import { RunnerCommandAccounting, type RunnerSession } from '../runner-session-types.ts'; import { IOS_SIMULATOR } from './device-fixtures.ts'; -import { captureDiagnostics } from './runner-session-fixtures.ts'; +import { + captureDiagnostics, + makeRunnerSession as makeLocalRunnerSession, +} from './runner-session-fixtures.ts'; import { startFakeRunnerServer, type FakeRunnerResponse, @@ -32,7 +35,7 @@ afterEach(async () => { }); function sessionFor(port: number): RunnerSession { - return { + return makeLocalRunnerSession({ sessionId: `exchange:${port}`, device: IOS_SIMULATOR, deviceId: IOS_SIMULATOR.id, @@ -43,7 +46,7 @@ function sessionFor(port: number): RunnerSession { child: { pid: process.pid, exitCode: null }, state: 'ready', commandCharges: new RunnerCommandAccounting(), - }; + }); } test('the exchange preflights a mutation and settles only its answer', async () => { diff --git a/packages/apple-runner/src/__tests__/runner-launcher-provider.test.ts b/packages/apple-runner/src/__tests__/runner-launcher-provider.test.ts new file mode 100644 index 0000000000..096c43afaa --- /dev/null +++ b/packages/apple-runner/src/__tests__/runner-launcher-provider.test.ts @@ -0,0 +1,449 @@ +import { expect, test, vi } from 'vitest'; +import { createLaunchedAppleRunnerProvider } from '../runner-launcher-provider.ts'; +import type { RunnerLauncher } from '../runner-launcher.ts'; +import { IOS_SIMULATOR } from './device-fixtures.ts'; +import { appleRunnerTestHost } from '../test-host.ts'; + +// A launcher is the production boundary for remote work; it has no local PID or artifact. +test('managed launch preparation consumes the same explicit startup budget as local preparation', async () => { + const clock = vi.spyOn(Date, 'now').mockReturnValue(100_000); + const localTool = vi.fn(() => { + throw new Error('Managed launch reached local tooling'); + }); + appleRunnerTestHost.update({ runXcrun: localTool, runCmdBackground: localTool }); + let finish!: () => void; + const exited = new Promise((resolve) => { + finish = resolve; + }); + const launcher: RunnerLauncher = { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch() { + clock.mockReturnValue(150_000); + return { + exited, + output: (async function* () { + yield ''; + await exited; + })(), + terminate: async () => { + finish(); + }, + }; + }, + }; + const provider = createLaunchedAppleRunnerProvider(IOS_SIMULATOR, launcher, {}); + try { + const session = await provider.ensure({ startupTimeoutMs: 60_000 }); + expect(session.launchDeadline?.remainingMs()).toBe(10_000); + expect(session.launch.local).toBeUndefined(); + expect(session.xctestrunArtifact).toBeUndefined(); + expect(localTool).not.toHaveBeenCalled(); + } finally { + await provider.stop(); + clock.mockRestore(); + } +}); + +for (const operation of ['stop', 'release'] as const) { + test(`${operation} cancels a pending launch and admits a subsequent generation`, async () => { + let attempts = 0; + let entered!: () => void; + const launching = new Promise((resolve) => { + entered = resolve; + }); + let finish!: () => void; + const exited = new Promise((resolve) => { + finish = resolve; + }); + const provider = createLaunchedAppleRunnerProvider( + IOS_SIMULATOR, + { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch(_endpoint, signal) { + attempts++; + if (attempts === 1) { + entered(); + await new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }); + } + return { + exited, + output: (async function* () { + yield ''; + await exited; + })(), + terminate: async () => { + finish(); + }, + }; + }, + }, + {}, + ); + try { + const starting = provider.ensure({}); + const canceled = expect(starting).rejects.toMatchObject({ + details: { runnerStartRetired: true }, + }); + await launching; + if (operation === 'stop') await provider.stop(); + else await provider.release(false); + await canceled; + await provider.ensure({}); + expect(attempts).toBe(2); + } finally { + await provider.stop(); + } + }); +} + +test('a failed termination stays owned and blocks replacement until termination succeeds', async () => { + let attempts = 0; + let failTermination = true; + let finish!: () => void; + const provider = createLaunchedAppleRunnerProvider( + IOS_SIMULATOR, + { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch() { + attempts++; + const exited = new Promise((resolve) => { + finish = resolve; + }); + return { + exited, + output: (async function* () { + yield ''; + await exited; + })(), + terminate: async () => { + if (failTermination) throw new Error('termination unavailable'); + finish(); + }, + }; + }, + }, + {}, + ); + try { + await provider.ensure({}); + await expect(provider.stop()).rejects.toMatchObject({ + details: { reason: 'runner_termination_failed' }, + }); + await expect(provider.ensure({})).rejects.toMatchObject({ + details: { reason: 'runner_termination_failed' }, + }); + expect(attempts).toBe(1); + failTermination = false; + await provider.ensure({}); + expect(attempts).toBe(2); + } finally { + failTermination = false; + await provider.stop(); + } +}); + +test('an output failure and failed termination remain observable without an unhandled rejection', async () => { + let failTermination = true; + let failOutput!: () => void; + let finish!: () => void; + const outputFailure = new Promise((resolve) => { + failOutput = resolve; + }); + const exited = new Promise((resolve) => { + finish = resolve; + }); + const provider = createLaunchedAppleRunnerProvider( + IOS_SIMULATOR, + { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch() { + return { + exited, + output: (async function* () { + yield ''; + await outputFailure; + throw new Error('output lost'); + })(), + terminate: async () => { + if (failTermination) throw new Error('termination unavailable'); + finish(); + }, + }; + }, + }, + {}, + ); + try { + const session = await provider.ensure({}); + failOutput(); + await expect(session.launch?.output).resolves.toBeUndefined(); + expect(session.state).toBe('draining'); + await expect(provider.ensure({})).rejects.toMatchObject({ + details: { reason: 'runner_termination_failed' }, + }); + } finally { + failTermination = false; + await provider.stop(); + } +}); + +test('retained reopen shares a generation and an exited generation flushes logs before replacement', async () => { + const { startFakeRunnerServer } = await import('./fake-runner-server.ts'); + const { mkdtempForTestSync } = await import('./tmp-dir.ts'); + const { readFile } = await import('node:fs/promises'); + const path = await import('node:path'); + const server = await startFakeRunnerServer({}); + const logPath = path.join(mkdtempForTestSync('managed-runner-'), 'runner.log'); + let attempts = 0; + let finish!: () => void; + const provider = createLaunchedAppleRunnerProvider( + IOS_SIMULATOR, + { + endpoint: { port: server.port, http: { url: `http://127.0.0.1:${server.port}/command` } }, + async launch() { + const generation = ++attempts; + const exited = new Promise((resolve) => { + finish = resolve; + }); + return { + exited, + output: (async function* () { + await exited; + yield `generation ${generation} ended\n`; + })(), + terminate: async () => { + finish(); + }, + }; + }, + }, + { AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS: '0' }, + ); + try { + const first = await provider.ensure({ logPath }); + await provider.runCommand(IOS_SIMULATOR, { command: 'uptime' }, { logPath }); + await provider.release(true); + const retained = await provider.ensure({ logPath }); + expect(retained.sessionId).toBe(first.sessionId); + expect(attempts).toBe(1); + expect(server.requests.some((request) => request.command === 'shutdown')).toBe(false); + finish(); + await first.launch?.output; + expect(provider.hasLiveSession()).toBe(false); + const replacement = await provider.ensure({ logPath }); + expect(replacement.sessionId).not.toBe(first.sessionId); + expect(replacement.endpoint.token).not.toBe(first.endpoint.token); + expect(attempts).toBe(2); + expect(await readFile(logPath, 'utf8')).toContain('generation 1 ended'); + } finally { + await provider.stop(); + await server.close(); + } +}); + +test('a canceled launch retains a generation whose termination failed before publication', async () => { + let attempts = 0; + let failTermination = true; + let finish!: () => void; + const cancellation = new AbortController(); + const provider = createLaunchedAppleRunnerProvider( + IOS_SIMULATOR, + { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch() { + attempts++; + const exited = new Promise((resolve) => { + finish = resolve; + }); + cancellation.abort(new Error('caller left')); + return { + exited, + output: (async function* () { + yield ''; + await exited; + })(), + terminate: async () => { + if (failTermination) throw new Error('termination unavailable'); + finish(); + }, + }; + }, + }, + {}, + ); + try { + await expect(provider.ensure({ signal: cancellation.signal })).rejects.toMatchObject({ + details: { reason: 'request_canceled' }, + }); + await expect(provider.ensure({})).rejects.toMatchObject({ + details: { reason: 'runner_termination_failed' }, + }); + expect(attempts).toBe(1); + failTermination = false; + await provider.ensure({}); + expect(attempts).toBe(2); + } finally { + failTermination = false; + await provider.stop(); + } +}); + +test('an immediate output failure cannot publish a draining generation as ready', async () => { + let attempts = 0; + let failTermination = true; + let finish!: () => void; + const provider = createLaunchedAppleRunnerProvider( + IOS_SIMULATOR, + { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch() { + const generation = ++attempts; + const exited = new Promise((resolve) => { + finish = resolve; + }); + return { + exited, + output: (async function* () { + if (generation === 1) throw new Error('output lost before publication'); + yield ''; + await exited; + })(), + terminate: async () => { + if (failTermination) throw new Error('termination unavailable'); + finish(); + }, + }; + }, + }, + {}, + ); + try { + await expect(provider.ensure({})).rejects.toMatchObject({ + details: { reason: 'runner_termination_failed' }, + }); + await expect(provider.ensure({})).rejects.toMatchObject({ + details: { reason: 'runner_termination_failed' }, + }); + expect(attempts).toBe(1); + failTermination = false; + await provider.ensure({}); + expect(attempts).toBe(2); + } finally { + failTermination = false; + await provider.stop(); + } +}); + +for (const operation of ['abort', 'stop'] as const) { + test(`global ${operation} finishes all sweeps after a registered termination fails`, async () => { + const { abortAllIosRunnerSessions, stopAllIosRunnerSessions } = + await import('../runner-session.ts'); + let failRegistered = true; + const registered = (suffix: string, shouldFail: () => boolean) => { + let finish!: () => void; + const exited = new Promise((resolve) => { + finish = resolve; + }); + return createLaunchedAppleRunnerProvider( + { ...IOS_SIMULATOR, id: `${IOS_SIMULATOR.id}-${suffix}` }, + { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch() { + return { + exited, + output: (async function* () { + yield ''; + await exited; + })(), + async terminate() { + if (shouldFail()) throw new Error('registered termination unavailable'); + finish(); + }, + }; + }, + }, + {}, + ); + }; + const failing = registered('failed-stop', () => failRegistered); + const healthy = registered('healthy-stop', () => false); + await failing.ensure({}); + await healthy.ensure({}); + let enter!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + let aborted!: () => void; + const canceled = new Promise((resolve) => { + aborted = resolve; + }); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let finish!: () => void; + const exited = new Promise((resolve) => { + finish = resolve; + }); + let failTermination = true; + const provider = createLaunchedAppleRunnerProvider( + IOS_SIMULATOR, + { + endpoint: { port: 8100, http: { url: 'https://instance.invalid/command' } }, + async launch(_endpoint, signal) { + signal.addEventListener('abort', aborted, { once: true }); + enter(); + await held; + return { + exited, + output: (async function* () { + yield ''; + await exited; + })(), + terminate: async () => { + if (failTermination) throw new Error('termination unavailable'); + finish(); + }, + }; + }, + }, + {}, + ); + try { + const starting = provider.ensure({}); + const failedStart = expect(starting).rejects.toMatchObject({ + details: { reason: 'runner_termination_failed' }, + }); + await entered; + const shutdown = + operation === 'abort' ? abortAllIosRunnerSessions() : stopAllIosRunnerSessions(); + let shutdownSettled = false; + const shutdownResult = shutdown.then( + () => ({ status: 'resolved' as const, error: undefined }), + (error: unknown) => ({ status: 'rejected' as const, error }), + ); + void shutdownResult.then(() => { + shutdownSettled = true; + }); + await canceled; + await new Promise((resolve) => setImmediate(resolve)); + const waitedForLaunch = !shutdownSettled; + release(); + await failedStart; + const result = await shutdownResult; + expect(waitedForLaunch).toBe(true); + expect(result.status).toBe('rejected'); + expect(result.error).toBeInstanceOf(AggregateError); + expect(provider.isStarted()).toBe(true); + expect(failing.isStarted()).toBe(true); + expect(healthy.isStarted()).toBe(false); + } finally { + failTermination = false; + failRegistered = false; + release(); + await Promise.all([provider.stop(), failing.stop(), healthy.stop()]); + } + }); +} diff --git a/packages/apple-runner/src/__tests__/runner-lifecycle-readiness-preflight.test.ts b/packages/apple-runner/src/__tests__/runner-lifecycle-readiness-preflight.test.ts index 28fbeb496d..66c7c79a6e 100644 --- a/packages/apple-runner/src/__tests__/runner-lifecycle-readiness-preflight.test.ts +++ b/packages/apple-runner/src/__tests__/runner-lifecycle-readiness-preflight.test.ts @@ -53,6 +53,10 @@ vi.mock('../runner-xctestrun.ts', async () => { import { prepareIosRunner, runAppleRunnerCommand } from '../index.ts'; import { resetRunnerRecycleLedgerForTests } from '../runner-recycle-ledger.ts'; +import { executeRunnerCommand, prepareLocalIosRunner } from '../runner-lifecycle.ts'; +import { executeRunnerExchange } from '../runner-exchange.ts'; +import { startFakeRunnerServer } from './fake-runner-server.ts'; +import type { RunnerCommand } from '../runner-contract.ts'; // What `executeRunnerCommand` decides when a readiness preflight refuses the runner // before the command was ever written: the table's own verdict axes say whether the @@ -223,3 +227,138 @@ test('a boot that exited early does not wipe a restored runner artifact', async assert.equal(mockMarkRunnerXctestrunArtifactBadForRun.mock.calls.length, 0); assert.equal(mockEnsureRunnerSession.mock.calls.length, 1); }); + +for (const status of [502, 503, 504]) { + for (const operation of ['mutation preflight', 'cold read', 'preparation'] as const) { + test(`forwarding ${status} exhaustion retains the run during ${operation}`, async () => { + const preflight = operation === 'mutation preflight'; + const server = await startFakeRunnerServer({ + uptime: [{ kind: 'httpErrorAlways', status }], + snapshot: [{ kind: 'httpErrorAlways', status }], + }); + try { + const session = makeRunnerSession({ + port: server.port, + state: preflight ? 'ready' : 'starting', + endpoint: { + port: server.port, + token: 'runner-token', + http: { url: `http://127.0.0.1:${server.port}/command`, headers: {} }, + }, + }); + const ensure = mockEnsureRunnerSession.mockResolvedValue(session); + const invalidate = mockInvalidateRunnerSession.mockResolvedValue(undefined); + mockReadRunnerSessionLiveness.mockReturnValue({ + sessionId: session.sessionId, + liveness: session.state, + }); + mockExecuteRunnerCommandWithSession.mockImplementation( + (device, target, command, logPath, timeoutMs, signal) => + executeRunnerExchange( + device, + target, + command, + logPath, + Math.min(timeoutMs, 100), + async (reason) => invalidate(target, reason), + signal, + ), + ); + const execute = executeRunnerCommand; + const command: RunnerCommand = preflight + ? { command: 'tap', x: 10, y: 20 } + : { command: 'snapshot' }; + const pending = + operation === 'preparation' + ? prepareLocalIosRunner(IOS_SIMULATOR, { healthTimeoutMs: 100 }) + : execute(IOS_SIMULATOR, command, {}); + await assert.rejects(pending, (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'runner_forwarding_unavailable'); + assert.equal(error.details?.status, status); + assert.equal(error.details?.dispatched, preflight ? 'no' : 'unknown'); + assert.equal(error.details?.runnerConnectFailureReason, undefined); + return true; + }); + assert.equal(ensure.mock.calls.length, 1); + assert.equal(invalidate.mock.calls.length, 0); + assert.ok(server.requests.length > 0); + const sentCommand = operation === 'cold read' ? 'snapshot' : 'uptime'; + assert.ok(server.requests.every((request) => request.command === sentCommand)); + } finally { + await server.close(); + } + }); + } +} + +test.each([false, true])( + 'a forwarding outage retains the replacement runner after a real connection failure, preflight=%s', + async (preflight) => { + const oldServer = await startFakeRunnerServer([{ kind: 'exit' }]); + const replacement = await startFakeRunnerServer({ + uptime: [{ kind: 'httpErrorAlways', status: 502 }], + snapshot: [{ kind: 'httpErrorAlways', status: 502 }], + }); + try { + const sessions = [oldServer, replacement].map((server) => + makeRunnerSession({ + port: server.port, + state: preflight ? 'ready' : 'starting', + endpoint: { + port: server.port, + token: 'runner-token', + http: { url: `http://127.0.0.1:${server.port}/command` }, + }, + }), + ); + const ensure = mockEnsureRunnerSession + .mockResolvedValueOnce(sessions[0]) + .mockResolvedValue(sessions[1]); + const invalidate = mockInvalidateRunnerSession.mockResolvedValue(undefined); + mockReadRunnerSessionLiveness.mockReturnValue({ + sessionId: sessions[0]!.sessionId, + liveness: sessions[0]!.state, + }); + mockExecuteRunnerCommandWithSession.mockImplementation( + (device, session, command, logPath, timeoutMs, signal) => + executeRunnerExchange( + device, + session, + command, + logPath, + Math.min(timeoutMs, 100), + async (reason) => invalidate(session, reason), + signal, + ), + ); + const execute = executeRunnerCommand; + await assert.rejects( + execute( + IOS_SIMULATOR, + preflight ? { command: 'tap', x: 10, y: 20 } : { command: 'snapshot' }, + {}, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'runner_forwarding_unavailable'); + assert.equal(error.details?.dispatched, preflight ? 'no' : 'unknown'); + assert.equal(error.details?.runnerRestarted, true); + return true; + }, + ); + assert.equal(ensure.mock.calls.length, 2); + assert.equal(invalidate.mock.calls.length, 1); + assert.equal(invalidate.mock.calls[0]![0], sessions[0]); + assert.ok(replacement.requests.length > 0); + assert.ok( + replacement.requests.every( + (request) => request.command === (preflight ? 'uptime' : 'snapshot'), + ), + ); + } finally { + await oldServer.close(); + await replacement.close(); + } + }, +); diff --git a/packages/apple-runner/src/__tests__/runner-session-close-finalization.test.ts b/packages/apple-runner/src/__tests__/runner-session-close-finalization.test.ts index 9e34963c38..a7aa63fcd6 100644 --- a/packages/apple-runner/src/__tests__/runner-session-close-finalization.test.ts +++ b/packages/apple-runner/src/__tests__/runner-session-close-finalization.test.ts @@ -432,7 +432,11 @@ test('a runner disposal failure propagates out of close (#2615)', async () => { throw new Error('runner artifact cleanup failed'); }); - await assert.rejects(() => closeRunner(device, false), /runner artifact cleanup failed/); + try { + await assert.rejects(() => closeRunner(device, false), /runner artifact cleanup failed/); + } finally { + mockCleanupTempFile.mockReset(); + } }); test('a drain that lands after close is issued cannot return a busy runner to reuse (#2615)', async () => { @@ -446,7 +450,7 @@ test('a drain that lands after close is issued cannot return a busy runner to re stdout: string; stderr: string; }) => void = () => {}; - session.testPromise = new Promise((resolve) => { + session.launch.local!.testPromise = new Promise((resolve) => { settleRunnerExit = resolve; }); @@ -575,3 +579,41 @@ async function ensureReadyRunner(device: DeviceInfo): Promise { await session.listenerWatch?.ready; return session; } + +for (const operation of ['abort', 'stop'] as const) { + test(`global ${operation} cancels a local start waiting for simulator boot`, async () => { + const device = { ...deviceNamed(`shutdown-boot-${operation}`), booted: false }; + let entered!: () => void; + const booting = new Promise((resolve) => { + entered = resolve; + }); + let releaseBoot!: (reason: unknown) => void; + let bootSignal: AbortSignal | undefined; + mockRunXcrun.mockImplementation(async (args, options) => { + if (!args.includes('bootstatus')) return { exitCode: 0, stdout: '', stderr: '' }; + bootSignal = options.signal; + entered(); + return await new Promise((_resolve, reject) => { + releaseBoot = reject; + options.signal?.addEventListener('abort', () => reject(options.signal.reason), { + once: true, + }); + }); + }); + const starting = ensureRunnerSession(device, {}); + const canceled = assert.rejects(starting); + await booting; + const shutdown = + operation === 'abort' ? abortAllIosRunnerSessions() : stopAllIosRunnerSessions(); + try { + assert.equal(bootSignal?.aborted, true); + await shutdown; + await canceled; + assert.equal(mockRunCmdBackground.mock.calls.length, 0); + assert.equal(readRunnerSessionLiveness(device.id), null); + } finally { + releaseBoot(new Error('test cleanup')); + await Promise.allSettled([starting, shutdown]); + } + }); +} diff --git a/packages/apple-runner/src/__tests__/runner-session-fixtures.ts b/packages/apple-runner/src/__tests__/runner-session-fixtures.ts index 724d001909..356d45c153 100644 --- a/packages/apple-runner/src/__tests__/runner-session-fixtures.ts +++ b/packages/apple-runner/src/__tests__/runner-session-fixtures.ts @@ -1,3 +1,4 @@ +import { observeLocalRunnerLaunch, type LocalRunnerLaunchResources } from '../runner-disposal.ts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { EventEmitter } from 'node:events'; @@ -59,22 +60,44 @@ export function makeRunnerArtifact( /** A ready runner session; `port` is shorthand for an endpoint carrying the fixture token. */ export function makeRunnerSession( - overrides: Partial & { port?: number } = {}, -): RunnerSession { - const { port = 8100, ...rest } = overrides; + overrides: Partial & Partial & { port?: number } = {}, +): RunnerSession & { launch: ReturnType } { + const { + port = 8100, + xctestrunPath = '/tmp/runner.xctestrun', + jsonPath = '/tmp/runner.json', + testPromise = Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }), + child = { pid: 1234, exitCode: null }, + lease, + readLogTail, + endOutputObservation, + ...rest + } = overrides; + const device = rest.device ?? IOS_SIMULATOR; + const deviceId = rest.deviceId ?? device.id; + const sessionId = rest.sessionId ?? `session-${port}`; return { - sessionId: `session-${port}`, - device: IOS_SIMULATOR, - deviceId: IOS_SIMULATOR.id, + sessionId, + device, + deviceId, endpoint: { port, token: RUNNER_TOKEN_FIXTURE }, - xctestrunPath: '/tmp/runner.xctestrun', - jsonPath: '/tmp/runner.json', - testPromise: Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }), - child: { pid: 1234, exitCode: null }, state: 'ready', commandCharges: new RunnerCommandAccounting(), ...rest, - } as RunnerSession; + launch: observeLocalRunnerLaunch({ + device, + deviceId, + sessionId, + xctestrunPath, + jsonPath, + testPromise, + child, + lease: lease ?? makeRunnerLease({ deviceId, sessionId }), + readLogTail, + endOutputObservation, + runnerLogPath: rest.runnerLogPath, + }), + }; } export function makeRunnerLease( diff --git a/packages/apple-runner/src/__tests__/runner-session-lifecycle.test.ts b/packages/apple-runner/src/__tests__/runner-session-lifecycle.test.ts index 4559eeffb0..fff0aa8ae2 100644 --- a/packages/apple-runner/src/__tests__/runner-session-lifecycle.test.ts +++ b/packages/apple-runner/src/__tests__/runner-session-lifecycle.test.ts @@ -245,7 +245,7 @@ test('every launch mints its own listener token for the runner, the session, and ); assert.match(first.endpoint.token, /^[A-Za-z0-9_-]{32}$/); assert.notEqual(first.endpoint.token, second.endpoint.token); - assert.equal(second.lease?.runnerToken, second.endpoint.token); + assert.equal(second.launch.local?.lease?.runnerToken, second.endpoint.token); }); test('stopping the same registered runner twice tears it down once', async () => { @@ -276,13 +276,12 @@ test('stop and invalidate do not start a second disposal already in progress', a await vi.waitFor(() => assert.equal(session.state, 'draining')); assert.equal(hasLiveIosRunnerSession(device), false); - await stopIosRunnerSession(device.id); - await invalidateRunnerSession(session, 'late command failure'); + const stop = stopIosRunnerSession(device.id); + const invalidation = invalidateRunnerSession(session, 'late command failure'); - assert.equal(vi.mocked(disposeRunnerSession).mock.calls.length, 1); assert.equal(mockWaitForRunner.mock.calls.length, 1); resolveShutdown(runnerResponse({ uptimeMs: 1 })); - await disposal; + await Promise.all([disposal, stop, invalidation]); assert.equal(session.state, 'stopped'); }); @@ -339,7 +338,7 @@ test('shutdown detach moves a handed-off session to stopped without killing its const device = { ...IOS_SIMULATOR, id: 'runner-lifecycle-detach' }; const session = await ensureRunnerSession(device, {}); await serveOneCommand(device, session); - const runnerPid = session.child.pid; + const runnerPid = session.launch.local!.child?.pid; assert.ok(runnerPid); runnerStateTransitions.length = 0; @@ -593,7 +592,7 @@ test("the handoff releases this daemon's log bookkeeping only after the lease sa await serveOneCommand(device, session); // What a release done too early would show: the lease this callback reads back is still owned. const tokenWhenReleased: string[] = []; - session.endOutputObservation = () => { + session.launch.local!.endOutputObservation = () => { tokenWhenReleased.push(leaseRaw(device.id)); }; @@ -606,7 +605,7 @@ test("the handoff releases this daemon's log bookkeeping only after the lease sa test('a registered runner whose process died is recycled instead of reused', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-lifecycle-dead-process' }; const first = await ensureRunnerSession(device, {}); - const firstPid = first.child.pid; + const firstPid = first.launch.local?.child?.pid; assert.ok(firstPid); runnerStateTransitions.length = 0; @@ -621,7 +620,7 @@ test('a registered runner whose process died is recycled instead of reused', asy const second = await ensureRunnerSession(device, {}); assert.notEqual(second.sessionId, first.sessionId); - assert.equal(second.child.pid, firstPid + 1); + assert.equal(second.launch.local?.child?.pid, firstPid + 1); assert.equal(first.state, 'stopped'); assert.deepEqual(runnerStateTransitions, ['draining', 'stopped']); assert.deepEqual(readRunnerSessionLiveness(device.id), { @@ -650,7 +649,7 @@ test('the same udid in another simulator set starts its own runner instead of re assert.match(leaseRaw(tenantB.id), /"simulatorSetPath": "\/tmp\/tenant-b\/simulators"/); }); -test('a draining session is never reused while its next command starts a fresh runner', async () => { +test('a replacement waits for the draining generation to finish cleanup', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-lifecycle-draining-reuse' }; const first = await ensureRunnerSession(device, {}); let resolveShutdown!: (response: Response) => void; @@ -666,16 +665,23 @@ test('a draining session is never reused while its next command starts a fresh r mockGetFreePort.mockResolvedValueOnce(8124); mockRunCmdBackground.mockReturnValueOnce(makeBackgroundRunner(4243)); - const second = await ensureRunnerSession(device, {}); + let replacementStarted = false; + const replacement = ensureRunnerSession(device, {}).then((session) => { + replacementStarted = true; + return session; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(replacementStarted, false); + resolveShutdown(runnerResponse({ uptimeMs: 1 })); + const second = await replacement; assert.notEqual(second.sessionId, first.sessionId); - assert.equal(first.state, 'draining'); + assert.equal(first.state, 'stopped'); assert.deepEqual(readRunnerSessionLiveness(device.id), { sessionId: second.sessionId, liveness: 'starting', }); - resolveShutdown(runnerResponse({ uptimeMs: 1 })); await disposal; assert.equal(first.state, 'stopped'); assert.equal(second.state, 'starting'); @@ -734,3 +740,44 @@ test('runner session restarts alive runner when expected xctestrun artifact chan assert.equal(restarted.xctestrunArtifact?.derived, '/tmp/derived-next'); assert.equal(mockRunCmdBackground.mock.calls.length, 2); }); + +test('shutdown retires a published macOS prepare without bypassing XCTest interrupt grace', async () => { + const { openRunnerStartAdmission, finishRunnerStartAdmission } = + await import('../runner-xctestrun.ts'); + const device = { ...MACOS_DEVICE, id: 'macos-shutdown-prepare' }; + const admission = openRunnerStartAdmission(device.id); + const process = makeBackgroundRunner(4242); + let finish!: (result: Awaited) => void; + process.wait = new Promise((resolve) => { + finish = resolve; + }); + mockRunCmdBackground.mockReturnValueOnce(process); + const session = await ensureRunnerSession(device, { startAdmission: admission }); + const executionSignal = mockRunCmdBackground.mock.calls[0]?.[2]?.signal as AbortSignal; + let shutdown: Promise | undefined; + vi.useFakeTimers(); + try { + shutdown = abortAllIosRunnerSessions(); + assert.equal(admission.signal.aborted, true); + assert.equal(executionSignal.aborted, false); + await vi.advanceTimersByTimeAsync(4_999); + assert.deepEqual( + mockSignalProcessGroupBestEffort.mock.calls.map(([, signal]) => signal), + ['SIGINT'], + ); + await vi.advanceTimersByTimeAsync(1); + assert.deepEqual( + mockSignalProcessGroupBestEffort.mock.calls.map(([, signal]) => signal), + ['SIGINT', 'SIGTERM'], + ); + mockIsProcessAlive.mockReturnValue(false); + finish({ exitCode: 0, stdout: '', stderr: '' }); + await shutdown; + assert.equal(session.state, 'stopped'); + } finally { + finish({ exitCode: 0, stdout: '', stderr: '' }); + finishRunnerStartAdmission(admission); + await shutdown; + vi.useRealTimers(); + } +}); diff --git a/packages/apple-runner/src/__tests__/runner-session.test.ts b/packages/apple-runner/src/__tests__/runner-session.test.ts index 9b6544696e..b3f17ac959 100644 --- a/packages/apple-runner/src/__tests__/runner-session.test.ts +++ b/packages/apple-runner/src/__tests__/runner-session.test.ts @@ -276,7 +276,7 @@ test('runner session starts xcodebuild through provider seams and reuses an aliv assert.equal(reused, session); assert.equal(session.endpoint.port, 8123); - assert.equal(session.xctestrunPath, '/tmp/session-runner.xctestrun'); + assert.equal(session.launch.local!.xctestrunPath, '/tmp/session-runner.xctestrun'); assert.equal(mockRunCmdBackground.mock.calls.length, 1); assert.equal(mockRunCmdBackground.mock.calls[0]?.[0], 'xcodebuild'); const xcodebuildArgs = mockRunCmdBackground.mock.calls[0]?.[1]; @@ -822,7 +822,7 @@ test('runner session startup reclaims a foreign runner lease whose owner state d // hold a live runner connection, so the old runner (pid 4321) must be // killed and a FRESH runner launched instead of silently adopting it. assert.equal(mockRunCmdBackground.mock.calls.length, 1); - assert.notEqual(session.child.pid, 4_321); + assert.notEqual(session.launch.local!.child?.pid, 4_321); const pkillCalls = mockRunAppleToolCommand.mock.calls.filter(isXcodebuildPkillCall); assert.ok(pkillCalls.length >= 2); assert.match(String(pkillCalls[0]?.[1]?.[2] ?? ''), /owner-state-dir-gone/); diff --git a/packages/apple-runner/src/__tests__/runner-startup-transport-early-exit.test.ts b/packages/apple-runner/src/__tests__/runner-startup-transport-early-exit.test.ts index d7acbb7652..dc874438a5 100644 --- a/packages/apple-runner/src/__tests__/runner-startup-transport-early-exit.test.ts +++ b/packages/apple-runner/src/__tests__/runner-startup-transport-early-exit.test.ts @@ -1,3 +1,4 @@ +import { makeRunnerSession as makeLocalRunnerSession } from './runner-session-fixtures.ts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; @@ -38,7 +39,7 @@ function sessionFailingWith( ): RunnerSession { const runnerLogPath = path.join(mkdtempForTestSync('runner-early-exit-'), 'runner.log'); fs.writeFileSync(runnerLogPath, log); - return { + return makeLocalRunnerSession({ sessionId: 'early-exit-session', device: { platform: 'apple', id: 'device-1', name: 'iPhone', kind: 'device', booted: true }, deviceId: 'device-1', @@ -53,7 +54,7 @@ function sessionFailingWith( state: 'starting', startupDeviceStates, commandCharges: new RunnerCommandAccounting(), - }; + }); } const IMAGE_DOWN_STATES = { @@ -179,8 +180,10 @@ const SET_WITHOUT_UDID = CAPTURED_SCOPED_SIMULATOR.setWithoutUdid; // An external xctestrun: the session carries no build of its own, so the Xcode it names is the one a // runner cache decision in this process already read, if any. function simulatorSessionFailingWith(log: string, simulatorSetPath?: string): RunnerSession { - return { - ...sessionFailingWith(log), + const failed = sessionFailingWith(log); + return makeLocalRunnerSession({ + ...failed, + ...failed.launch.local, device: { platform: 'apple', id: CAPTURED_SCOPED_SIMULATOR.udid, @@ -199,7 +202,7 @@ function simulatorSessionFailingWith(log: string, simulatorSetPath?: string): Ru buildMs: 0, xctestrunPathSource: 'external', }, - }; + }); } test('a scoped-set simulator whose destination is missing names its set and the Xcode', async () => { diff --git a/packages/apple-runner/src/__tests__/runner-startup-transport-service-hub.test.ts b/packages/apple-runner/src/__tests__/runner-startup-transport-service-hub.test.ts index 1807fc2e52..77f668b533 100644 --- a/packages/apple-runner/src/__tests__/runner-startup-transport-service-hub.test.ts +++ b/packages/apple-runner/src/__tests__/runner-startup-transport-service-hub.test.ts @@ -1,3 +1,4 @@ +import { makeRunnerSession as makeLocalRunnerSession } from './runner-session-fixtures.ts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; @@ -44,7 +45,7 @@ const BLESS_LOG = buildFixtureById('simulator-service-hub-bless-refused').output function sessionWithLaunchLog(log: string, exitCode: number | null): RunnerSession { const runnerLogPath = path.join(mkdtempForTestSync('runner-service-hub-'), 'runner.log'); fs.writeFileSync(runnerLogPath, log); - return { + return makeLocalRunnerSession({ sessionId: 'service-hub-session', device: iosSimulator, deviceId: iosSimulator.id, @@ -61,7 +62,7 @@ function sessionWithLaunchLog(log: string, exitCode: number | null): RunnerSessi child: { pid: 4242, exitCode } as ExecBackgroundResult['child'], state: 'starting', commandCharges: new RunnerCommandAccounting(), - }; + }); } /** The device states a simulator startup carries; the bless cause must outrank them (#2683). */ diff --git a/packages/apple-runner/src/__tests__/runner-startup-transport.test.ts b/packages/apple-runner/src/__tests__/runner-startup-transport.test.ts index cbaf54a4b6..97a7248bab 100644 --- a/packages/apple-runner/src/__tests__/runner-startup-transport.test.ts +++ b/packages/apple-runner/src/__tests__/runner-startup-transport.test.ts @@ -39,6 +39,8 @@ import { clearDeviceTunnelIpCache } from '../runner-command-route.ts'; import { readRunnerLogTail } from '../runner-io.ts'; import { resolveRunnerEarlyExitHint, waitForRunner } from '../runner-startup-transport.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; +import { makeRunnerSession } from './runner-session-fixtures.ts'; +import { buildFixtureById } from './runner-startup-failure-fixtures.ts'; beforeEach(() => { clearDeviceTunnelIpCache(); @@ -60,6 +62,40 @@ afterEach(() => { vi.unstubAllGlobals(); }); +test.each(['process exit', 'launch log'] as const)( + 'forwarding failure does not hide a confirmed runner %s', + async (failure) => { + const session = makeRunnerSession({ + device: iosSimulator, + state: 'starting', + readLogTail: () => + failure === 'launch log' + ? buildFixtureById('simulator-service-hub-bless-refused').output + : '', + endpoint: { + ...RUNNER_ENDPOINT_FIXTURE, + http: { url: 'https://instance.invalid/command' }, + }, + }); + vi.stubGlobal('fetch', async () => { + if (failure === 'process exit') session.launch.local!.child!.exitCode = 65; + return new Response('Bad Gateway', { status: 502 }); + }); + await assert.rejects( + waitForRunner(iosSimulator, session.endpoint, { command: 'uptime' }, undefined, 100, session), + (error: unknown) => { + assert.ok(error instanceof AppError); + if (failure === 'process exit') { + assert.equal(error.details?.runnerConnectFailureReason, 'xcodebuild_exited_early'); + } else { + assert.equal(error.details?.reason, 'simulator_service_hub_unavailable'); + } + return true; + }, + ); + }, +); + test('waitForRunner propagates request cancellation without fallback', async () => { const signal = AbortSignal.abort(); await assert.rejects( @@ -272,12 +308,12 @@ test('waitForRunner types a simulator runner that never accepts as a refused con }); test('waitForRunner discloses no when the runner exited before any attempt could write', async () => { - const session: RunnerSession = { + const session: RunnerSession = makeRunnerSession({ ...makeReadyRunnerSession(), device: iosDevice, deviceId: iosDevice.id, child: { pid: 1234, exitCode: 65 } as ExecBackgroundResult['child'], - }; + }); await assert.rejects( () => waitForRunner( @@ -420,7 +456,7 @@ test('waitForRunner preserves xcodebuild diagnostics when the runner exits durin runnerLogPath, 'The application could not be launched because the Developer App Certificate is not trusted.\n', ); - const session: RunnerSession = { + const session: RunnerSession = makeRunnerSession({ sessionId: 'starting-device-session', device: xctestIosDevice, deviceId: xctestIosDevice.id, @@ -434,9 +470,9 @@ test('waitForRunner preserves xcodebuild diagnostics when the runner exits durin child: { pid: 1234, exitCode: null } as ExecBackgroundResult['child'], state: 'starting', commandCharges: new RunnerCommandAccounting(), - }; + }); mockUsbmuxPostCommand.mockImplementation(async () => { - (session.child as { exitCode: number | null }).exitCode = 65; + (session.launch.local!.child as { exitCode: number | null }).exitCode = 65; throw new Error('ECONNREFUSED'); }); @@ -472,7 +508,7 @@ test('waitForRunner preserves xcodebuild diagnostics when the runner exits durin test('waitForRunner carries the disk-image state when the runner is still alive at the connect deadline (#2683)', async () => { // The alive-child twin of the early exit: `xcodebuild` never exits, the runner never answers, and // the failure a locked phone produces this way must still say what the phone reported. - const session: RunnerSession = { + const session: RunnerSession = makeRunnerSession({ sessionId: 'starting-device-session', device: xctestIosDevice, deviceId: xctestIosDevice.id, @@ -488,7 +524,7 @@ test('waitForRunner carries the disk-image state when the runner is still alive developerDiskImage: 'unavailable', developerDiskImageHint: 'Unlock the iPhone so it can mount the developer disk image.', }, - }; + }); mockUsbmuxPostCommand.mockRejectedValue(new Error('ECONNREFUSED')); vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('ECONNREFUSED'))); @@ -582,7 +618,7 @@ test('waitForRunner falls back to the tunnel route inside the same attempt', asy }); function makeReadyRunnerSession(): RunnerSession { - return { + return makeRunnerSession({ sessionId: 'ready-session', device: iosSimulator, deviceId: iosSimulator.id, @@ -593,7 +629,7 @@ function makeReadyRunnerSession(): RunnerSession { child: { pid: 1234, exitCode: null } as ExecBackgroundResult['child'], state: 'ready', commandCharges: new RunnerCommandAccounting(), - }; + }); } test('resolveRunnerEarlyExitHint surfaces busy-connecting guidance', () => { diff --git a/packages/apple-runner/src/runner-adoption.ts b/packages/apple-runner/src/runner-adoption.ts index a333660b34..b678c684b7 100644 --- a/packages/apple-runner/src/runner-adoption.ts +++ b/packages/apple-runner/src/runner-adoption.ts @@ -27,7 +27,7 @@ import { type RunnerLease, type RunnerLeaseAdoptionRefusal, } from './runner-lease.ts'; -import { isRunnerProcessAlive } from './runner-disposal.ts'; +import { isRunnerProcessAlive, observeLocalRunnerLaunch } from './runner-disposal.ts'; import { resolveExpectedRunnerCacheMetadata, resolveRunnerDerivedPath, @@ -115,7 +115,7 @@ export async function detachRunnerSessionsForShutdown( deviceId, lane: outcome.lane, sessionId: session.sessionId, - runnerPid: session.child.pid, + runnerPid: session.launch.local?.child.pid, port: session.endpoint.port, runnerLogPath: session.runnerLogPath, }, @@ -145,11 +145,11 @@ function detachRunnerSessionForShutdown(session: RunnerSession): RunnerDetachOut if (!decision.detach) { return { detached: false, lane, reason: decision.reason }; } - const lease = session.lease; + const lease = session.launch.local?.lease; if (!lease) { return { detached: false, lane, reason: 'lease_absent' }; } - if (!isRunnerProcessAlive(session.child.pid)) { + if (!isRunnerProcessAlive(session.launch.local?.child.pid)) { return { detached: false, lane, reason: 'runner_process_dead' }; } try { @@ -162,7 +162,7 @@ function detachRunnerSessionForShutdown(session: RunnerSession): RunnerDetachOut // the runner's log: until that write lands the session is still owned, and an owned session that // stopped following its runner's output is worse off than one that never handed anything off. // The runner holds its own descriptor, so this cannot disturb it either way (#2681). - session.endOutputObservation?.(); + session.launch.local?.endOutputObservation?.(); advanceRunnerSessionState(session, 'stopped'); return { detached: true, lane }; } @@ -247,7 +247,7 @@ export async function tryAdoptRunnerSessionFromLease( const session = buildAdoptedRunnerSession(device, lease, endpoint, runnerPid, expectedArtifact); try { - writeRunnerLease(session.lease); + writeRunnerLease(session.launch.local.lease); } catch { return skip('lease_write_failed', lease); } @@ -396,7 +396,7 @@ function buildAdoptedRunnerSession( endpoint: RunnerEndpoint, runnerPid: number, expectedArtifact: ExpectedRunnerArtifact, -): RunnerSession & { lease: RunnerLease } { +): RunnerSession & { launch: ReturnType } { const sessionId = lease.sessionId; const artifact: RunnerXctestrunArtifact = { xctestrunPath: lease.xctestrunPath, @@ -408,22 +408,16 @@ function buildAdoptedRunnerSession( reason: 'adopted_from_lease', }; const { child, wait } = watchDetachedRunnerProcess(runnerPid); - return { - sessionId, + const local = { device, deviceId: device.id, + sessionId, endpoint, xctestrunPath: lease.xctestrunPath, - xctestrunArtifact: artifact, jsonPath: lease.jsonPath, - testPromise: wait, child, - // The runner appends to this file for its whole life, so the log the previous daemon handed over - // is still the one worth quoting (#2681). + testPromise: wait, runnerLogPath: lease.runnerLogPath, - // The probe already proved the runner answers commands. - state: 'ready', - commandCharges: new RunnerCommandAccounting(), lease: buildRunnerLease({ device, sessionId, @@ -435,6 +429,17 @@ function buildAdoptedRunnerSession( runnerLogPath: lease.runnerLogPath, }), }; + return { + sessionId, + device, + deviceId: device.id, + endpoint, + xctestrunArtifact: artifact, + launch: observeLocalRunnerLaunch(local), + runnerLogPath: lease.runnerLogPath, + state: 'ready', + commandCharges: new RunnerCommandAccounting(), + }; } // The adopted xcodebuild was spawned by a dead process, so there is no diff --git a/packages/apple-runner/src/runner-artifact-env.ts b/packages/apple-runner/src/runner-artifact-env.ts index fada4f0fec..0795dfc629 100644 --- a/packages/apple-runner/src/runner-artifact-env.ts +++ b/packages/apple-runner/src/runner-artifact-env.ts @@ -59,12 +59,6 @@ function escapeForExtendedRegex(value: string): string { return value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); } -const RUNNER_XCTESTRUN_CAPTURE_OPTIONS = { - PreferredScreenCaptureFormat: 'screenshots', - SystemAttachmentLifetime: 'keepNever', - UserAttachmentLifetime: 'keepNever', -} as const; - type XctestrunTarget = { TestBundlePath?: unknown; EnvironmentVariables?: EnvMap; @@ -216,3 +210,9 @@ function toXctestrunTarget( if (options.requireTestBundlePath && !target.TestBundlePath) return null; return target; } + +const RUNNER_XCTESTRUN_CAPTURE_OPTIONS = { + PreferredScreenCaptureFormat: 'screenshots', + SystemAttachmentLifetime: 'keepNever', + UserAttachmentLifetime: 'keepNever', +} as const; diff --git a/packages/apple-runner/src/runner-artifact.ts b/packages/apple-runner/src/runner-artifact.ts index 05bec97de6..a32fd52ac6 100644 --- a/packages/apple-runner/src/runner-artifact.ts +++ b/packages/apple-runner/src/runner-artifact.ts @@ -98,6 +98,7 @@ export type RunnerStartRetirementReason = 'device_teardown' | 'last_waiter_cance */ export type RunnerStartAdmission = Readonly<{ deviceId: string; + readonly signal: AbortSignal; /** Whether preparation and publication are still admitted for a start holding this token. */ admitted: boolean; /** Why admission closed, or `undefined` while it is open. */ @@ -328,12 +329,16 @@ function registerStartAdmission(admission: StartAdmission): StartAdmission { function createStartAdmission(deviceId: string, mintedByStart: boolean): StartAdmission { const interestedWaiters = new Set(); + let cancellation = new AbortController(); let retired: RunnerStartRetirementReason | undefined; return { deviceId, interestedWaiters, mintedByStart, retryPending: false, + get signal() { + return cancellation.signal; + }, get admitted() { return retired === undefined; }, @@ -343,6 +348,7 @@ function createStartAdmission(deviceId: string, mintedByStart: boolean): StartAd close(reason) { if (retired) return false; retired = reason; + cancellation.abort(runnerStartRetiredError(reason)); emitDiagnostic({ level: 'debug', phase: 'ios_runner_start_admission_retired', @@ -352,6 +358,7 @@ function createStartAdmission(deviceId: string, mintedByStart: boolean): StartAd }, readmit() { retired = undefined; + cancellation = new AbortController(); emitDiagnostic({ level: 'debug', phase: 'ios_runner_start_admission_readmitted', diff --git a/packages/apple-runner/src/runner-command-recovery.ts b/packages/apple-runner/src/runner-command-recovery.ts index 0743272682..a5070fb59f 100644 --- a/packages/apple-runner/src/runner-command-recovery.ts +++ b/packages/apple-runner/src/runner-command-recovery.ts @@ -11,7 +11,11 @@ import { type RunnerResponsePayload, } from './runner-contract.ts'; import { isReadOnlyRunnerCommand } from './runner-command-traits.ts'; -import { RUNNER_REPLY_LOST_REASON } from './runner-error-classification.ts'; +import { + RUNNER_REPLY_LOST_REASON, + shouldRetainRunnerSession, + shouldRecoverRunnerStatus, +} from './runner-error-classification.ts'; import { RUNNER_IN_FLIGHT_LIFECYCLE_STATES, settleRunnerChargeForTerminalStatus, @@ -210,9 +214,13 @@ async function tryRecoverRunnerCommandAfterTransportError( options: AppleRunnerCommandOptions, signal?: AbortSignal, ): Promise { - if (command.command === 'status' || !command.commandId?.trim()) { + if ( + !shouldRecoverRunnerStatus(transportError) || + command.command === 'status' || + !command.commandId?.trim() + ) { return { - type: 'retainInvalidation', + type: shouldRetainRunnerSession(transportError) ? 'skipInvalidation' : 'retainInvalidation', reason: 'status_recovery_unavailable', lostReply: { recovery: 'status_recovery_unavailable', diff --git a/packages/apple-runner/src/runner-command-route.ts b/packages/apple-runner/src/runner-command-route.ts index 597a063a8d..8a0eb19ba5 100644 --- a/packages/apple-runner/src/runner-command-route.ts +++ b/packages/apple-runner/src/runner-command-route.ts @@ -1,3 +1,4 @@ +import type { RunnerEndpoint } from './runner-contract.ts'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; import { resolveIosPhysicalDeviceControl } from './host.ts'; @@ -38,7 +39,11 @@ const deviceTunnelIpCache = new Map(); * back to the tunnel route. The tunnel lookup and its cache therefore only run * for devices that genuinely need them, and cabled devices never pay for them. */ -export function createRunnerCommandRouteResolver(device: DeviceInfo, port: number) { +export function createRunnerCommandRouteResolver( + device: DeviceInfo, + port: number, + http?: RunnerEndpoint['http'], +) { let requestTunnelIp: string | null | undefined; let usbmuxUnattached = false; return { @@ -53,6 +58,7 @@ export function createRunnerCommandRouteResolver(device: DeviceInfo, port: numbe timeoutBudgetMs?: number, forceRefresh = false, ): Promise => { + if (http) return { kind: 'network', url: http.url, cachedTunnelIp: false }; if (runsOnThisHost(device)) return buildLoopbackRoute(port); const control = resolveIosPhysicalDeviceControl(device); if (control.backend === 'xctest') return buildUsbmuxRoute(device, port); diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 24c7bfa56c..bf5973d75e 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -488,17 +488,26 @@ const RUNNER_TOKEN_BYTES = 24; export const RUNNER_TOKEN_UNSUPPORTED_REASON = 'runner_token_unsupported'; /** How to reach one runner generation: its listener port and the token it requires. */ -export type RunnerEndpoint = Readonly<{ port: number; token: string }>; +export type RunnerEndpoint = Readonly<{ + port: number; + token: string; + /** A provider's authenticated forwarding route; omitted for local and USB runners. */ + http?: Readonly<{ url: string; headers?: Readonly> }>; +}>; export function mintRunnerToken(): string { return crypto.randomBytes(RUNNER_TOKEN_BYTES).toString('base64url'); } /** What the runner reads at launch to open its listener; see `RunnerTests+Environment.swift`. */ -export function runnerListenerEnv(endpoint: RunnerEndpoint): Record { +export function runnerListenerEnv( + endpoint: RunnerEndpoint, + bindAddress?: string, +): Record { return { AGENT_DEVICE_RUNNER_PORT: String(endpoint.port), AGENT_DEVICE_RUNNER_TOKEN: endpoint.token, + ...(bindAddress ? { AGENT_DEVICE_RUNNER_BIND_IP: bindAddress } : {}), }; } diff --git a/packages/apple-runner/src/runner-disposal.ts b/packages/apple-runner/src/runner-disposal.ts index 7b5231f149..a48a8fe3b3 100644 --- a/packages/apple-runner/src/runner-disposal.ts +++ b/packages/apple-runner/src/runner-disposal.ts @@ -9,7 +9,8 @@ import { runAppleToolCommand, runXcrun, } from './host.ts'; -import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; +import type { RunnerLaunchObservation } from './runner-launcher.ts'; +import type { ExecResult, ExecBackgroundResult } from '@agent-device/host-kit/command'; import { buildRunnerSessionXctestrunPathCleanupPattern } from './runner-artifact-env.ts'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; import { cleanupTempFile } from './runner-io.ts'; @@ -20,6 +21,7 @@ import { currentRunnerLeaseOwnerToken, releaseRunnerLease, withRunnerLeaseLock, + type RunnerLease, type RunnerLeaseCleanupAdapter, type RunnerXcodebuildCleanupTarget, } from './runner-lease.ts'; @@ -29,7 +31,11 @@ import { runnerPrepProcessChildren, runnerPrepProcessChildrenWithoutLiveOwner, } from './runner-xctestrun.ts'; -import { advanceRunnerSessionState, type RunnerSession } from './runner-session-types.ts'; +import { + advanceRunnerSessionState, + type RunnerProcessHandle, + type RunnerSession, +} from './runner-session-types.ts'; export const RUNNER_INVALIDATE_WAIT_TIMEOUT_MS = 1_000; @@ -50,6 +56,7 @@ export const runnerLeaseCleanupAdapter: RunnerLeaseCleanupAdapter = { export type RunnerDisposalOptions = { graceful?: boolean; + abort?: boolean; waitTimeoutMs?: number; /** * The caller already executes inside {@link withRunnerLeaseLock} for this @@ -59,33 +66,35 @@ export type RunnerDisposalOptions = { leaseLockHeld?: boolean; }; -export async function disposeRunnerSession( +export function disposeRunnerSession( session: RunnerSession, options: RunnerDisposalOptions = {}, +): Promise { + if (session.disposal) return session.disposal; + const disposal = disposeRunnerGeneration(session, options); + session.disposal = disposal; + void disposal.catch(() => { + session.disposal = undefined; + }); + return disposal; +} + +async function disposeRunnerGeneration( + session: RunnerSession, + options: RunnerDisposalOptions, ): Promise { // From here the session is going away: it still owns the lease and the process is still up, so // it can answer a request, and it must not be chosen for new work while it is being taken down. session.retention?.cancel(); session.listenerWatch?.close(); advanceRunnerSessionState(session, 'draining'); - let processExitHandled = false; - if (options.graceful !== false) { - processExitHandled = await shutdownRunnerSessionGracefully(session); - } else if (isMacOs(session.device)) { - await interruptMacOsRunnerSessions([session]); - await cleanupRunnerSessionResources(session, options); - return; - } else { - await killRunnerProcessTree(session.child.pid, 'SIGTERM'); - } - - if (!processExitHandled) { + const graceful = options.graceful !== false && (await shutdownRunnerSessionGracefully(session)); + if (graceful) { await waitForRunnerProcessExit(session, options.waitTimeoutMs ?? RUNNER_STOP_WAIT_TIMEOUT_MS); - if (isRunnerProcessTreeAlive(session.child.pid)) { - await killRunnerProcessTree(session.child.pid, 'SIGKILL'); - } } - await cleanupRunnerSessionResources(session, options); + await session.launch.terminate({ ...options, graceful }); + await session.launch.cleanup(options); + advanceRunnerSessionState(session, 'stopped'); } export async function cleanupOwnedIosRunnerLease(deviceId: string): Promise { @@ -95,26 +104,21 @@ export async function cleanupOwnedIosRunnerLease(deviceId: string): Promise { - const prepProcesses = runnerPrepProcessChildren(); - const macOsSessions = activeSessions.filter((session) => isMacOs(session.device)); - const otherSessions = activeSessions.filter((session) => !isMacOs(session.device)); - for (const session of activeSessions) { - session.retention?.cancel(); - session.listenerWatch?.close(); - advanceRunnerSessionState(session, 'draining'); - } - await signalRunnerSessions(otherSessions, 'SIGINT'); - await signalRunnerPrepProcesses(prepProcesses, 'SIGINT'); - await signalRunnerSessions(otherSessions, 'SIGTERM'); - await signalRunnerPrepProcesses(prepProcesses, 'SIGTERM'); - await signalRunnerSessions(otherSessions, 'SIGKILL'); - await signalRunnerPrepProcesses(prepProcesses, 'SIGKILL'); - await interruptMacOsRunnerSessions(macOsSessions); - await Promise.allSettled( - activeSessions.map(async (session) => { - await cleanupRunnerSessionResources(session); - }), - ); + const results = await Promise.allSettled([ + ...activeSessions.map((session) => + disposeRunnerSession(session, { graceful: false, abort: true }), + ), + (async () => { + const prep = runnerPrepProcessChildren(); + for (const signal of ['SIGINT', 'SIGTERM', 'SIGKILL'] as const) { + await signalRunnerPrepProcesses(prep, signal); + } + })(), + ]); + const failures = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason); + if (failures.length) throw new AggregateError(failures, 'Runner launchers failed to terminate'); } /** @@ -155,6 +159,11 @@ async function stopPrepProcessList( } async function shutdownRunnerSessionGracefully(session: RunnerSession): Promise { + const exited = new AbortController(); + void session.launch.finished.then( + () => exited.abort(), + () => exited.abort(), + ); try { await waitForRunner( session.device, @@ -164,41 +173,39 @@ async function shutdownRunnerSessionGracefully(session: RunnerSession): Promise< } as RunnerCommand), undefined, RUNNER_SHUTDOWN_TIMEOUT_MS, + undefined, + exited.signal, ); - return false; + return true; } catch { - if (isMacOs(session.device)) { - await interruptMacOsRunnerSessions([session]); - return true; - } else { - await killRunnerProcessTree(session.child.pid, 'SIGTERM'); - return false; - } + return false; } } +export function waitForRunnerLaunchExit( + finished: Promise, + timeoutMs: number, +): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), timeoutMs); + const exited = () => { + clearTimeout(timer); + resolve(true); + }; + void finished.then(exited, exited); + }); +} + async function waitForRunnerProcessExit( session: RunnerSession, waitTimeoutMs: number, ): Promise { - let timeout: NodeJS.Timeout | undefined; - try { - const exited = await Promise.race([ - session.testPromise.then( - () => true, - () => true, - ), - new Promise((resolve) => { - timeout = setTimeout(() => resolve(false), waitTimeoutMs); - }), - ]); - return exited || !isRunnerProcessTreeAlive(session.child.pid); - } finally { - if (timeout) clearTimeout(timeout); - } + return session.launch.waitForExit(waitTimeoutMs); } -async function interruptMacOsRunnerSessions(sessions: readonly RunnerSession[]): Promise { +async function interruptMacOsRunnerSessions( + sessions: readonly LocalRunnerLaunchResources[], +): Promise { if (sessions.length === 0) return; // CONSERVATIVE: XCTest disables the host screen saver while macOS UI automation runs and @@ -222,24 +229,22 @@ async function interruptMacOsRunnerSessions(sessions: readonly RunnerSession[]): * exit — the escalation step signals exactly these. */ async function runnerSessionsWithRunningProcesses( - sessions: readonly RunnerSession[], + sessions: readonly LocalRunnerLaunchResources[], waitTimeoutMs: number, -): Promise { +): Promise { const exited = await Promise.all( - sessions.map(async (session) => await waitForRunnerProcessExit(session, waitTimeoutMs)), + sessions.map(async (session) => await waitForLocalRunnerProcessExit(session, waitTimeoutMs)), ); return sessions.filter((_, index) => !exited[index]); } -async function cleanupRunnerSessionResources( - session: RunnerSession, +async function cleanupLocalRunnerResources( + session: LocalRunnerLaunchResources, options: Pick = {}, ): Promise { await settleOwnedRunnerDeviceState(session, options); cleanupTempFile(session.xctestrunPath); cleanupTempFile(session.jsonPath); - // The session's own resources are gone: no lease, no temp files. - advanceRunnerSessionState(session, 'stopped'); } /** @@ -253,7 +258,7 @@ async function cleanupRunnerSessionResources( * and an unreleased own lease turns stale once this process exits. */ async function settleOwnedRunnerDeviceState( - session: RunnerSession, + session: LocalRunnerLaunchResources, options: Pick, ): Promise { const settle = async () => { @@ -283,9 +288,9 @@ async function settleOwnedRunnerDeviceState( } } -function runnerLeaseOwnedElsewhere(session: RunnerSession): boolean { +function runnerLeaseOwnedElsewhere(session: LocalRunnerLaunchResources): boolean { const onDiskToken = currentRunnerLeaseOwnerToken(session.deviceId); - return onDiskToken !== null && onDiskToken !== session.lease?.ownerToken; + return onDiskToken !== null && onDiskToken !== session.lease.ownerToken; } async function terminateRunnerSimulatorApps(device: DeviceInfo): Promise { @@ -325,7 +330,7 @@ export function isRunnerProcessAlive(pid: number | undefined): boolean { } async function signalRunnerSessions( - activeSessions: readonly RunnerSession[], + activeSessions: readonly LocalRunnerLaunchResources[], signal: 'SIGINT' | 'SIGTERM' | 'SIGKILL', ): Promise { await Promise.allSettled( @@ -388,3 +393,70 @@ async function killRunnerXcodebuildProcesses(target: RunnerXcodebuildCleanupTarg } } } + +export type LocalRunnerLaunchResources = { + device: DeviceInfo; + deviceId: string; + sessionId: string; + child: RunnerProcessHandle; + testPromise: Promise; + output?: Promise; + xctestrunPath: string; + jsonPath: string; + lease: RunnerLease; + endOutputObservation?: () => void; + readLogTail?: (maxBytes: number) => string; + runnerLogPath?: string; +}; + +export function observeLocalRunnerLaunch( + local: LocalRunnerLaunchResources, +): RunnerLaunchObservation & { local: LocalRunnerLaunchResources } { + const finished = local.testPromise.then( + () => {}, + () => {}, + ); + return { + local, + get exited() { + return local.child.exitCode !== null; + }, + finished, + output: local.output ?? finished, + isRunning: () => isRunnerProcessAlive(local.child.pid), + waitForExit: (timeoutMs) => waitForLocalRunnerProcessExit(local, timeoutMs), + terminate: async (options = {}) => { + if (options.graceful) { + if (isRunnerProcessTreeAlive(local.child.pid)) + await killRunnerProcessTree(local.child.pid, 'SIGKILL'); + } else if (isMacOs(local.device)) { + await interruptMacOsRunnerSessions([local]); + } else if (options.abort) { + for (const signal of ['SIGINT', 'SIGTERM', 'SIGKILL'] as const) + await killRunnerProcessTree(local.child.pid, signal); + } else { + await killRunnerProcessTree(local.child.pid, 'SIGTERM'); + await waitForLocalRunnerProcessExit( + local, + options.waitTimeoutMs ?? RUNNER_STOP_WAIT_TIMEOUT_MS, + ); + if (isRunnerProcessTreeAlive(local.child.pid)) + await killRunnerProcessTree(local.child.pid, 'SIGKILL'); + } + }, + cleanup: (options) => cleanupLocalRunnerResources(local, options), + readLogTail: (maxBytes) => local.readLogTail?.(maxBytes) ?? '', + earlyExitError: async (port, logPath, startupDeviceStates) => { + const { buildLocalRunnerEarlyExitError } = await import('./runner-startup-transport.ts'); + return buildLocalRunnerEarlyExitError({ ...local, startupDeviceStates }, port, logPath); + }, + }; +} + +async function waitForLocalRunnerProcessExit( + local: LocalRunnerLaunchResources, + waitTimeoutMs: number, +): Promise { + const exited = await waitForRunnerLaunchExit(local.testPromise, waitTimeoutMs); + return exited || !isRunnerProcessTreeAlive(local.child.pid); +} diff --git a/packages/apple-runner/src/runner-error-classification.ts b/packages/apple-runner/src/runner-error-classification.ts index a063b63b05..73fc87edbf 100644 --- a/packages/apple-runner/src/runner-error-classification.ts +++ b/packages/apple-runner/src/runner-error-classification.ts @@ -128,11 +128,7 @@ const hasReadinessPreflightFailure: RunnerErrorDetailsMatch = (details) => details.runnerReadinessPreflightFailed === true && !isRequestCanceledDetails(details); type RunnerErrorVerdicts = { - /** - * isRetryableRunnerError: worth a same-session resend, either a transport failure or a structured - * refusal that executed nothing. Whether a lost response needs status recovery is a separate - * question ({@link isStructuredRunnerFailure}). - */ + /** A transport failure or structured refusal permits a same-session resend. */ retryable?: boolean; /** isRunnerBusyError: the runner refused fast while abandoned main-thread work drains. */ drainResend?: boolean; @@ -146,6 +142,10 @@ type RunnerErrorVerdicts = { restartAfterReadinessPreflight?: boolean; /** The runner never accepted a connection, so the restored artifact itself is suspect. */ artifactSuspect?: boolean; + /** A transport outage leaves the current generation owned for the next command. */ + retainSession?: boolean; + /** Status recovery can reach the runner after a lost response. */ + postSendRecovery?: boolean; }; /** @@ -358,6 +358,22 @@ export const RUNNER_ERROR_RULES: readonly RunnerErrorRule[] = [ match: { code: 'COMMAND_FAILED', foreignMessageIncludesAll: ['device is busy', 'connecting'] }, verdicts: { retryable: false, connectRetry: false }, }, + { + reason: 'runner_forwarding_unavailable', + match: { + code: 'COMMAND_FAILED', + details: (details) => details.reason === 'runner_forwarding_unavailable', + }, + verdicts: { + retryable: true, + retainSession: true, + postSendRecovery: false, + connectRetry: true, + restartBeforeSend: false, + restartAfterReadinessPreflight: false, + artifactSuspect: false, + }, + }, { // The marker is the whole fact: the preflight gave up before the command was written, so // replaying it cannot duplicate anything. It is asked for alone because the preflight fails in @@ -657,6 +673,14 @@ export function isRetryableRunnerError(err: unknown): boolean { return runnerErrorVerdict(err, 'retryable') ?? false; } +export function shouldRetainRunnerSession(error: unknown): boolean { + return runnerErrorVerdict(error, 'retainSession') ?? false; +} + +export function shouldRecoverRunnerStatus(error: unknown): boolean { + return runnerErrorVerdict(error, 'postSendRecovery') ?? true; +} + /** * What one runner failure proves about the runner's main thread: it is still draining the * watchdog-abandoned XCTest work, it has drained, or the failure says nothing either way. The third diff --git a/packages/apple-runner/src/runner-launched-session.ts b/packages/apple-runner/src/runner-launched-session.ts new file mode 100644 index 0000000000..439bd77af2 --- /dev/null +++ b/packages/apple-runner/src/runner-launched-session.ts @@ -0,0 +1,149 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { randomUUID } from 'node:crypto'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { Deadline, emitDiagnostic } from './host.ts'; +import { disposeRunnerSession, waitForRunnerLaunchExit } from './runner-disposal.ts'; +import { mintRunnerToken } from './runner-contract.ts'; +import { logChunk, resolveRunnerLaunchLogPath, flushRunnerLogAppends } from './runner-io.ts'; +import type { RunnerLauncher, RunnerLaunchObservation } from './runner-launcher.ts'; +import type { AppleRunnerLifecycleOptions } from './runner-provider.ts'; +import { + advanceRunnerSessionState, + RunnerCommandAccounting, + type RunnerSession, +} from './runner-session-types.ts'; +import { resolveRunnerLaunchReadinessMs, type RunnerStartBudget } from './runner-start-budget.ts'; + +export async function launchRunnerSession( + device: DeviceInfo, + launcher: RunnerLauncher, + options: AppleRunnerLifecycleOptions, + budget: RunnerStartBudget, + admissionSignal: AbortSignal, +): Promise { + const endpoint = { ...launcher.endpoint, token: mintRunnerToken() }; + const abort = new AbortController(); + const signal = budget.phase.signal + ? AbortSignal.any([budget.phase.signal, admissionSignal]) + : admissionSignal; + const cancel = () => abort.abort(signal.reason); + signal?.addEventListener('abort', cancel, { once: true }); + let launch; + try { + signal?.throwIfAborted(); + launch = await launcher.launch(endpoint, abort.signal); + } finally { + signal?.removeEventListener('abort', cancel); + } + const watch = { + lost: false, + ready: Promise.resolve(true), + close() { + this.lost = true; + }, + }; + let termination: Promise | undefined; + const observation: RunnerLaunchObservation = { + finished: launch.exited, + exited: false, + output: Promise.resolve(), + isRunning: () => !observation.exited, + waitForExit: (timeoutMs) => waitForRunnerLaunchExit(launch.exited, timeoutMs), + cleanup: async () => { + await observation.output; + await flushRunnerLogAppends(logPath).catch((error: unknown) => { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_log_flush_failed', + data: { error: String(error), logPath }, + }); + }); + }, + readLogTail: (maxBytes) => output.slice(-maxBytes), + earlyExitError: async (port, requestedLogPath) => + new AppError( + 'COMMAND_FAILED', + 'Runner did not accept connection (launcher exited early)', + { + reason: 'runner_process_exited', + port, + logPath: logPath ?? requestedLogPath, + output: observation.readLogTail(16_384), + }, + observation.failure, + ), + terminate: () => + (termination ??= (async () => { + if (!observation.exited) await launch.terminate(); + await launch.exited.catch(() => {}); + })().catch((error: unknown) => { + termination = undefined; + throw new AppError( + 'COMMAND_FAILED', + 'Runner launcher could not terminate its generation', + { + reason: 'runner_termination_failed', + deviceId: device.id, + }, + error, + ); + })), + }; + let output = ''; + const logPath = resolveRunnerLaunchLogPath(options.logPath, device.id); + const session: RunnerSession = { + sessionId: randomUUID(), + device, + deviceId: device.id, + endpoint, + state: 'starting', + commandCharges: new RunnerCommandAccounting(), + speculative: options.speculative, + launch: observation, + listenerWatch: watch, + runnerLogPath: logPath, + launchDeadline: Deadline.fromTimeoutMs(resolveRunnerLaunchReadinessMs(budget)), + }; + const settled = (failure?: unknown) => { + observation.exited = true; + observation.failure = failure; + watch.close(); + session.retention?.cancel(); + }; + void launch.exited.then(() => settled(), settled); + observation.output = (async () => { + for await (const text of launch.output) { + output = (output + text).slice(-16_384); + logChunk(text, logPath, options.traceLogPath, options.verbose); + } + })().catch(async (error: unknown) => { + observation.failure = error; + advanceRunnerSessionState(session, 'draining'); + watch.close(); + session.retention?.cancel(); + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_output_failed', + data: { error: String(error), deviceId: device.id }, + }); + await observation.terminate().catch((terminationError: unknown) => { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_termination_failed', + data: { deviceId: device.id, error: String(terminationError) }, + }); + }); + }); + return session; +} + +export async function refuseFailedRunnerPublication(session: RunnerSession): Promise { + if (session.state !== 'draining') return; + await disposeRunnerSession(session, { graceful: false }); + throw new AppError( + 'COMMAND_FAILED', + 'Runner output failed during startup', + { reason: 'runner_output_failed', deviceId: session.deviceId }, + session.launch.failure, + ); +} diff --git a/packages/apple-runner/src/runner-launcher-provider.ts b/packages/apple-runner/src/runner-launcher-provider.ts new file mode 100644 index 0000000000..746eef831e --- /dev/null +++ b/packages/apple-runner/src/runner-launcher-provider.ts @@ -0,0 +1,37 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { RunnerLauncher } from './runner-launcher.ts'; +import type { + AppleRunnerCommandOptions, + AppleRunnerLifecycleOptions, + AppleRunnerPrepareOptions, +} from './runner-provider.ts'; +import type { RunnerCommand } from './runner-contract.ts'; +import { executeRunnerCommand, prepareLocalIosRunner } from './runner-lifecycle.ts'; +import { + ensureRunnerSession, + readRunnerSessionLiveness, + releaseRunnerOnClose, + releaseSpeculativeIosRunnerSession, + stopRunnerSession, +} from './runner-session.ts'; + +/** Uses the same session registry, admissions and close policy as the local launcher. */ +export function createLaunchedAppleRunnerProvider( + device: DeviceInfo, + launcher: RunnerLauncher, + env: NodeJS.ProcessEnv, +) { + return { + runCommand: (target: DeviceInfo, command: RunnerCommand, options: AppleRunnerCommandOptions) => + executeRunnerCommand(target, command, { ...options, launcher }), + prepare: (target: DeviceInfo, options: AppleRunnerPrepareOptions) => + prepareLocalIosRunner(target, { ...options, launcher }), + hasLiveSession: () => readRunnerSessionLiveness(device.id)?.liveness === 'ready', + ensure: (options: AppleRunnerLifecycleOptions) => + ensureRunnerSession(device, { ...options, launcher }), + stop: () => stopRunnerSession(device.id), + release: (retain: boolean) => releaseRunnerOnClose(device.id, { retain }, env), + isStarted: () => readRunnerSessionLiveness(device.id) !== null, + releaseSpeculative: () => releaseSpeculativeIosRunnerSession(device.id), + }; +} diff --git a/packages/apple-runner/src/runner-launcher.ts b/packages/apple-runner/src/runner-launcher.ts new file mode 100644 index 0000000000..383266af29 --- /dev/null +++ b/packages/apple-runner/src/runner-launcher.ts @@ -0,0 +1,35 @@ +import type { AppError } from '@agent-device/kernel/errors'; +import type { IosRunnerDeviceStates } from './runner-error-classification.ts'; +import type { LocalRunnerLaunchResources, RunnerDisposalOptions } from './runner-disposal.ts'; +import type { RunnerEndpoint } from './runner-contract.ts'; + +/** A launched runner owns its output and settles both streams when terminated. */ +export type RunnerLaunch = { + exited: Promise; + output: AsyncIterable; + terminate(): Promise; +}; + +/** Launch mechanisms supply resources; the session owner controls their lifetime. */ +export type RunnerLauncher = { + endpoint: Omit; + launch(endpoint: RunnerEndpoint, signal: AbortSignal): Promise; +}; + +export type RunnerLaunchObservation = { + exited: boolean; + failure?: unknown; + finished: Promise; + output: Promise; + isRunning(): boolean; + waitForExit(timeoutMs: number): Promise; + terminate(options?: RunnerDisposalOptions): Promise; + cleanup(options?: RunnerDisposalOptions): Promise; + readLogTail(maxBytes: number): string; + earlyExitError( + port: number, + logPath?: string, + deviceStates?: IosRunnerDeviceStates, + ): Promise; + local?: LocalRunnerLaunchResources; +}; diff --git a/packages/apple-runner/src/runner-lifecycle.ts b/packages/apple-runner/src/runner-lifecycle.ts index 1ff50b905e..90388ae1df 100644 --- a/packages/apple-runner/src/runner-lifecycle.ts +++ b/packages/apple-runner/src/runner-lifecycle.ts @@ -30,6 +30,7 @@ import { } from './runner-contract.ts'; import { isRetryableRunnerError, + shouldRetainRunnerSession, isRunnerPreSendRefusal, isStructuredRunnerFailure, resolveFirstAttemptDispatch, @@ -115,11 +116,13 @@ async function runPrepareAttempt(params: { }): Promise { const { device, command, options, signal, attempt, recoveryReason } = params; const connectStartedAt = Date.now(); + assertRunnerStartAdmitsPreparation(device.id, options.startAdmission); const session = await ensureRunnerSession(device, { ...options, cleanStaleBundles: attempt > 1 ? true : options.cleanStaleBundles, ...readPrepareDeadlinePhaseTimeouts(options, 'runner_session'), }); + assertExpectedRunnerSession(session, options.expectedRunnerSessionId); const connectMs = Date.now() - connectStartedAt; try { const result = await runPrepareHealthCheck( @@ -158,11 +161,13 @@ async function handlePrepareHealthFailure(params: { }): Promise { const { device, session, command, options, signal, attempt, error } = params; assertRunnerStartAdmitsPreparation(device.id, options.startAdmission); + if (options.expectedRunnerSessionId !== undefined) throw error; const appErr = asAppError(error, 'COMMAND_FAILED'); if (isRequestCanceledError(appErr)) { await invalidateRunnerSessionBestEffort(session, 'prepare_runner_request_canceled'); throw error; } + if (shouldRetainRunnerSession(appErr)) throw error; if (attempt === 1 && shouldRecoverBadCachedRunnerArtifact(appErr, session)) { return { kind: 'prepared', @@ -364,6 +369,8 @@ async function executeRunnerCommandAttempt( appErr, ); } + const forwardingFailure = runnerPreSendFailureToRetain(appErr); + if (forwardingFailure) throw forwardingFailure; if (shouldRestartRunnerBeforeCommandSend(appErr, command) && session) { assertRunnerRequestActive(options.requestId); return await restartSessionAndRunCommand({ @@ -467,6 +474,9 @@ async function restartSessionAndRunCommand(params: { return recovered; } catch (error) { const retryAppErr = asAppError(error, 'COMMAND_FAILED'); + const forwardingFailure = runnerPreSendFailureToRetain(retryAppErr); + if (forwardingFailure) + throw markRunnerRestartError(forwardingFailure, params, restartedSession); if (isRetryableRunnerError(retryAppErr) && !isStructuredRunnerFailure(retryAppErr)) { try { return await handleRunnerTransportErrorAfterCommandSend({ @@ -696,3 +706,8 @@ function emitPrepareDiagnostic( }, }); } + +function runnerPreSendFailureToRetain(error: AppError): AppError | undefined { + if (!shouldRetainRunnerSession(error)) return; + if (error.details?.runnerReadinessPreflightFailed === true) return discloseDispatch(error, 'no'); +} diff --git a/packages/apple-runner/src/runner-process-launch.ts b/packages/apple-runner/src/runner-process-launch.ts index 1091769b4c..6413cb74fd 100644 --- a/packages/apple-runner/src/runner-process-launch.ts +++ b/packages/apple-runner/src/runner-process-launch.ts @@ -1,3 +1,5 @@ +import { buildRunnerLease } from './runner-lease.ts'; +import { observeLocalRunnerLaunch } from './runner-disposal.ts'; import fs from 'node:fs'; import path from 'node:path'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -30,6 +32,7 @@ type LaunchRunnerProcessInput = Readonly<{ export type LaunchedRunnerProcess = ExecBackgroundResult & Readonly<{ startupRetryWake: AbortSignal; + output: Promise; /** Gives up this process's sides of the runner's log; see {@link launchRunnerProcess}. */ endOutputObservation: () => void; /** Reads the end of the runner's log, which the runner itself keeps writing to (#2681). */ @@ -112,9 +115,10 @@ export function launchRunnerProcess(input: LaunchRunnerProcessInput): LaunchedRu closeOutputFd(); listenerReady.finish(); }; - void launched.wait.then(onProcessSettled, onProcessSettled); + const output = launched.wait.then(onProcessSettled, onProcessSettled); return { ...launched, + output, startupRetryWake: listenerReady.wake, endOutputObservation, readLogTail: (maxBytes) => readRunnerLogTail(logFile, maxBytes), @@ -133,3 +137,41 @@ function once(task: () => void): () => void { task(); }; } + +export function launchLocalRunner( + input: LaunchRunnerProcessInput & { + sessionId: string; + endpoint: import('./runner-contract.ts').RunnerEndpoint; + jsonPath: string; + cacheKey?: string; + }, +) { + const process = launchRunnerProcess(input); + const lease = buildRunnerLease({ + device: input.device, + sessionId: input.sessionId, + runnerPid: process.child.pid, + endpoint: input.endpoint, + xctestrunPath: input.xctestrunPath, + cacheKey: input.cacheKey, + jsonPath: input.jsonPath, + runnerLogPath: input.logPath, + }); + return { + launch: observeLocalRunnerLaunch({ + device: input.device, + deviceId: input.device.id, + sessionId: input.sessionId, + child: process.child, + testPromise: process.wait, + output: process.output, + xctestrunPath: input.xctestrunPath, + jsonPath: input.jsonPath, + lease, + endOutputObservation: process.endOutputObservation, + readLogTail: process.readLogTail, + runnerLogPath: input.logPath, + }), + startupRetryWake: process.startupRetryWake, + }; +} diff --git a/packages/apple-runner/src/runner-provider.ts b/packages/apple-runner/src/runner-provider.ts index 6a4d9263c9..2b482607c3 100644 --- a/packages/apple-runner/src/runner-provider.ts +++ b/packages/apple-runner/src/runner-provider.ts @@ -1,3 +1,4 @@ +import type { RunnerLauncher } from './runner-launcher.ts'; import { AsyncLocalStorage } from 'node:async_hooks'; import type { AppleRunnerRequestOptions } from '@agent-device/contracts/apple-runner-request'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -6,6 +7,7 @@ import type { RunnerCommand } from './runner-contract.ts'; import type { RunnerXctestrunArtifactState, RunnerXctestrunCacheKind } from './runner-xctestrun.ts'; export type AppleRunnerCommandOptions = AppleRunnerRequestOptions & { + launcher?: RunnerLauncher; signal?: AbortSignal; cleanStaleBundles?: boolean; startupTimeoutMs?: number; diff --git a/packages/apple-runner/src/runner-retention.ts b/packages/apple-runner/src/runner-retention.ts index 3e7224926d..36b10b34f8 100644 --- a/packages/apple-runner/src/runner-retention.ts +++ b/packages/apple-runner/src/runner-retention.ts @@ -11,6 +11,18 @@ import { const LISTENER_CONNECT_TIMEOUT_MS = 500; const RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; +type RetainableRunnerSession = Pick< + RunnerSession, + | 'device' + | 'deviceId' + | 'sessionId' + | 'state' + | 'runnerMainThreadBusy' + | 'commandCharges' + | 'listenerWatch' + | 'retention' +>; + type StopRetainedRunner = ( retention: RunnerRetention, reason: 'idle_timeout' | 'listener_lost', @@ -64,8 +76,9 @@ export function observeRunnerListener( /** Retains a ready generation already observed by its listener connection, with one idle timer. */ export async function retainRunnerSession( - session: RunnerSession, + session: RetainableRunnerSession, stop: StopRetainedRunner, + env: NodeJS.ProcessEnv = hostEnvironment().env, ): Promise { if ( session.state !== 'ready' || @@ -92,7 +105,7 @@ export async function retainRunnerSession( }, }; session.retention = retention; - const idleMs = resolveRunnerIdleStopMs(); + const idleMs = resolveRunnerIdleStopMs(env); if (idleMs > 0) { timer = setTimeout(() => { void stop(retention, 'idle_timeout').catch((error: unknown) => { @@ -113,7 +126,7 @@ export async function retainRunnerSession( return true; } -function needsListenerWatch(session: RunnerSession): boolean { +function needsListenerWatch(session: Pick): boolean { return session.device.kind === 'simulator' && isIosFamily(session.device); } diff --git a/packages/apple-runner/src/runner-session-types.ts b/packages/apple-runner/src/runner-session-types.ts index 7b42300270..b2b3fe5dc7 100644 --- a/packages/apple-runner/src/runner-session-types.ts +++ b/packages/apple-runner/src/runner-session-types.ts @@ -1,9 +1,8 @@ +import type { RunnerLaunchObservation } from './runner-launcher.ts'; import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/runner-lease-context'; -import type { ExecResult } from '@agent-device/host-kit/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Deadline } from './host.ts'; import type { RunnerXctestrunArtifact } from './runner-xctestrun.ts'; -import type { RunnerLease } from './runner-lease.ts'; import type { IosRunnerDeviceStates } from './runner-error-classification.ts'; import type { RunnerCommand, RunnerEndpoint } from './runner-contract.ts'; @@ -64,28 +63,15 @@ export type RunnerSession = { device: DeviceInfo; deviceId: string; endpoint: RunnerEndpoint; - xctestrunPath: string; xctestrunArtifact?: RunnerXctestrunArtifact; - jsonPath: string; /** * Where this runner's own output goes: the file handed to the child as its stdout/stderr, which * the runner keeps appending to across a daemon handoff (#2681). A session adopted from an older * lease has none, because that runner wrote into pipes its own daemon held. */ runnerLogPath?: string; - testPromise: Promise; - child: RunnerProcessHandle; - /** - * Gives up this daemon's sides of the runner's log: the tail it follows and its copy of the log's - * write end. Only a session this process launched has either. The runner keeps its own descriptor, - * so handing off is a bookkeeping step and cannot disturb a running runner (#2681). - */ - endOutputObservation?: () => void; - /** - * Reads the end of {@link runnerLogPath}. The module that opened the file answers for it, so the - * code quoting a runner's failure does not have to know how the log is stored (#2681). - */ - readLogTail?: (maxBytes: number) => string; + launch: RunnerLaunchObservation; + disposal?: Promise; /** Moves only through {@link advanceRunnerSessionState}. */ state: RunnerSessionState; listenerWatch?: RunnerListenerWatch; @@ -133,7 +119,6 @@ export type RunnerSession = { */ startupDeviceStates?: IosRunnerDeviceStates; logicalLeaseContext?: RunnerLogicalLeaseContext; - lease?: RunnerLease; }; // A session goes forward through the lifecycle once: a state already reached is never re-entered, diff --git a/packages/apple-runner/src/runner-session.ts b/packages/apple-runner/src/runner-session.ts index 85788d33a7..ad3378a2bb 100644 --- a/packages/apple-runner/src/runner-session.ts +++ b/packages/apple-runner/src/runner-session.ts @@ -24,7 +24,6 @@ import { openRunnerStartAdmission, prepareXctestrunWithEnv, readmitRunnerStartAdmission, - requireRunnerPhaseRemainingMs, resolveExpectedRunnerCacheMetadata, resolveRunnerDerivedPath, retireAllRunnerStartAdmissions, @@ -43,7 +42,6 @@ import { import { enrichRunnerStartupFailureWithDeviceStates } from './runner-error-classification.ts'; import { isRunnerReadinessProbeCommand } from './runner-command-traits.ts'; import { - buildRunnerLease, prepareRunnerLeaseForStartup, runnerOwnerToken, withRunnerLeaseLock, @@ -58,7 +56,6 @@ import { abortRunnerSessionsAndPrepProcesses, cleanupOwnedIosRunnerLease, disposeRunnerSession, - isRunnerProcessAlive, runnerLeaseCleanupAdapter, RUNNER_INVALIDATE_WAIT_TIMEOUT_MS, stopRunnerPrepProcesses, @@ -68,14 +65,13 @@ import { buildRunnerSessionId, canWorkWithRunnerSession, isRunnerMainThreadOccupied, - normalizeRunnerStartupTimeoutMs, resolveRunnerSessionLiveness, RunnerCommandAccounting, type RunnerSession, type RunnerSessionLiveness, type RunnerSessionRegistration, } from './runner-session-types.ts'; -import { launchRunnerProcess, type LaunchedRunnerProcess } from './runner-process-launch.ts'; +import { launchLocalRunner } from './runner-process-launch.ts'; import { isSameRunnerSimulator } from './runner-device-set.ts'; // The start-budget members are read through function-scoped imports: this module sits in the // façade closures the eager-closure budget holds at its merge-base size, and the budget module is @@ -107,6 +103,7 @@ export async function ensureRunnerSession( // other start takes a token for itself. Both are taken synchronously, before the first await and // registered with the device, so a teardown beginning in this same turn closes it even while // this start queues behind the work another start holds the session lock for (#3220). + const launcher = options.launcher; const ownedAdmission = options.startAdmission; const startAdmission = ownedAdmission ?? openRunnerStartAdmission(device.id); const start = withRunnerSessionLock(device.id, async () => { @@ -139,6 +136,31 @@ export async function ensureRunnerSession( } } + if (launcher) { + assertRunnerSessionMayStart(options.expectedRunnerSessionId); + const { launchRunnerSession, refuseFailedRunnerPublication } = + await import('./runner-launched-session.ts'); + const session = await launchRunnerSession( + device, + launcher, + options, + budget, + startAdmission.signal, + ); + try { + if (budget.phase.signal?.aborted) { + await disposeRunnerSession(session, { graceful: false }); + budget.phase.signal.throwIfAborted(); + } + await refuseRetiredRunnerPublication(device, session, startAdmission); + await refuseFailedRunnerPublication(session); + } catch (error) { + if (session.state !== 'stopped') runnerSessions().set(device.id, session); + throw error; + } + runnerSessions().set(device.id, session); + return session; + } return await withRunnerLeaseLock( device.id, async () => await startRunnerSessionWithLease(device, options, budget, startAdmission), @@ -211,7 +233,10 @@ async function startRunnerSessionWithLease( }); await measureRunnerStartupStep(startupTimings, 'ensure_booted', async () => { assertRunnerStartAdmitsPreparation(device.id, startAdmission); - await ensureBootedIfNeeded(device); + await ensureBootedIfNeeded( + device, + AbortSignal.any([startAdmission.signal, ...(signal ? [signal] : [])]), + ); }); // Device first, host second: both answers can be wrong at once, and the phone's own state is the // one the caller can act on without admin rights. Probing the host first would publish only the @@ -254,7 +279,7 @@ async function startRunnerSessionWithLease( let xctestrunPath: string; let jsonPath: string; const runnerLogPath = resolveRunnerLaunchLogPath(options.logPath, device.id); - let runnerProcess: LaunchedRunnerProcess; + let runnerProcess: Awaited>; // One catch for everything between here and a runner that answers, because the device's own answer // belongs on all of it (#2690 review): a cold build, a warm derived cache that fails at install, and // an external xctestrun that never launches are different steps, and a caller told "developer disk @@ -267,7 +292,10 @@ async function startRunnerSessionWithLease( await ensureXctestrunArtifact(device, { ...options, startAdmission, - budget: createRunnerPhaseBudget(resolveRunnerBuildTimeoutMs(options, budget), signal), + budget: createRunnerPhaseBudget( + (await import('./runner-start-budget.ts')).resolveRunnerBuildTimeoutMs(options, budget), + signal, + ), }), ); startupTimings[BUILD_XCTESTRUN_TIMING] = xctestrunArtifact.buildMs; @@ -306,7 +334,11 @@ async function startRunnerSessionWithLease( // it; otherwise a queued build line reads as the runner's own failure output (#2681). await flushRunnerLogAppends(runnerLogPath).catch(() => {}); assertRunnerStartAdmitsPreparation(device.id, startAdmission); - return await launchRunnerProcess({ + return await launchLocalRunner({ + sessionId: buildRunnerSessionId(device.id, endpoint.port), + endpoint, + jsonPath, + cacheKey: xctestrunArtifact.cacheKey, device, xctestrunPath, derivedPath: xctestrunArtifact.derived, @@ -320,38 +352,24 @@ async function startRunnerSessionWithLease( } catch (error) { throw enrichRunnerStartupFailureWithDeviceStates(error, deviceStates); } - const sessionId = buildRunnerSessionId(device.id, endpoint.port); - const lease = buildRunnerLease({ - device, - sessionId, - runnerPid: runnerProcess.child.pid, - endpoint, - xctestrunPath, - cacheKey: xctestrunArtifact.cacheKey, - jsonPath, - runnerLogPath, - }); + const { lease, sessionId } = runnerProcess.launch.local; const session: RunnerSession = { sessionId, device, deviceId: device.id, endpoint, - xctestrunPath, xctestrunArtifact, - jsonPath, runnerLogPath, - testPromise: runnerProcess.wait, - child: runnerProcess.child, - endOutputObservation: runnerProcess.endOutputObservation, - readLogTail: runnerProcess.readLogTail, + launch: runnerProcess.launch, state: 'starting', commandCharges: new RunnerCommandAccounting(), startupRetryWake: runnerProcess.startupRetryWake, - launchDeadline: Deadline.fromTimeoutMs(resolveRunnerLaunchReadinessMs(budget)), + launchDeadline: Deadline.fromTimeoutMs( + (await import('./runner-start-budget.ts')).resolveRunnerLaunchReadinessMs(budget), + ), startupTimings, startupDeviceStates: deviceStates, logicalLeaseContext, - lease, speculative: options.speculative === true, }; if (signal?.aborted) { @@ -401,10 +419,10 @@ async function refuseRetiredRunnerPublication( } export function assertExpectedRunnerSession( - session: Pick, + session: Pick | undefined, expectedRunnerSessionId: string | undefined, ): void { - if (expectedRunnerSessionId !== undefined && session.sessionId !== expectedRunnerSessionId) { + if (expectedRunnerSessionId !== undefined && session?.sessionId !== expectedRunnerSessionId) { throw runnerSessionOwnershipChanged(); } } @@ -426,6 +444,10 @@ async function isRunnerSessionServing( device: DeviceInfo, existing: RunnerSession, ): Promise { + if (existing.state === 'draining') { + await stopRunnerSessionInternal(device.id, existing, { graceful: false }); + return false; + } const liveness = readRunnerSessionLivenessFor(existing); if (liveness === 'gone') { await measureRunnerStartupStep({}, 'stop_stale_session', async () => { @@ -472,7 +494,7 @@ async function resolveReusableRunnerSession( if (!(await isRunnerSessionServing(device, existing))) return null; const existingArtifact = existing.xctestrunArtifact; - if (existingArtifact?.cache === 'external') { + if (!existing.launch.local || existingArtifact?.cache === 'external') { emitDiagnostic({ level: 'debug', phase: 'ios_runner_session_reuse', @@ -480,7 +502,7 @@ async function resolveReusableRunnerSession( deviceId: device.id, sessionId: existing.sessionId, ready: existing.state === 'ready', - cache: existingArtifact.cache, + cache: existingArtifact?.cache, logicalLeaseContext: existing.logicalLeaseContext, }, }); @@ -578,7 +600,7 @@ export function readRunnerSessionLiveness(deviceId: string): RunnerSessionRegist function readRunnerSessionLivenessFor(session: RunnerSession): RunnerSessionLiveness { return resolveRunnerSessionLiveness({ state: session.state, - processRunning: isRunnerProcessAlive(session.child.pid), + processRunning: session.launch.isRunning(), }); } @@ -614,10 +636,8 @@ async function stopRunnerSessionInternal( ): Promise { const session = sessionOverride ?? runnerSessions().get(deviceId); if (!session) return; - // Once disposal has begun or finished, this session has no runner to wait on; a repeat stop - // would only re-signal a process that is already leaving and re-emit a teardown for a reason - // that has nothing left to tear down. - if (!canWorkWithRunnerSession(session)) return; + // A failed managed termination still owns its endpoint and must be retried before replacement. + if (!canWorkWithRunnerSession(session) && session.state !== 'draining') return; await disposeRunnerSession(session, options); if (runnerSessions().get(deviceId) === session) { runnerSessions().delete(deviceId); @@ -645,7 +665,10 @@ async function stopRetainedRunner( } /** The first command that is not a readiness probe makes the session the caller's, not a guess. */ -export function markRunnerSessionServed(session: RunnerSession, command: RunnerCommand): void { +export function markRunnerSessionServed( + session: Pick, + command: RunnerCommand, +): void { if (session.speculative && !isRunnerReadinessProbeCommand(command)) { session.speculative = false; } @@ -671,48 +694,63 @@ export async function releaseSpeculativeIosRunnerSession(deviceId: string): Prom ready: session.state === 'ready', }, }); - await stopIosRunnerSession(deviceId); + await stopRunnerSession(deviceId); return true; }); } export async function stopIosRunnerSession(deviceId: string): Promise { - await stopIosRunnerSessionDevice(deviceId); + await stopRunnerSession(deviceId, () => + withRunnerLeaseLock(deviceId, () => cleanupOwnedIosRunnerLease(deviceId)), + ); } -/** - * Stops the device's runner under the session lock. `fenceSettle`, when given, is called as the - * last step INSIDE that lock, so a start queued behind this stop wakes to a device whose teardown - * has fully settled and runs on rather than reading the still-pending fence as its own refusal. - */ -async function stopIosRunnerSessionDevice( +export async function stopRunnerSession( + deviceId: string, + cleanupOrphan?: () => Promise, +): Promise { + const settle = fenceRunnerStartAdmissionsForTeardown(deviceId); + await stopRunnerSessionDevice(deviceId, settle, cleanupOrphan); +} + +async function stopRunnerSessionDevice( deviceId: string, - fenceSettle?: () => void, + fenceSettle: () => void, + cleanupOrphan?: () => Promise, ): Promise { try { await withRunnerSessionLock(deviceId, async () => { - await withRunnerLeaseLock(deviceId, async () => { - const session = runnerSessions().get(deviceId); - await stopRunnerSessionInternal(deviceId, session, { - leaseLockHeld: true, - graceful: - session?.state === 'ready' && - !isRunnerMainThreadOccupied(session) && - !session.listenerWatch?.lost, - }); - await cleanupOwnedIosRunnerLease(deviceId); + const session = runnerSessions().get(deviceId); + await stopRunnerSessionInternal(deviceId, session, { + graceful: + session?.state === 'ready' && + !isRunnerMainThreadOccupied(session) && + !session.listenerWatch?.lost, }); - fenceSettle?.(); + await cleanupOrphan?.(); + fenceSettle(); }); } finally { - fenceSettle?.(); + fenceSettle(); } } -/** Retires startup recovery at close; only a serving, observed generation can remain idle. */ export async function releaseIosRunnerOnClose( deviceId: string, options: { retain: boolean }, + env?: NodeJS.ProcessEnv, +): Promise { + await releaseRunnerOnClose(deviceId, options, env, () => + withRunnerLeaseLock(deviceId, () => cleanupOwnedIosRunnerLease(deviceId)), + ); +} + +/** Retires startup recovery at close; only a serving, observed generation can remain idle. */ +export async function releaseRunnerOnClose( + deviceId: string, + options: { retain: boolean }, + env?: NodeJS.ProcessEnv, + cleanupOrphan?: () => Promise, ): Promise { const candidate = runnerSessions().get(deviceId); const busyAtClose = isRunnerMainThreadOccupied(candidate); @@ -724,8 +762,10 @@ export async function releaseIosRunnerOnClose( if (!session) return false; const { retainRunnerSession } = await import('./runner-retention.ts'); if ( - await retainRunnerSession(session, (retention, reason) => - stopRetainedRunner(session, retention, reason), + await retainRunnerSession( + session, + (retention, reason) => stopRetainedRunner(session, retention, reason), + env, ) ) { settleFence(); @@ -749,7 +789,7 @@ export async function releaseIosRunnerOnClose( }); } await stopRunnerPrepProcesses(deviceId); - await stopIosRunnerSessionDevice(deviceId, settleFence); + await stopRunnerSessionDevice(deviceId, settleFence, cleanupOrphan); } finally { settleFence(); } @@ -764,12 +804,23 @@ export async function abortAllIosRunnerSessions(): Promise { activeSessions.map((session) => session.deviceId), ); try { - await abortRunnerSessionsAndPrepProcesses(activeSessions); - for (const session of activeSessions) { - if (runnerSessions().get(session.deviceId) === session) { + const failures: unknown[] = []; + await abortRunnerSessionsAndPrepProcesses(activeSessions).catch((error: unknown) => { + failures.push(error); + }); + await Promise.allSettled(runnerSessionLocks().values()); + const unpublished = [...runnerSessions().values()].filter( + (session) => !activeSessions.includes(session), + ); + await abortRunnerSessionsAndPrepProcesses(unpublished).catch((error: unknown) => { + failures.push(error); + }); + for (const session of [...activeSessions, ...unpublished]) { + if (session.state === 'stopped' && runnerSessions().get(session.deviceId) === session) { runnerSessions().delete(session.deviceId); } } + if (failures.length) throw new AggregateError(failures, 'Runner shutdown failed'); } finally { // The abort settles even on its error path, so no device stays fenced past this teardown. settleFences(); @@ -793,21 +844,30 @@ export async function stopAllIosRunnerSessions(): Promise { // instead of answering the final prep sweep with another build (#3220). const settleFences = retireAllRunnerStartAdmissions(runnerSessions().keys()); try { - await abortAllIosRunnerSessions(); + const failures: unknown[] = []; + await abortAllIosRunnerSessions().catch((error: unknown) => { + failures.push(error); + }); const pending = Array.from(runnerSessions().keys()); - await Promise.allSettled( + const stopped = await Promise.allSettled( pending.map(async (deviceId) => { - await stopIosRunnerSession(deviceId); + await stopRunnerSession(deviceId); }), ); - await stopRunnerPrepProcesses(); + failures.push( + ...stopped.filter((result) => result.status === 'rejected').map((result) => result.reason), + ); + await stopRunnerPrepProcesses().catch((error: unknown) => { + failures.push(error); + }); + if (failures.length) throw new AggregateError(failures, 'Runner shutdown failed'); return owned; } finally { settleFences(); } } -function ensureBootedIfNeeded(device: DeviceInfo): Promise { +function ensureBootedIfNeeded(device: DeviceInfo, signal?: AbortSignal): Promise { if (device.kind !== 'simulator') { return Promise.resolve(); } @@ -819,12 +879,13 @@ function ensureBootedIfNeeded(device: DeviceInfo): Promise { }); return Promise.resolve(); } - return ensureBooted(device); + return ensureBooted(device, signal); } -async function ensureBooted(device: DeviceInfo): Promise { +async function ensureBooted(device: DeviceInfo, signal?: AbortSignal): Promise { await runXcrun(buildSimctlArgsForDevice(device, ['bootstatus', device.id, '-b']), { timeoutMs: RUNNER_STARTUP_TIMEOUT_MS, + signal, }); } @@ -873,34 +934,6 @@ export async function executeRunnerCommandWithSession( return data; } -/** - * What the xctestrun build may spend: the rest of the start budget, which an explicit - * `buildTimeoutMs` can only shorten. The build is the one step with no ceiling of its own, so a - * start with no caller left is still ended by the clock it opened (#2894). Throws when the start - * has nothing left, so a spent budget fails before xcodebuild is spawned. - */ -function resolveRunnerBuildTimeoutMs( - options: RunnerSessionOptions, - budget: RunnerStartBudget, -): number { - const remainingMs = - requireRunnerPhaseRemainingMs(budget.phase, 'runner_xctestrun_build') ?? budget.timeoutMs; - const explicitMs = normalizeRunnerStartupTimeoutMs(options.buildTimeoutMs); - return explicitMs === undefined ? remainingMs : Math.min(explicitMs, remainingMs); -} - -/** - * What the launched runner has to answer its first command, measured from launch. An explicit - * `startupTimeoutMs` bounds the whole start, readiness included, so readiness gets what is left of - * it. A defaulted start keeps the runner's own readiness window ({@link RUNNER_STARTUP_TIMEOUT_MS}): - * the default budget is sized for a cold build, and a runner that never answers must not be joined - * for the rest of it. Neither exceeds what the start budget has left. - */ -function resolveRunnerLaunchReadinessMs(budget: RunnerStartBudget): number { - const remainingMs = Math.floor(budget.phase.deadline?.remainingMs() ?? budget.timeoutMs); - return budget.explicit ? remainingMs : Math.min(RUNNER_STARTUP_TIMEOUT_MS, remainingMs); -} - async function measureRunnerStartupStep( timings: Record, phase: string, diff --git a/packages/apple-runner/src/runner-start-budget.ts b/packages/apple-runner/src/runner-start-budget.ts index ea9dc30dc7..73b9bff730 100644 --- a/packages/apple-runner/src/runner-start-budget.ts +++ b/packages/apple-runner/src/runner-start-budget.ts @@ -1,3 +1,4 @@ +import { RUNNER_STARTUP_TIMEOUT_MS } from './runner-startup-transport.ts'; import { AppError, createRequestCanceledError, @@ -9,6 +10,7 @@ import { addRunnerStartWaiter, cancelRunnerStartWaiter, createRunnerPhaseBudget, + requireRunnerPhaseRemainingMs, markRunnerStartRetryPending, releaseRunnerStartWaiter, type RunnerPhaseBudget, @@ -192,3 +194,31 @@ function emitDetachedRunnerStartFailed(error: unknown): void { }, }); } + +/** + * What the launched runner has to answer its first command, measured from launch. An explicit + * `startupTimeoutMs` bounds the whole start, readiness included, so readiness gets what is left of + * it. A defaulted start keeps the runner's own readiness window ({@link RUNNER_STARTUP_TIMEOUT_MS}): + * the default budget is sized for a cold build, and a runner that never answers must not be joined + * for the rest of it. Neither exceeds what the start budget has left. + */ +export function resolveRunnerLaunchReadinessMs(budget: RunnerStartBudget): number { + const remainingMs = Math.floor(budget.phase.deadline?.remainingMs() ?? budget.timeoutMs); + return budget.explicit ? remainingMs : Math.min(RUNNER_STARTUP_TIMEOUT_MS, remainingMs); +} + +/** + * What the xctestrun build may spend: the rest of the start budget, which an explicit + * `buildTimeoutMs` can only shorten. The build is the one step with no ceiling of its own, so a + * start with no caller left is still ended by the clock it opened (#2894). Throws when the start + * has nothing left, so a spent budget fails before xcodebuild is spawned. + */ +export function resolveRunnerBuildTimeoutMs( + options: RunnerSessionOptions, + budget: RunnerStartBudget, +): number { + const remainingMs = + requireRunnerPhaseRemainingMs(budget.phase, 'runner_xctestrun_build') ?? budget.timeoutMs; + const explicitMs = normalizeRunnerStartupTimeoutMs(options.buildTimeoutMs); + return explicitMs === undefined ? remainingMs : Math.min(explicitMs, remainingMs); +} diff --git a/packages/apple-runner/src/runner-startup-transport.ts b/packages/apple-runner/src/runner-startup-transport.ts index 8d0fa08775..01f293e9d4 100644 --- a/packages/apple-runner/src/runner-startup-transport.ts +++ b/packages/apple-runner/src/runner-startup-transport.ts @@ -17,6 +17,7 @@ import { classifyRunnerStartupFailure, enrichRunnerStartupFailureWithDeviceStates, isRunnerCommandProvablyUnwritten, + shouldRetainRunnerSession, isUsbmuxDeviceUnattachedError, RUNNER_CACHE_RECOVERY_HINT, runnerConnectFailureDetails, @@ -29,18 +30,18 @@ import { type RunnerEndpoint, } from './runner-contract.ts'; import type { RunnerSession } from './runner-session-types.ts'; +import type { LocalRunnerLaunchResources } from './runner-disposal.ts'; -export type RunnerConnectionSession = Pick< +type RunnerProcessConnectionSession = Pick< RunnerSession, - | 'startupRetryWake' - | 'child' - | 'startupDeviceStates' - | 'testPromise' - | 'readLogTail' - | 'device' - | 'runnerLogPath' - | 'state' + 'startupRetryWake' | 'launch' | 'startupDeviceStates' | 'device' | 'runnerLogPath' | 'state' >; +export type RunnerConnectionSession = RunnerProcessConnectionSession; + +function runnerProcessExited(session: RunnerConnectionSession | undefined): boolean { + return session?.launch.exited === true; +} + import { runnerLaunchLogClassificationDetails, runnerSimulatorSetFailureDetails, @@ -68,7 +69,11 @@ export async function waitForRunner( ): Promise { const deadline = Deadline.fromTimeoutMs(timeoutMs); const port = endpoint.port; - const { resolveRoute, markUsbmuxUnattached } = createRunnerCommandRouteResolver(device, port); + const { resolveRoute, markUsbmuxUnattached } = createRunnerCommandRouteResolver( + device, + port, + endpoint.http, + ); let route = await resolveRoute(deadline.remainingMs()); let lastError: unknown = null; // Every attempt posts the command itself, so the loop gives up "before send" only when no @@ -142,7 +147,7 @@ export async function waitForRunner( throw canceledRunnerConnect(commandMayHaveBeenWritten); } - if (session?.child.exitCode !== null && session?.child.exitCode !== undefined) { + if (session && runnerProcessExited(session)) { throw withRunnerWriteEvidence( await buildRunnerEarlyExitError({ session, port, logPath }), commandMayHaveBeenWritten, @@ -231,7 +236,7 @@ async function ensureRunnerAttemptCanStart(params: { timeoutMs: params.timeoutMs, }); } - if (params.session?.child.exitCode !== null && params.session?.child.exitCode !== undefined) { + if (params.session && runnerProcessExited(params.session)) { throw await buildRunnerEarlyExitError({ session: params.session, port: params.endpoint.port, @@ -248,7 +253,7 @@ async function ensureRunnerAttemptCanStart(params: { // from this generation's offset, and no child process is spawned for it. const session = params.session; if (session?.state !== 'starting') return; - const launchLogOutput = session.readLogTail?.(RUNNER_EARLY_EXIT_LOG_TAIL_BYTES) ?? ''; + const launchLogOutput = session.launch.readLogTail(RUNNER_EARLY_EXIT_LOG_TAIL_BYTES); if (!launchLogOutput) return; const terminalError = buildRunnerConnectError({ device: session.device, @@ -468,9 +473,11 @@ function buildRunnerConnectError(params: { const launchLogCause = resolveRunnerLaunchLogCause({ message, output: - params.launchLogOutput ?? session?.readLogTail?.(RUNNER_EARLY_EXIT_LOG_TAIL_BYTES) ?? '', + params.launchLogOutput ?? session?.launch.readLogTail(RUNNER_EARLY_EXIT_LOG_TAIL_BYTES) ?? '', device, }); + if (!launchLogCause && lastError instanceof AppError && shouldRetainRunnerSession(lastError)) + return lastError; const error = new AppError('COMMAND_FAILED', launchLogCause?.message ?? message, { port, url, @@ -497,11 +504,26 @@ function buildRunnerConnectError(params: { } export async function buildRunnerEarlyExitError(params: { - session: RunnerConnectionSession; + session: RunnerProcessConnectionSession; port: number; logPath?: string; }): Promise { - const { session, port, logPath } = params; + return params.session.launch.earlyExitError( + params.port, + params.logPath, + params.session.startupDeviceStates, + ); +} + +export async function buildLocalRunnerEarlyExitError( + session: Pick< + LocalRunnerLaunchResources, + 'testPromise' | 'readLogTail' | 'device' | 'runnerLogPath' + > & + Pick, + port: number, + logPath?: string, +): Promise { const result = await session.testPromise; const message = 'Runner did not accept connection (xcodebuild exited early)'; // The runner writes its own output file, so the exec result holds nothing for a file-backed @@ -524,7 +546,7 @@ export async function buildRunnerEarlyExitError(params: { // pointing at the request's log would advertise a file that does not contain what is quoted (#2681). logPath: session.runnerLogPath ?? logPath, xcodebuild: { - exitCode: result.exitCode, + exitCode: result?.exitCode, // One merged file since #2681: the tail is reported under `stderr`, which is where readers // already look, next to the file it came from. stderr: output, diff --git a/packages/apple-runner/src/runner-transport.ts b/packages/apple-runner/src/runner-transport.ts index 842097ee03..2657af38ed 100644 --- a/packages/apple-runner/src/runner-transport.ts +++ b/packages/apple-runner/src/runner-transport.ts @@ -42,7 +42,7 @@ async function postRunnerCommandOnce( ): Promise { throwIfCanceledBeforeSend(signal); const deadline = Deadline.fromTimeoutMs(timeoutMs); - const resolver = createRunnerCommandRouteResolver(device, endpoint.port); + const resolver = createRunnerCommandRouteResolver(device, endpoint.port, endpoint.http); let route = await resolver.resolveRoute(deadline.remainingMs()); if (route.kind === 'usbmux') { try { @@ -102,16 +102,24 @@ export async function postRunnerCommandOverRoute( 'no', ); } - return await fetchWithTimeout( + const response = await fetchWithTimeout( route.url, { method: 'POST', - headers: runnerRequestHeaders(endpoint), + headers: forwardedRunnerHeaders(endpoint), body: JSON.stringify(command), }, timeoutMs, signal, ); + if (endpoint.http && [502, 503, 504].includes(response.status)) { + await response.body?.cancel(); + throw new AppError('COMMAND_FAILED', 'Runner HTTP forwarding is unavailable', { + reason: 'runner_forwarding_unavailable', + status: response.status, + }); + } + return response; } /** A cancellation that lands before the POST is issued proves the runner received nothing. */ @@ -155,3 +163,11 @@ export async function fetchWithTimeout( throw error; } } + +function forwardedRunnerHeaders(endpoint: RunnerEndpoint): Record { + if (!endpoint.http?.headers) return runnerRequestHeaders(endpoint); + const headers = new Headers(endpoint.http.headers); + for (const [key, value] of Object.entries(runnerRequestHeaders(endpoint))) + headers.set(key, value); + return Object.fromEntries(headers); +} diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 01673064f1..169c264ad9 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -6,8 +6,8 @@ import { appleRunnerHost } from './runner-host.ts'; * the real host capabilities from `runner-host.ts`, and the module daemon, * platform, and CLI consumers import the host-bound operations from, so every * such consumer evaluates after the binding. Runner modules must not reach the - * host while they evaluate: they load before this module binds it. Types and - * host-free helpers come from the package façade directly. + * host while they evaluate: they load before this module binds it. Import types and + * host-free helpers from the package façade when no host binding is needed. */ bindAppleRunnerHost(appleRunnerHost); diff --git a/scripts/package-ios-simulator-runner.sh b/scripts/package-ios-simulator-runner.sh index 2018fa5792..73468419bc 100755 --- a/scripts/package-ios-simulator-runner.sh +++ b/scripts/package-ios-simulator-runner.sh @@ -92,6 +92,17 @@ if [ -n "$EXPECTED_RUNNER_BUNDLE_ID" ] && [ "$RUNNER_BUNDLE_ID" != "$EXPECTED_RU fi ditto "$RUNNER_APP_PATH" "$STAGED_APP_PATH" +# Remote simulators cannot resolve dependencies from the build machine's Xcode paths. +PLATFORM_PATH="$(xcrun --sdk iphonesimulator --show-sdk-platform-path)" +INTEROP_LIBRARY="$PLATFORM_PATH/Developer/usr/lib/lib_TestingInterop.dylib" +mkdir -p "$STAGED_APP_PATH/Frameworks" +if [ -f "$INTEROP_LIBRARY" ]; then + cp "$INTEROP_LIBRARY" "$STAGED_APP_PATH/Frameworks/" +elif [ -d "$STAGED_APP_PATH/Frameworks/Testing.framework" ] && + [ ! -f "$STAGED_APP_PATH/Frameworks/lib_TestingInterop.dylib" ]; then + echo "Testing.framework requires lib_TestingInterop.dylib, but it was not found at $INTEROP_LIBRARY" >&2 + exit 1 +fi rm -f "$ARCHIVE_PATH" ( cd "$STAGE_DIR"