refactor(apple): support managed XCTest runner launchers

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerListenerPeer.swift

@@ -3,15 +3,18 @@ import Network
 /// Who reached the runner listener, decided once per connection from its path. The host reaches the
 /// runner only over loopback (a simulator or macOS host, or usbmux on a device) or through the
 /// device's CoreDevice tunnel, a `utun` interface of type `.other`. A peer on any other interface
-/// reached the device directly over a network it shares with it, and is closed before it can hold a
-/// connection open.
+/// reached the device directly over a shared network and requires explicit forwarding support.
 enum RunnerListenerPeer: Equatable {
   /// May hold an idle connection: the simulator listener watch (`runner-retention.ts`) does, to
   /// learn when the runner dies.
   case loopback
   case tunnel
   case directNetwork
 
+  func isAdmitted(acceptsForwardedConnections: Bool) -> Bool {
+    self != .directNetwork || acceptsForwardedConnections
+  }
+
   static func of(_ connection: NWConnection) -> RunnerListenerPeer {
     classify(
       isLoopbackAddress: isLoopback(connection.endpoint),

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift

@@ -9,6 +9,7 @@ import Network
 struct RunnerListenerConfiguration {
   let port: NWEndpoint.Port
   let token: RunnerListenerToken
+  let bindAddress: IPv4Address?
 
   static func fromEnvironment(
     _ environment: [String: String] = ProcessInfo.processInfo.environment
@@ -20,6 +21,11 @@ struct RunnerListenerConfiguration {
     else {
       return nil
     }
-    return RunnerListenerConfiguration(port: port, token: token)
+    var bindAddress: IPv4Address?
+    if let rawAddress = environment["AGENT_DEVICE_RUNNER_BIND_IP"] {
+      guard let address = IPv4Address(rawAddress), address != .any else { return nil }
+      bindAddress = address
+    }
+    return RunnerListenerConfiguration(port: port, token: token, bindAddress: bindAddress)
   }
 }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -124,8 +124,17 @@ extension RunnerTests {
       )
     }
     let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png"
-    let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
+#if os(iOS)
+    // A managed XCTest host can override TMPDIR outside the app container.
+    // The returned tmp/ path must resolve inside the runner's data container.
+    let temporaryDirectory = URL(fileURLWithPath: NSHomeDirectory(), isDirectory: true)
+      .appendingPathComponent("tmp", isDirectory: true)
+#else
+    let temporaryDirectory = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true)
+#endif
+    let filePath = temporaryDirectory.appendingPathComponent(fileName).path
     do {
+      try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true)
       try pngData.write(to: URL(fileURLWithPath: filePath))
     } catch {
       return Response(

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift

@@ -36,12 +36,12 @@ extension RunnerTests {
 
   private func admit(_ connection: NWConnection, token: RunnerListenerToken) {
     let peer = RunnerListenerPeer.of(connection)
-    guard peer != .directNetwork else {
+    guard peer.isAdmitted(acceptsForwardedConnections: acceptsForwardedConnections) else {
       connection.cancel()
       return
     }
     let inbound = RunnerInboundRequest(token: token)
-    if peer == .tunnel {
+    if peer != .loopback {
       armDeadline(for: inbound, over: connection)
     }
     receiveRequest(inbound, over: connection)

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -81,6 +81,7 @@ final class RunnerTests: XCTestCase {
   static let defaultRecordingFps: Int32 = 15
   var listener: NWListener?
   var listenerToken: RunnerListenerToken?
+  var acceptsForwardedConnections = false
   var doneExpectation: XCTestExpectation?
   var transportConnections: [ObjectIdentifier: NWConnection] = [:]
   let shutdownResponseWrites = DispatchGroup()
@@ -342,13 +343,14 @@ final class RunnerTests: XCTestCase {
     NSLog("AGENT_DEVICE_RUNNER_HEADLESS_STARTUP=1")
     guard let configuration = RunnerListenerConfiguration.fromEnvironment() else {
       NSLog("AGENT_DEVICE_RUNNER_LISTENER_CONFIGURATION_MISSING")
-      XCTFail("runner listener port or token is missing; refusing to listen")
+      XCTFail("runner listener port, token, or bind address is missing or invalid; refusing to listen")
       return
     }
     listenerToken = configuration.token
+    acceptsForwardedConnections = configuration.bindAddress != nil
     NSLog("AGENT_DEVICE_RUNNER_AUTH=required")
     NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", configuration.port.rawValue)
-    listener = try Self.makeRunnerListener(port: configuration.port)
+    listener = try Self.makeRunnerListener(port: configuration.port, bindAddress: configuration.bindAddress)
     // Reads the OS prompt strings off the alert path, so the first accept/dismiss deadline does not pay for it.
     DispatchQueue.global(qos: .utility).async { _ = RunnerAlertLabelVocabulary.system }
     DispatchQueue.global(qos: .utility).async { _ = RunnerLaunchConfirmationPrompt.system }
@@ -397,9 +399,14 @@ final class RunnerTests: XCTestCase {
     }
   }
 
-  /// Simulators and macOS bind loopback: the host reaches them only through 127.0.0.1. A physical
-  /// device listens on every interface because the CoreDevice tunnel arrives on a network address.
-  static func makeRunnerListener(port: NWEndpoint.Port) throws -> NWListener {
+  /// Cloud simulators bind their assigned address; local simulators keep the loopback listener.
+  /// Physical devices accept CoreDevice tunnel connections on their network interface.
+  static func makeRunnerListener(port: NWEndpoint.Port, bindAddress: IPv4Address? = nil) throws -> NWListener {
+    if let bindAddress {
+      let parameters = NWParameters.tcp
+      parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(bindAddress), port: port)
+      return try NWListener(using: parameters)
+    }
     #if targetEnvironment(simulator) || os(macOS)
       let parameters = NWParameters.tcp
       parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(.loopback), port: port)

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m

@@ -114,34 +114,33 @@
   }
 }
 
-// One window read. Both answers are required: an app with no window answers `frame` with
-// `(0,0 0x0)` without raising, so only `exists` says whether the window is there. The window
-// object is handed back because resolving this instance is what its `screen` will answer from.
-// The frame is reported even when refused, so a refusal names what it looked at.
+// Check existence before geometry: reading a missing window's frame records an XCTest failure,
+// even when the exception is caught. Keep the resolved instance for its screen lookup.
 static RunnerApplicationScreenFailure RunnerResolveUsableWindow(
   id application,
   id _Nullable *window,
   CGRect *observedFrame
 ) {
   __block id readWindow = nil;
   __block CGRect frame = CGRectNull;
+  __block BOOL frameRead = NO;
   __block BOOL resolved = NO;
   NSString *exception = [RunnerObjCExceptionCatcher catchException:^{
     id candidate = [[application valueForKey:@"windows"] valueForKey:@"firstMatch"];
     NSNumber *present = [candidate valueForKey:@"exists"];
     resolved = [present isKindOfClass:NSNumber.class] && present.boolValue;
+    if (!resolved) return;
     NSValue *frameValue = [candidate valueForKey:@"frame"];
     [frameValue getValue:&frame size:sizeof(frame)];
+    frameRead = YES;
     readWindow = candidate;
   }];
   *observedFrame = frame;
   if (exception != nil || !resolved || !AgentDeviceForegroundFrameIsMeasurable(frame)) {
     NSLog(
-      @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) raised=%@ resolved=%@",
-      frame.origin.x,
-      frame.origin.y,
-      frame.size.width,
-      frame.size.height,
+      @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=%@ raised=%@ resolved=%@",
+      frameRead ? [NSString stringWithFormat:@"(%g,%g %gx%g)",
+        frame.origin.x, frame.origin.y, frame.size.width, frame.size.height] : @"unread",
       exception != nil ? @"yes" : @"no",
       resolved ? @"yes" : @"no"
     );

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+EnvironmentTests.swift

@@ -31,5 +31,15 @@ extension RunnerTests {
       )
     }
   }
+  func testListenerConfigurationAcceptsOnlyConcreteBindAddresses() {
+    let base = ["AGENT_DEVICE_RUNNER_PORT": "8100", "AGENT_DEVICE_RUNNER_TOKEN": String(repeating: "t", count: 32)]
+    XCTAssertNil(RunnerListenerConfiguration.fromEnvironment(base)?.bindAddress)
+    let remote = base.merging(["AGENT_DEVICE_RUNNER_BIND_IP": "10.244.2.3"]) { $1 }
+    XCTAssertEqual(RunnerListenerConfiguration.fromEnvironment(remote)?.bindAddress?.debugDescription, "10.244.2.3")
+    for invalid in ["", "0.0.0.0", "localhost", "10.244.2.999"] {
+      XCTAssertNil(RunnerListenerConfiguration.fromEnvironment(base.merging(["AGENT_DEVICE_RUNNER_BIND_IP": invalid]) { $1 }))
+    }
+  }
+
 }
 #endif

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift

@@ -101,15 +101,19 @@ extension RunnerTests {
     let message = try XCTUnwrap(response.data?.message)
     // The answer names the file rather than carrying bytes, and the name is a host-resolvable form:
     // absolute on macOS, container-relative `tmp/…` on iOS. Both name the one file just written
-    // into the runner's temporary directory.
+    // into the runner's data container on iOS, regardless of the host's TMPDIR.
     let fileName = URL(fileURLWithPath: message).lastPathComponent
     XCTAssertTrue(message.hasSuffix(fileName), message)
 #if os(iOS)
     XCTAssertTrue(message.hasPrefix("tmp/"), message)
 #elseif os(macOS)
     XCTAssertTrue(message.hasPrefix("/"), message)
 #endif
+#if os(iOS)
+    let storedPath = (NSHomeDirectory() as NSString).appendingPathComponent(message)
+#else
     let storedPath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
+#endif
     XCTAssertEqual(try Data(contentsOf: URL(fileURLWithPath: storedPath)), pngData)
     XCTAssertEqual(response.data?.screenshotMetadata?.pixelsPerPoint, 3)
     try? FileManager.default.removeItem(atPath: storedPath)

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ListenerPeerTests.swift

@@ -22,6 +22,15 @@ extension RunnerTests {
     }
   }
 
+  func testListenerPeerRequiresExplicitForwardingForDirectNetworkAccess() {
+    XCTAssertFalse(RunnerListenerPeer.directNetwork.isAdmitted(acceptsForwardedConnections: false))
+    XCTAssertTrue(RunnerListenerPeer.directNetwork.isAdmitted(acceptsForwardedConnections: true))
+    for peer: RunnerListenerPeer in [.loopback, .tunnel] {
+      XCTAssertTrue(peer.isAdmitted(acceptsForwardedConnections: false))
+      XCTAssertTrue(peer.isAdmitted(acceptsForwardedConnections: true))
+    }
+  }
+
   func testListenerPeerRecognizesEveryLoopbackAddressForm() {
     let peer = { (host: String) in NWEndpoint.hostPort(host: NWEndpoint.Host(host), port: 8100) }
     XCTAssertTrue(RunnerListenerPeer.isLoopback(peer("127.0.0.1")))

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift

@@ -151,6 +151,16 @@ extension RunnerTests {
     #endif
   }
 
+  func testRunnerListenerRequiresTheConfiguredBindAddressAndPort() throws {
+    let address = try XCTUnwrap(IPv4Address("10.23.45.67"))
+    let port = try XCTUnwrap(NWEndpoint.Port(rawValue: 8100))
+    let listener = try Self.makeRunnerListener(port: port, bindAddress: address)
+    XCTAssertEqual(
+      listener.parameters.requiredLocalEndpoint,
+      .hostPort(host: .ipv4(address), port: port)
+    )
+  }
+
   func testListenerRefusesRequestsWithoutTheTokenBeforeDecoding() throws {
     let listener = try startTransportListener()
     defer { listener.cancel() }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift

@@ -202,6 +202,7 @@ extension RunnerTests {
     XCTAssertEqual(failure, .unresolvedWindow)
     XCTAssertNil(screen)
     XCTAssertEqual(displayID, 0)
+    XCTAssertFalse(window.resolved, "a missing window's frame lookup records an XCTest failure")
   }
   /// The sole-screen rule's decision, at the seam the production probe feeds: the values the
   /// AXRuntime probe reads in, never a device list. The windowed/unknown rows below fail without

packages/apple-runner/src/__tests__/fake-runner-server.ts

@@ -16,6 +16,7 @@ export type FakeRunnerResponse =
   | { kind: 'hangUp' }
   /** Hangs up on this request and every later one for the command: the entry is never consumed. */
   | { kind: 'hangUpAlways' }
+  | { kind: 'httpErrorAlways'; status: number }
   /** Hangs up and stops listening, as a runner process that died mid-command. */
   | { kind: 'exit' }
   /** Never answers, as a runner still executing the command when its caller gives up. */
@@ -104,7 +105,9 @@ function recordFakeRunnerRequest(req: http.IncomingMessage, raw: string): FakeRu
 function takeScriptedResponse(
   queue: FakeRunnerResponse[] | undefined,
 ): FakeRunnerResponse | undefined {
-  return queue?.[0]?.kind === 'hangUpAlways' ? queue[0] : queue?.shift();
+  return queue?.[0]?.kind === 'hangUpAlways' || queue?.[0]?.kind === 'httpErrorAlways'
+    ? queue[0]
+    : queue?.shift();
 }
 
 function writeFakeRunnerResponse(
@@ -117,6 +120,10 @@ function writeFakeRunnerResponse(
     return;
   }
   if (next.kind === 'hold') return;
+  if (next.kind === 'httpErrorAlways') {
+    res.writeHead(next.status).end('upstream unavailable');
+    return;
+  }
   if (next.kind === 'hangUp' || next.kind === 'hangUpAlways') {
     res.destroy();
     return;

packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts

@@ -1,3 +1,4 @@
+import { makeRunnerSession as makeLocalRunnerSession } from './runner-session-fixtures.ts';
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import { afterEach, test, vi } from 'vitest';
@@ -40,7 +41,7 @@ afterEach(async () => {
 });
 
 function runnerSession(port: number): RunnerSession {
-  return {
+  return makeLocalRunnerSession({
     sessionId: `fake:${port}`,
     device: IOS_SIMULATOR,
     deviceId: IOS_SIMULATOR.id,
@@ -51,7 +52,7 @@ function runnerSession(port: number): RunnerSession {
     child: { pid: process.pid, exitCode: null },
     state: 'ready',
     commandCharges: new RunnerCommandAccounting(),
-  };
+  });
 }
 
 const TAP: RunnerCommand = { command: 'tap', x: 10, y: 10, commandId: 'cmd-1' };

packages/apple-runner/src/__tests__/index-recovery-wiring.test.ts

@@ -11,7 +11,10 @@ import { withAppleRunnerProvider } from '../runner-provider.ts';
 import { classifyRunnerReportedError, type RunnerCommand } from '../runner-contract.ts';
 import { RUNNER_REPLY_LOST_REASON } from '../runner-error-classification.ts';
 import { createRunnerPhaseBudget, requireRunnerPhaseRemainingMs } from '../runner-phase-budget.ts';
-import { captureDiagnostics } from './runner-session-fixtures.ts';
+import {
+  captureDiagnostics,
+  makeRunnerSession as makeLocalRunnerSession,
+} from './runner-session-fixtures.ts';
 import {
   startFakeRunnerServer,
   type FakeRunnerResponse,
@@ -104,7 +107,7 @@ afterEach(async () => {
 });
 
 function makeRunnerSession(port: number, sessionId = `wiring:${port}`): RunnerSession {
-  const session: RunnerSession = {
+  const session: RunnerSession = makeLocalRunnerSession({
     sessionId,
     device: IOS_SIMULATOR,
     deviceId: IOS_SIMULATOR.id,
@@ -115,7 +118,7 @@ function makeRunnerSession(port: number, sessionId = `wiring:${port}`): RunnerSe
     child: { pid: process.pid, exitCode: null },
     state: 'ready',
     commandCharges: new RunnerCommandAccounting(),
-  };
+  });
   return session;
 }
 

packages/apple-runner/src/__tests__/index-runner-requests.test.ts

@@ -73,6 +73,7 @@ test('runner-internal request sites build exactly their runner-requests.json ent
   });
   const session = makeRunnerSession({
     port: server.port,
+    testPromise: new Promise(() => {}),
     xctestrunPath: path.join(scratch, 'runner.xctestrun'),
     jsonPath: path.join(scratch, 'runner.json'),
   });
@@ -91,6 +92,7 @@ test('runner-internal request sites build exactly their runner-requests.json ent
     async (request) => await executeRunnerCommand(IOS_SIMULATOR, request, {}),
   );
   await notifyIosRunnerAppRelaunched(IOS_SIMULATOR);
+  vi.spyOn(session.launch, 'waitForExit').mockResolvedValue(true);
   await disposeRunnerSession(session);
 
   appleRunnerTestHost.update({

packages/apple-runner/src/__tests__/runner-adoption.test.ts

@@ -191,11 +191,11 @@ test('adoption succeeds for a live, matching, probe-healthy runner', async () =>
   expect(session).not.toBeNull();
   expect(session?.endpoint.port).toBe(lease.port);
   expect(session?.state).toBe('ready');
-  expect(session?.child.pid).toBe(424242);
+  expect(session?.launch.local?.child?.pid).toBe(424242);
   expect(session?.sessionId).toBe(lease.sessionId);
   expect(session?.xctestrunArtifact?.reason).toBe('adopted_from_lease');
   expect(session?.xctestrunArtifact?.cacheKey).toBe(lease.cacheKey);
-  expect(session?.lease?.cacheKey).toBe(lease.cacheKey);
+  expect(session?.launch.local?.lease?.cacheKey).toBe(lease.cacheKey);
   // Adoption transfers ownership: the lease on disk now belongs to us.
   expect(readStaleRunnerLease(simulator.id)).toBeNull();
 });
@@ -213,7 +213,7 @@ test('the probe and the adopted session carry the token the runner was launched
   });
   expect(session).toMatchObject({
     endpoint: { port: lease.port, token: 'runner-token-fixture' },
-    lease: { runnerToken: 'runner-token-fixture' },
+    launch: { local: { lease: { runnerToken: 'runner-token-fixture' } } },
   });
 });
 
@@ -365,7 +365,7 @@ test('adoption accepts a legacy lease whose live pid is runner-shaped', async ()
   const session = await tryAdoptRunnerSessionFromLease(simulator, {});
 
   expect(session?.state).toBe('ready');
-  expect(session?.child.pid).toBe(424242);
+  expect(session?.launch.local?.child?.pid).toBe(424242);
 });
 
 test('a runner for a simulator in a custom simulator set is adopted', async () => {
@@ -492,7 +492,7 @@ test('physical coredevice lane adopts the detached runner and names its lane', a
   const session = await tryAdoptRunnerSessionFromLease(physicalCoreDevice, {});
 
   expect(session?.state).toBe('ready');
-  expect(session?.child.pid).toBe(lease.runnerPid);
+  expect(session?.launch.local?.child?.pid).toBe(lease.runnerPid);
   expect(session?.endpoint.port).toBe(lease.port);
   expect(
     emittedDiagnostics.find((event) => event.phase === 'ios_runner_lease_adopted')?.data,