fix(ios): book every surface on the lit panel, not the first window

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift

@@ -21,16 +21,24 @@ enum RunnerAppScreenCaptureFailure: String, Error {
   case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED"
   case unresolvedScreen = "APP_SCREEN_UNRESOLVED"
   case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE"
+  case offLitPanel = "APP_SCREEN_OFF_LIT_PANEL"
 
   init(_ failure: RunnerApplicationScreenFailure) {
     switch failure {
     case .unresolvedWindow:
       self = .unresolvedWindow
+    case .offLitPanel:
+      self = .offLitPanel
     default:
       self = .unresolvedScreen
     }
   }
 
+  /// The refusal as a runner reply.
+  var response: Response {
+    Response(ok: false, error: ErrorPayload(code: rawValue, message: message, hint: hint))
+  }
+
   var message: String {
     switch self {
     case .unresolvedWindow:
@@ -39,6 +47,8 @@ enum RunnerAppScreenCaptureFailure: String, Error {
       return "The resolved app window did not report the display it is shown on."
     case .unrenderableImage:
       return "The resolved display handed back an image that could not be drawn upright."
+    case .offLitPanel:
+      return "Every window of the target app is on the panel the host reports as dark."
     }
   }
 
@@ -50,6 +60,8 @@ enum RunnerAppScreenCaptureFailure: String, Error {
       return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved."
     case .unrenderableImage:
       return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question."
+    case .offLitPanel:
+      return "Unfold or fold the device so the target app is on the lit panel, or bring it forward there; runner.log names the panel in AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW."
     }
   }
 }
@@ -82,14 +94,17 @@ extension RunnerTests {
   ///
   /// Never activates `app`: an observation must not change which app is foregrounded just to get a
   /// screen, so the caller passes the app context it already intends to observe.
+  ///
+  /// `litPanel` is the panel the request named; nil keeps the app's first window.
   func captureResolvedAppScreen(
-    app: XCUIApplication
+    app: XCUIApplication,
+    litPanel: RunnerLitPanel?
   ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
     var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
       .unresolvedScreen
     )
     let capture = {
-      outcome = Self.resolveCapturedAppScreen(app: app)
+      outcome = Self.resolveCapturedAppScreen(app: app, litPanel: litPanel)
     }
     if Thread.isMainThread {
       capture()
@@ -109,22 +124,33 @@ extension RunnerTests {
   /// Only an unresolved *window* moves on to the system surface. A window that resolved and then
   /// refused to name its display is the failure the host has to see, and no second question about a
   /// different process should bury it.
-  func captureObservedScreen(app: XCUIApplication) -> Result<
+  func captureObservedScreen(app: XCUIApplication, litPanel: RunnerLitPanel?) -> Result<
     CapturedAppScreen, RunnerAppScreenCaptureFailure
   > {
     selectObservedScreenCapture(
-      resolving: { captureResolvedAppScreen(app: app) },
-      fallingBack: { captureResolvedAppScreen(app: springboard) }
+      resolving: { captureResolvedAppScreen(app: app, litPanel: litPanel) },
+      fallingBack: { captureResolvedAppScreen(app: springboard, litPanel: litPanel) }
     )
   }
 
   private static func resolveCapturedAppScreen(
-    app: XCUIApplication
+    app: XCUIApplication,
+    litPanel: RunnerLitPanel?
   ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
     var screen: AnyObject?
     var displayID: UInt = 0
     var resolution: RunnerApplicationScreenFailure = .unresolvedScreen
-    guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution),
+    var resolved: Bool
+    if let litPanel {
+      var window: AnyObject?
+      var frame = CGRect.null
+      resolution = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame, &screen)
+      resolved = resolution == .none
+      displayID = litPanel.displayID
+    } else {
+      resolved = RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution)
+    }
+    guard resolved,
       let captureScreen = screen as? XCUIScreen
     else {
       return .failure(RunnerAppScreenCaptureFailure(resolution))

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift

@@ -0,0 +1,131 @@
+import XCTest
+
+// The lit panel of the command in flight (ADR 0025), main-owned and written behind main work a
+// previous command left queued, so that work still reads its own command's panel.
+struct LitPanelBookingKey: Equatable {
+  let bundleID: String
+  let requestID: UUID
+}
+
+struct LitPanelSurface {
+  let key: LitPanelBookingKey
+  let displayID: UInt
+  let window: XCUIElement
+  let frame: CGRect
+}
+
+/// The command's lit panel and what it booked on it. Read and written only on the main thread.
+final class RunnerLitPanelState {
+  var requestID: UUID?
+  var panel: RunnerLitPanel?
+  var surface: LitPanelSurface?
+}
+
+extension RunnerTests {
+  @discardableResult
+  func setRequestLitPanel(from command: Command) -> UUID {
+    let requestID = UUID()
+    var panel: RunnerLitPanel?
+    if let fact = command.litPanel, fact.displayId != 0, fact.widthPt.isFinite, fact.heightPt.isFinite,
+      fact.widthPt > 0, fact.heightPt > 0
+    {
+      panel = RunnerLitPanel(displayID: fact.displayId, widthPt: fact.widthPt, heightPt: fact.heightPt)
+    }
+    let state = litPanelState
+    let write = {
+      state.requestID = requestID
+      state.panel = panel
+      state.surface = nil
+    }
+    if Thread.isMainThread {
+      write()
+    } else {
+      DispatchQueue.main.async(execute: write)
+    }
+    return requestID
+  }
+
+  func endRequestLitPanel(_ requestID: UUID) {
+    let state = litPanelState
+    let clear = {
+      guard state.requestID == requestID else { return }
+      state.requestID = nil
+      state.panel = nil
+      state.surface = nil
+    }
+    if Thread.isMainThread {
+      clear()
+    } else {
+      DispatchQueue.main.async(execute: clear)
+    }
+  }
+
+  /// The lit panel the current request named, or nil for a single-panel request. Main only.
+  var requestLitPanel: RunnerLitPanel? { litPanelState.panel }
+
+  /// Books `app`'s lit-panel window for the request, or returns why it cannot.
+  func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? {
+    guard let litPanel = requestLitPanel else { return nil }
+    guard let requestID = litPanelState.requestID,
+      let bundleID = safely("LIT_PANEL_APP_IDENTITY", { app.value(forKey: "bundleID") as? String })?.trimmedNonEmpty
+    else { return .unresolvedScreen }
+    let key = LitPanelBookingKey(bundleID: bundleID, requestID: requestID)
+    if litPanelState.surface?.key == key { return nil }
+    litPanelState.surface = nil
+    var window: AnyObject?
+    var frame = CGRect.null
+    var screen: AnyObject?
+    let failure = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame, &screen)
+    guard failure == .none, let element = window as? XCUIElement else {
+      return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure)
+    }
+    litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, frame: frame)
+    return nil
+  }
+
+  func requireRequestLitPanelSurface(app: XCUIApplication) throws -> LitPanelSurface {
+    if let refusal = bookRequestLitPanelSurface(app: app) { throw refusal }
+    guard let surface = litPanelState.surface else {
+      throw RunnerAppScreenCaptureFailure.unresolvedScreen
+    }
+    return surface
+  }
+
+  /// The roots of every element read: the app, or the windows proven on the lit display. A walk that
+  /// cannot read the list, or that records an issue (#2995), throws `APP_SCREEN_UNRESOLVED`.
+  func elementSearchRoots(root: XCUIElement, litDisplayID: UInt?) throws -> [XCUIElement] {
+    guard let litDisplayID else { return [root] }
+    var windows: [XCUIElement]?
+    let walk = holdingRecordedIssues {
+      windows = RunnerCopyLitPanelWindows(root, litDisplayID)?.compactMap { $0 as? XCUIElement }
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS
+      if litPanelWindowWalkIssueForTesting {
+        record(XCTIssue(type: .assertionFailure, compactDescription: "Window vanished during the lit-panel walk"))
+      }
+#endif
+    }
+    guard walk.issues.isEmpty, walk.exceptionMessage == nil, let windows, !windows.isEmpty else {
+      for issue in walk.issues {
+        NSLog("AGENT_DEVICE_RUNNER_LIT_PANEL_WALK_ISSUE=%@", issue.compactDescription)
+      }
+      NSLog(
+        "AGENT_DEVICE_RUNNER_LIT_PANEL_WALK_REFUSED issues=%ld exception=%@ windows=%ld",
+        walk.issues.count, walk.exceptionMessage ?? "none", windows?.count ?? -1
+      )
+      throw RunnerAppScreenCaptureFailure.unresolvedScreen
+    }
+    return windows
+  }
+
+  func elementQueries(app: XCUIApplication, matching type: XCUIElement.ElementType) throws -> [XCUIElementQuery] {
+    try elementQueries(root: app, matching: type, litDisplayID: requestLitPanel?.displayID)
+  }
+
+  func elementQueries(
+    root: XCUIElement,
+    matching type: XCUIElement.ElementType,
+    litDisplayID: UInt?
+  ) throws -> [XCUIElementQuery] {
+    try elementSearchRoots(root: root, litDisplayID: litDisplayID).map { $0.descendants(matching: type) }
+  }
+}

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift

@@ -190,7 +190,7 @@ extension RunnerTests {
   func resolvedScrollViewport(
     app: XCUIApplication,
     context: SynthesizedCoordinateContext
-  ) -> RunnerScrollViewport {
+  ) throws -> RunnerScrollViewport {
 #if os(iOS)
     // Every scroll reports its decision, including the two ways it avoids reading the keyboard at
     // all: unavailable accessibility health, and a probe that finds no keyboard.
@@ -223,7 +223,7 @@ extension RunnerTests {
     }
     return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: clip)
 #else
-    let fallbackFrame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
+    let fallbackFrame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
     return ScrollViewportPolicy.frames(referenceFrame: fallbackFrame, clip: .unobstructed)
 #endif
   }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift

@@ -290,7 +290,7 @@ extension RunnerTests {
         timeout: 1,
         timeoutError: snapshotMainThreadTimeoutError("reading private AX viewport")
       ) {
-        self.safeSnapshotViewport(app: app, readingOrientation: true)
+        try self.safeSnapshotViewport(app: app, readingOrientation: true)
       }
       if case .missing = reported {
         return fallback

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift

@@ -95,7 +95,7 @@ extension RunnerTests {
   }
 
   @MainActor
-  func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) -> Response {
+  func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) throws -> Response {
     if action == "accept" || action == "dismiss" {
       guard let button = chooseAlertButton(alert, action: action) else {
         // The choice refused before any activation, so this discloses dispatched: no like the
@@ -141,7 +141,7 @@ extension RunnerTests {
       // (#2546). The post-tap settle stays, because the verification below reads the alert this tap
       // replaces; an alert that dismisses and presents an identical replacement passes through a
       // window with no alert, and a first read landing there reports a dismissal nothing proved.
-      guard let outcome = activateAlertButton(alert, button: button, action: action, frame: buttonFrame, deadline: deadline) else {
+      guard let outcome = try activateAlertButton(alert, button: button, action: action, frame: buttonFrame, deadline: deadline) else {
         return alertVerificationResponse(.timedOut, action: action, activated: false)
       }
       if let response = unsupportedResponse(for: outcome) {
@@ -190,8 +190,9 @@ extension RunnerTests {
     action: String,
     frame: CGRect,
     deadline: Date
-  ) -> RunnerInteractionOutcome? {
+  ) throws -> RunnerInteractionOutcome? {
     var outcome: RunnerInteractionOutcome?
+    var activationError: Error?
     withUIInterruptionHandlingDisabledIfSupported(alert.ownerApp) {
       withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) {
         guard Date() < deadline else { return }
@@ -223,13 +224,20 @@ extension RunnerTests {
           }
           hostContext = context
         }
-        switch synthesizedTapAt(
-          app: alert.ownerApp,
-          x: frame.midX,
-          y: frame.midY,
-          context: hostContext,
-          deadline: deadline
-        ) {
+        let attempt: RunnerInteractionOutcome?
+        do {
+          attempt = try synthesizedTapAt(
+            app: alert.ownerApp,
+            x: frame.midX,
+            y: frame.midY,
+            context: hostContext,
+            deadline: deadline
+          )
+        } catch {
+          activationError = error
+          return
+        }
+        switch attempt {
         case .performed?:
           typealias WaitForQuiescence = @convention(c) (NSObject, Selector, Bool) -> Void
           let waitForQuiescence = unsafeBitCast(
@@ -247,15 +255,15 @@ extension RunnerTests {
           break
         }
 #else
-        outcome = activateElement(
-          app: alert.ownerApp,
-          element: button,
-          action: "alert \(action)",
-          resolvedFrame: frame
-        )
+        do {
+          outcome = try activateElement(app: alert.ownerApp, element: button, action: "alert \(action)", resolvedFrame: frame)
+        } catch {
+          activationError = error
+        }
 #endif
       }
     }
+    if let activationError { throw activationError }
     return outcome
   }
 

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift

@@ -7,6 +7,8 @@ extension RunnerTests {
   func executeAccepted(command: Command) throws -> Response {
     commandJournal.start(command: command)
     pendingTargetActivation = nil
+    let panelRequestID = setRequestLitPanel(from: command)
+    defer { endRequestLitPanel(panelRequestID) }
     do {
       let response = try executeDispatched(command: command)
       commandJournal.finish(command: command, response: response)
@@ -432,9 +434,11 @@ extension RunnerTests {
           outputPath: resolvedOutPath,
           fps: command.fps.map { Int32($0) }
         )
+        // The pump outlives this request, so the clip keeps the panel its `recordStart` named.
+        let clipLitPanel = requestLitPanel
         try startRecording(recorder) { [weak self] in
           guard let self else { return .failure(.unresolvedScreen) }
-          return self.captureRunnerFrameResult(app: activeApp)
+          return self.captureRunnerFrameResult(app: activeApp, litPanel: clipLitPanel)
         }
         activeRecording = recorder
         return Response(ok: true, data: DataPayload(message: "recording started"))