diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift index c71fde7267..dff4f67bb5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -21,16 +21,24 @@ enum RunnerAppScreenCaptureFailure: String, Error { case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED" case unresolvedScreen = "APP_SCREEN_UNRESOLVED" case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE" + case offLitPanel = "APP_SCREEN_OFF_LIT_PANEL" init(_ failure: RunnerApplicationScreenFailure) { switch failure { case .unresolvedWindow: self = .unresolvedWindow + case .offLitPanel: + self = .offLitPanel default: self = .unresolvedScreen } } + /// The refusal as a runner reply. + var response: Response { + Response(ok: false, error: ErrorPayload(code: rawValue, message: message, hint: hint)) + } + var message: String { switch self { case .unresolvedWindow: @@ -39,6 +47,8 @@ enum RunnerAppScreenCaptureFailure: String, Error { return "The resolved app window did not report the display it is shown on." case .unrenderableImage: return "The resolved display handed back an image that could not be drawn upright." + case .offLitPanel: + return "Every window of the target app is on the panel the host reports as dark." } } @@ -50,6 +60,8 @@ enum RunnerAppScreenCaptureFailure: String, Error { return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved." case .unrenderableImage: return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question." + case .offLitPanel: + return "Unfold or fold the device so the target app is on the lit panel, or bring it forward there; runner.log names the panel in AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW." } } } @@ -82,14 +94,17 @@ extension RunnerTests { /// /// Never activates `app`: an observation must not change which app is foregrounded just to get a /// screen, so the caller passes the app context it already intends to observe. + /// + /// `litPanel` is the panel the request named; nil keeps the app's first window. func captureResolvedAppScreen( - app: XCUIApplication + app: XCUIApplication, + litPanel: RunnerLitPanel? ) -> Result { var outcome: Result = .failure( .unresolvedScreen ) let capture = { - outcome = Self.resolveCapturedAppScreen(app: app) + outcome = Self.resolveCapturedAppScreen(app: app, litPanel: litPanel) } if Thread.isMainThread { capture() @@ -109,22 +124,33 @@ extension RunnerTests { /// Only an unresolved *window* moves on to the system surface. A window that resolved and then /// refused to name its display is the failure the host has to see, and no second question about a /// different process should bury it. - func captureObservedScreen(app: XCUIApplication) -> Result< + func captureObservedScreen(app: XCUIApplication, litPanel: RunnerLitPanel?) -> Result< CapturedAppScreen, RunnerAppScreenCaptureFailure > { selectObservedScreenCapture( - resolving: { captureResolvedAppScreen(app: app) }, - fallingBack: { captureResolvedAppScreen(app: springboard) } + resolving: { captureResolvedAppScreen(app: app, litPanel: litPanel) }, + fallingBack: { captureResolvedAppScreen(app: springboard, litPanel: litPanel) } ) } private static func resolveCapturedAppScreen( - app: XCUIApplication + app: XCUIApplication, + litPanel: RunnerLitPanel? ) -> Result { var screen: AnyObject? var displayID: UInt = 0 var resolution: RunnerApplicationScreenFailure = .unresolvedScreen - guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution), + var resolved: Bool + if let litPanel { + var window: AnyObject? + var frame = CGRect.null + resolution = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame, &screen) + resolved = resolution == .none + displayID = litPanel.displayID + } else { + resolved = RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution) + } + guard resolved, let captureScreen = screen as? XCUIScreen else { return .failure(RunnerAppScreenCaptureFailure(resolution)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift new file mode 100644 index 0000000000..fa17a6bd0b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -0,0 +1,131 @@ +import XCTest + +// The lit panel of the command in flight (ADR 0025), main-owned and written behind main work a +// previous command left queued, so that work still reads its own command's panel. +struct LitPanelBookingKey: Equatable { + let bundleID: String + let requestID: UUID +} + +struct LitPanelSurface { + let key: LitPanelBookingKey + let displayID: UInt + let window: XCUIElement + let frame: CGRect +} + +/// The command's lit panel and what it booked on it. Read and written only on the main thread. +final class RunnerLitPanelState { + var requestID: UUID? + var panel: RunnerLitPanel? + var surface: LitPanelSurface? +} + +extension RunnerTests { + @discardableResult + func setRequestLitPanel(from command: Command) -> UUID { + let requestID = UUID() + var panel: RunnerLitPanel? + if let fact = command.litPanel, fact.displayId != 0, fact.widthPt.isFinite, fact.heightPt.isFinite, + fact.widthPt > 0, fact.heightPt > 0 + { + panel = RunnerLitPanel(displayID: fact.displayId, widthPt: fact.widthPt, heightPt: fact.heightPt) + } + let state = litPanelState + let write = { + state.requestID = requestID + state.panel = panel + state.surface = nil + } + if Thread.isMainThread { + write() + } else { + DispatchQueue.main.async(execute: write) + } + return requestID + } + + func endRequestLitPanel(_ requestID: UUID) { + let state = litPanelState + let clear = { + guard state.requestID == requestID else { return } + state.requestID = nil + state.panel = nil + state.surface = nil + } + if Thread.isMainThread { + clear() + } else { + DispatchQueue.main.async(execute: clear) + } + } + + /// The lit panel the current request named, or nil for a single-panel request. Main only. + var requestLitPanel: RunnerLitPanel? { litPanelState.panel } + + /// Books `app`'s lit-panel window for the request, or returns why it cannot. + func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { + guard let litPanel = requestLitPanel else { return nil } + guard let requestID = litPanelState.requestID, + let bundleID = safely("LIT_PANEL_APP_IDENTITY", { app.value(forKey: "bundleID") as? String })?.trimmedNonEmpty + else { return .unresolvedScreen } + let key = LitPanelBookingKey(bundleID: bundleID, requestID: requestID) + if litPanelState.surface?.key == key { return nil } + litPanelState.surface = nil + var window: AnyObject? + var frame = CGRect.null + var screen: AnyObject? + let failure = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame, &screen) + guard failure == .none, let element = window as? XCUIElement else { + return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) + } + litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, frame: frame) + return nil + } + + func requireRequestLitPanelSurface(app: XCUIApplication) throws -> LitPanelSurface { + if let refusal = bookRequestLitPanelSurface(app: app) { throw refusal } + guard let surface = litPanelState.surface else { + throw RunnerAppScreenCaptureFailure.unresolvedScreen + } + return surface + } + + /// The roots of every element read: the app, or the windows proven on the lit display. A walk that + /// cannot read the list, or that records an issue (#2995), throws `APP_SCREEN_UNRESOLVED`. + func elementSearchRoots(root: XCUIElement, litDisplayID: UInt?) throws -> [XCUIElement] { + guard let litDisplayID else { return [root] } + var windows: [XCUIElement]? + let walk = holdingRecordedIssues { + windows = RunnerCopyLitPanelWindows(root, litDisplayID)?.compactMap { $0 as? XCUIElement } +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + if litPanelWindowWalkIssueForTesting { + record(XCTIssue(type: .assertionFailure, compactDescription: "Window vanished during the lit-panel walk")) + } +#endif + } + guard walk.issues.isEmpty, walk.exceptionMessage == nil, let windows, !windows.isEmpty else { + for issue in walk.issues { + NSLog("AGENT_DEVICE_RUNNER_LIT_PANEL_WALK_ISSUE=%@", issue.compactDescription) + } + NSLog( + "AGENT_DEVICE_RUNNER_LIT_PANEL_WALK_REFUSED issues=%ld exception=%@ windows=%ld", + walk.issues.count, walk.exceptionMessage ?? "none", windows?.count ?? -1 + ) + throw RunnerAppScreenCaptureFailure.unresolvedScreen + } + return windows + } + + func elementQueries(app: XCUIApplication, matching type: XCUIElement.ElementType) throws -> [XCUIElementQuery] { + try elementQueries(root: app, matching: type, litDisplayID: requestLitPanel?.displayID) + } + + func elementQueries( + root: XCUIElement, + matching type: XCUIElement.ElementType, + litDisplayID: UInt? + ) throws -> [XCUIElementQuery] { + try elementSearchRoots(root: root, litDisplayID: litDisplayID).map { $0.descendants(matching: type) } + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index d6921c5269..955f9389fd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -190,7 +190,7 @@ extension RunnerTests { func resolvedScrollViewport( app: XCUIApplication, context: SynthesizedCoordinateContext - ) -> RunnerScrollViewport { + ) throws -> RunnerScrollViewport { #if os(iOS) // Every scroll reports its decision, including the two ways it avoids reading the keyboard at // all: unavailable accessibility health, and a probe that finds no keyboard. @@ -223,7 +223,7 @@ extension RunnerTests { } return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: clip) #else - let fallbackFrame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + let fallbackFrame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) return ScrollViewportPolicy.frames(referenceFrame: fallbackFrame, clip: .unobstructed) #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index d212491567..ad05f5c884 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -290,7 +290,7 @@ extension RunnerTests { timeout: 1, timeoutError: snapshotMainThreadTimeoutError("reading private AX viewport") ) { - self.safeSnapshotViewport(app: app, readingOrientation: true) + try self.safeSnapshotViewport(app: app, readingOrientation: true) } if case .missing = reported { return fallback diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index f7dd07b92f..16d6ca5d77 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -95,7 +95,7 @@ extension RunnerTests { } @MainActor - func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) -> Response { + func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) throws -> Response { if action == "accept" || action == "dismiss" { guard let button = chooseAlertButton(alert, action: action) else { // The choice refused before any activation, so this discloses dispatched: no like the @@ -141,7 +141,7 @@ extension RunnerTests { // (#2546). The post-tap settle stays, because the verification below reads the alert this tap // replaces; an alert that dismisses and presents an identical replacement passes through a // window with no alert, and a first read landing there reports a dismissal nothing proved. - guard let outcome = activateAlertButton(alert, button: button, action: action, frame: buttonFrame, deadline: deadline) else { + guard let outcome = try activateAlertButton(alert, button: button, action: action, frame: buttonFrame, deadline: deadline) else { return alertVerificationResponse(.timedOut, action: action, activated: false) } if let response = unsupportedResponse(for: outcome) { @@ -190,8 +190,9 @@ extension RunnerTests { action: String, frame: CGRect, deadline: Date - ) -> RunnerInteractionOutcome? { + ) throws -> RunnerInteractionOutcome? { var outcome: RunnerInteractionOutcome? + var activationError: Error? withUIInterruptionHandlingDisabledIfSupported(alert.ownerApp) { withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) { guard Date() < deadline else { return } @@ -223,13 +224,20 @@ extension RunnerTests { } hostContext = context } - switch synthesizedTapAt( - app: alert.ownerApp, - x: frame.midX, - y: frame.midY, - context: hostContext, - deadline: deadline - ) { + let attempt: RunnerInteractionOutcome? + do { + attempt = try synthesizedTapAt( + app: alert.ownerApp, + x: frame.midX, + y: frame.midY, + context: hostContext, + deadline: deadline + ) + } catch { + activationError = error + return + } + switch attempt { case .performed?: typealias WaitForQuiescence = @convention(c) (NSObject, Selector, Bool) -> Void let waitForQuiescence = unsafeBitCast( @@ -247,15 +255,15 @@ extension RunnerTests { break } #else - outcome = activateElement( - app: alert.ownerApp, - element: button, - action: "alert \(action)", - resolvedFrame: frame - ) + do { + outcome = try activateElement(app: alert.ownerApp, element: button, action: "alert \(action)", resolvedFrame: frame) + } catch { + activationError = error + } #endif } } + if let activationError { throw activationError } return outcome } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index 7f77331073..c3edc1ab95 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -7,6 +7,8 @@ extension RunnerTests { func executeAccepted(command: Command) throws -> Response { commandJournal.start(command: command) pendingTargetActivation = nil + let panelRequestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(panelRequestID) } do { let response = try executeDispatched(command: command) commandJournal.finish(command: command, response: response) @@ -432,9 +434,11 @@ extension RunnerTests { outputPath: resolvedOutPath, fps: command.fps.map { Int32($0) } ) + // The pump outlives this request, so the clip keeps the panel its `recordStart` named. + let clipLitPanel = requestLitPanel try startRecording(recorder) { [weak self] in guard let self else { return .failure(.unresolvedScreen) } - return self.captureRunnerFrameResult(app: activeApp) + return self.captureRunnerFrameResult(app: activeApp, litPanel: clipLitPanel) } activeRecording = recorder return Response(ok: true, data: DataPayload(message: "recording started")) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 526d334a2e..4bc64ae5f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -2,18 +2,47 @@ import XCTest import AgentDeviceSnapshotPresentation extension RunnerTests { + private func refusedTapResponse(_ message: String, hint: String?) -> Response { + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) + } + + /// Runs `command` against its prepared app; a thrown lit-panel refusal answers it (ADR 0025). @MainActor func executeOnMainPrepared( command: Command, activeApp: XCUIApplication, activeAppBundleId: String? = nil, alertDeadline: Date? = nil + ) throws -> Response { + do { + return try executePrepared( + command: command, + activeApp: activeApp, + activeAppBundleId: activeAppBundleId, + alertDeadline: alertDeadline + ) + } catch let refusal as RunnerAppScreenCaptureFailure { + return refusal.response + } + } + + @MainActor + private func executePrepared( + command: Command, + activeApp: XCUIApplication, + activeAppBundleId: String?, + alertDeadline: Date? ) throws -> Response { var activeApp = activeApp mainOwned.soleDisplayPortraitSize = nil if command.invalidatesRememberedTextEntryTap { clearRememberedTextEntryTap() } + // A screenshot resolves its own surface, the target app and then the system surface. + if command.command != .screenshot, let refusal = bookRequestLitPanelSurface(app: activeApp) { + return refusal.response + } switch command.command { case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime, .appState, .pasteboardWrite, .snapshot: @@ -35,7 +64,7 @@ extension RunnerTests { } else { expectedPoint = nil } - let match = findElement( + let match = try findElement( app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue, @@ -57,7 +86,7 @@ extension RunnerTests { // negative x), so it happily "contains" unreachable coordinates. // The DECISION lives in TapPointPolicy (golden parity table with the // TS twin); onScreenWindowFrame only supplies the frame. - if !match.usedNonHittableFallback + if try !match.usedNonHittableFallback && !TapPointPolicy.isAllowed( elementFrame: frame, windowFrame: onScreenWindowFrame(app: activeApp) @@ -69,26 +98,17 @@ extension RunnerTests { } let isTextEntry = isTextEntryElement(element) let touchPoint = expectedPoint ?? CGPoint(x: frame.midX, y: frame.midY) - let touchFrame = resolvedTouchVisualizationFrame( + let touchFrame = try resolvedTouchVisualizationFrame( app: activeApp, x: touchPoint.x, y: touchPoint.y ) var fallback: GestureFallback? if command.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.coordinateTap - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedTapAt( - app: activeApp, - x: touchPoint.x, - y: touchPoint.y, - context: context - ) - }) { + switch try performSynthesizedTapGesture(app: activeApp, x: touchPoint.x, y: touchPoint.y) { case .performed(let timing): if isTextEntry { - waitForTextEntryReadinessAfterTap(app: activeApp, element: element) + try waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } rememberTextEntryTap(isTextEntry ? element : nil) return gestureResponse( @@ -103,28 +123,27 @@ extension RunnerTests { : nil ) case .xctestFallback(let message, let hint): - fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + fallback = .coordinateTap(message, hint: hint) case .refused(_, let message, let hint): - clearRememberedTextEntryTap() - return unsupportedResponse(message: message, hint: hint) + return refusedTapResponse(message, hint: hint) } } - let (timing, outcome) = performGesture(activeApp) { + let (timing, outcome) = try performGesture(activeApp) { if expectedPoint != nil || match.usedNonHittableFallback { // Maestro compatibility: RN E2E backdoor controls can be 1x1 and // reported non-hittable by XCTest. Snapshot-resolved targets also // need coordinate delivery because XCUIElement.activate() does not // invoke every React Native accessibility wrapper. - return tapAt(app: activeApp, x: touchPoint.x, y: touchPoint.y) + return try tapAt(app: activeApp, x: touchPoint.x, y: touchPoint.y) } - return activateElement(app: activeApp, element: element, action: "tap by selector") + return try activateElement(app: activeApp, element: element, action: "tap by selector") } if let response = unsupportedResponse(for: outcome) { clearRememberedTextEntryTap() return response } if isTextEntry { - waitForTextEntryReadinessAfterTap(app: activeApp, element: element) + try waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } rememberTextEntryTap(isTextEntry ? element : nil) return gestureResponse( @@ -153,7 +172,7 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y) + textInput = try coordinateTapTextInputAt(app: activeApp, x: x, y: y) } else { textInput = nil // Clear before dispatch so a throwing gesture cannot retain the preceding field's witness. @@ -166,23 +185,18 @@ extension RunnerTests { } var fallback: GestureFallback? if command.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.coordinateTap - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - }) { + switch try performSynthesizedTapGesture(app: activeApp, x: x, y: y) { case .performed(let timing): rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) case .xctestFallback(let message, let hint): - fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + fallback = .coordinateTap(message, hint: hint) case .refused(_, let message, let hint): - clearRememberedTextEntryTap() - return unsupportedResponse(message: message, hint: hint) + return refusedTapResponse(message, hint: hint) } } - let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) - let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) } + let touchFrame = try resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) + let (timing, outcome) = try performGesture(activeApp) { try tapAt(app: activeApp, x: x, y: y) } if let response = unsupportedResponse(for: outcome) { clearRememberedTextEntryTap() return response @@ -201,7 +215,7 @@ extension RunnerTests { guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "mouseClick requires x and y")) } - let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) + let touchFrame = try resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) do { // mouseClick throws (it has no RunnerInteractionOutcome), so it keeps raw measureGesture // and only routes the success payload through gestureResponse. @@ -228,9 +242,9 @@ extension RunnerTests { let fallback: GestureFallback? #if os(iOS) let policyKind = SynthesizedGesturePolicyKind.coordinateLongPress - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) + let context = try synthesizedCoordinateContext(app: activeApp) + switch try performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { + try synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) }) { case .performed(let timing): let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) } @@ -243,9 +257,9 @@ extension RunnerTests { #else fallback = nil #endif - let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) - let (timing, outcome) = performGesture(activeApp) { - longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0) + let touchFrame = try resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) + let (timing, outcome) = try performGesture(activeApp) { + try longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0) } if let response = unsupportedResponse(for: outcome) { return response @@ -261,7 +275,7 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2")) } let defaults = runnerDragCommandDefaults(command) - return executeDragGesture( + return try executeDragGesture( activeApp: activeApp, x: x, y: y, @@ -279,13 +293,13 @@ extension RunnerTests { else { return invalidScrollDirectionResponse(commandName: "scroll") } - guard let scrollContext = synthesizedCoordinateContext(app: activeApp) else { + guard let scrollContext = try synthesizedCoordinateContext(app: activeApp) else { return Response( ok: false, error: ErrorPayload(message: "scroll could not resolve a usable interaction frame") ) } - let viewport = resolvedScrollViewport(app: activeApp, context: scrollContext) + let viewport = try resolvedScrollViewport(app: activeApp, context: scrollContext) let defaults = runnerDragCommandDefaults(command) switch viewport.gestureDispatch( direction: direction, @@ -316,7 +330,7 @@ extension RunnerTests { return invalidScrollDurationResponse(commandName: "scroll") } return gesture.attachingEvidence( - to: executeScrollDragGesture( + to: try executeScrollDragGesture( activeApp: activeApp, x: gesture.planFrame.minX + gesture.plan.x1, y: gesture.planFrame.minY + gesture.plan.y1, @@ -336,7 +350,7 @@ extension RunnerTests { return invalidScrollDirectionResponse(commandName: "desktopScroll") } let appFrame = activeApp.frame - let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: appFrame) + let frame = try resolvedTouchReferenceFrame(app: activeApp, appFrame: appFrame) guard frame.width > 0, frame.height > 0 else { return Response( ok: false, @@ -365,7 +379,7 @@ extension RunnerTests { guard scrollDurationIsValid(command.durationMs) else { return invalidScrollDurationResponse(commandName: "desktopScroll") } - let touchFrame = resolvedTouchVisualizationFrame( + let touchFrame = try resolvedTouchVisualizationFrame( app: activeApp, x: localX, y: localY @@ -406,24 +420,24 @@ extension RunnerTests { } return Response(ok: true, data: DataPayload(message: "remote pressed")) case .type: - var response: Response? + var outcome: Result? withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { - response = executeTypeCommand(activeApp: activeApp, command: command) + outcome = Result { try executeTypeCommand(activeApp: activeApp, command: command) } } - return response ?? Response(ok: false, error: ErrorPayload(message: "type produced no response")) + return try outcome?.get() ?? Response(ok: false, error: ErrorPayload(message: "type produced no response")) case .swipe: guard let direction = command.direction else { return Response(ok: false, error: ErrorPayload(message: "swipe requires direction")) } // swipe returns an optional frame (tvOS-only) rather than a RunnerInteractionOutcome, so it // keeps raw measureGesture and only routes the success payload through gestureResponse. - var executedFrame: DragVisualizationFrame? + var executedFrame: Result = .success(nil) let timing = measureGesture { withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { - executedFrame = swipe(app: activeApp, direction: direction) + executedFrame = Result { try swipe(app: activeApp, direction: direction) } } } - guard let dragFrame = executedFrame else { + guard let dragFrame = try executedFrame.get() else { return Response(ok: false, error: ErrorPayload(message: "swipe is only supported on tvOS")) } return gestureResponse(message: "swiped", timing: timing, frame: .drag(dragFrame)) @@ -431,18 +445,18 @@ extension RunnerTests { guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "findText requires text")) } - let found = findElement(app: activeApp, text: text) != nil + let found = try findElement(app: activeApp, text: text) != nil return Response(ok: true, data: DataPayload(found: found)) case .querySelector: guard let selectorKey = command.selectorKey, let selectorValue = command.selectorValue else { return Response(ok: false, error: ErrorPayload(message: "querySelector requires selectorKey and selectorValue")) } - return queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) + return try queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) case .readText: guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "readText requires x and y")) } - guard let text = readTextAt(app: activeApp, x: x, y: y) else { + guard let text = try readTextAt(app: activeApp, x: x, y: y) else { return Response(ok: false, error: ErrorPayload(message: "readText did not resolve text")) } return Response(ok: true, data: DataPayload(text: text)) @@ -487,7 +501,10 @@ extension RunnerTests { // session app, so on a fresh runner process `activeApp` is still the runner host app. The // requested bundle id is resolved here and never activated: an observation must not change // which app is foregrounded in order to see which display it is on (#2728). - switch captureObservedScreen(app: resolveAppWithoutActivation(command: command)) { + switch captureObservedScreen( + app: resolveAppWithoutActivation(command: command), + litPanel: requestLitPanel + ) { case .failure(let failure): return Response( ok: false, @@ -521,7 +538,7 @@ extension RunnerTests { ) #endif case .backInApp: - switch tapInAppBackControl(app: activeApp) { + switch try tapInAppBackControl(app: activeApp) { case .performed: return Response(ok: true, data: DataPayload(message: "backInApp")) case .unavailable: @@ -535,7 +552,7 @@ extension RunnerTests { return Response(ok: false, error: error) } case .backSystem: - if performSystemBackAction(app: activeApp) { + if try performSystemBackAction(app: activeApp) { return Response(ok: true, data: DataPayload(message: "backSystem")) } return Response(ok: false, error: ErrorPayload(message: "system back is not available")) @@ -545,7 +562,7 @@ extension RunnerTests { case .rotate: return executeRotateCommand(command) case .appSwitcher: - performAppSwitcherGesture(app: activeApp) + try performAppSwitcherGesture(app: activeApp) return Response(ok: true, data: DataPayload(message: "appSwitcher")) case .actionButton: guard pressActionButton() else { @@ -620,7 +637,7 @@ extension RunnerTests { ) switch resolveAlert(app: activeApp, deadline: deadline) { case .resolved(let alert): - return handleAlert(alert, action: action, deadline: deadline) + return try handleAlert(alert, action: action, deadline: deadline) case .absent: // Typed so the host retries on absence alone: a transport or runner failure carries no // code and must not be mistaken for "no alert yet" (ALERT_NOT_FOUND_RUNNER_CODE). @@ -657,7 +674,7 @@ extension RunnerTests { let first = plan.pointers[0].samples.first!.point let last = plan.pointers[0].samples.last!.point return canonicalPlannedGestureResponse( - executeDragGesture( + try executeDragGesture( activeApp: activeApp, x: first.x, y: first.y, @@ -669,19 +686,19 @@ extension RunnerTests { ) ) case .sampled: - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) { sampledPlannedGesture(app: activeApp, plan: plan) } return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) } case .gestureViewport: - let frame = gestureViewportFrame(app: activeApp) + let frame = try gestureViewportFrame(app: activeApp) guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else { return Response(ok: false, error: ErrorPayload(code: "COMMAND_FAILED", message: "Active app interaction viewport is unavailable")) } return Response(ok: true, data: DataPayload(message: "gestureViewport", x: frame.minX, y: frame.minY, x2: frame.width, y2: frame.height)) case .sequence: - return executeSequence(command: command, activeApp: activeApp) + return try executeSequence(command: command, activeApp: activeApp) } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift index 2d9e4b2395..b148a21c88 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift @@ -6,9 +6,9 @@ extension RunnerTests { ProcessInfo.processInfo.systemUptime * 1000 } - func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) { + func measureGesture(_ action: () throws -> Void) rethrows -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) { let gestureStartUptimeMs = currentUptimeMs() - action() + try action() return (gestureStartUptimeMs, currentUptimeMs()) } @@ -47,6 +47,10 @@ extension RunnerTests { let strategy: String let message: String let hint: String? + + static func coordinateTap(_ message: String, hint: String?) -> Self { + Self(strategy: "xctest-coordinate-tap", message: message, hint: hint) + } } @@ -61,19 +65,19 @@ extension RunnerTests { func performGesture( _ app: XCUIApplication, idleTimeout: Bool = true, - _ action: () -> RunnerInteractionOutcome - ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) { - var outcome = RunnerInteractionOutcome.performed + _ action: () throws -> RunnerInteractionOutcome + ) throws -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) { + var outcome: Result = .success(.performed) let timing = measureGesture { if idleTimeout { withBoundedInteractionIdleTimeoutIfSupported(app, waits: .bothSkipped) { - outcome = action() + outcome = Result { try action() } } } else { - outcome = action() + outcome = Result { try action() } } } - return (timing, outcome) + return (timing, try outcome.get()) } /// Single factory for the success payload every gesture returns (message + gesture timing + diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 3684adfee4..dd469d9c6a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -66,46 +66,46 @@ extension RunnerTests { let usedNonHittableFallback: Bool } - func performBackGesture(app: XCUIApplication) { + func performBackGesture(app: XCUIApplication) throws { if pressTvRemote(.menu) { return } - performCoordinateBackGesture(app: app) + try performCoordinateBackGesture(app: app) } - private func performCoordinateBackGesture(app: XCUIApplication) { + private func performCoordinateBackGesture(app: XCUIApplication) throws { #if !os(tvOS) - let target = resolveRunnerWindow(app: app).window ?? app + let target = try resolveRunnerWindow(app: app).window ?? app let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5)) let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5)) start.press(forDuration: 0.05, thenDragTo: end) #endif } - func performSystemBackAction(app: XCUIApplication) -> Bool { + func performSystemBackAction(app: XCUIApplication) throws -> Bool { #if os(macOS) return false #else if pressTvRemote(.menu) { return true } - performBackGesture(app: app) + try performBackGesture(app: app) return true #endif } - func performAppSwitcherGesture(app: XCUIApplication) { + func performAppSwitcherGesture(app: XCUIApplication) throws { if pressTvRemote(.home) { sleepFor(resolveTvRemoteDoublePressDelay()) _ = pressTvRemote(.home) return } - performCoordinateAppSwitcherGesture(app: app) + try performCoordinateAppSwitcherGesture(app: app) } - private func performCoordinateAppSwitcherGesture(app: XCUIApplication) { + private func performCoordinateAppSwitcherGesture(app: XCUIApplication) throws { #if !os(tvOS) - let target = resolveRunnerWindow(app: app).window ?? app + let target = try resolveRunnerWindow(app: app).window ?? app let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99)) let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7)) start.press(forDuration: 0.6, thenDragTo: end) @@ -166,10 +166,12 @@ extension RunnerTests { #endif } - func findElement(app: XCUIApplication, text: String) -> XCUIElement? { + func findElement(app: XCUIApplication, text: String) throws -> XCUIElement? { let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text) - let element = app.descendants(matching: .any).matching(predicate).firstMatch - return element.exists ? element : nil + return try elementQueries(app: app, matching: .any) + .lazy + .map { $0.matching(predicate).firstMatch } + .first { $0.exists } } func findElement( @@ -179,7 +181,7 @@ extension RunnerTests { allowNonHittableFallback: Bool = false, expectedPoint: CGPoint? = nil, rawMatchPolicy: DirectSelectorRawMatchPolicy = .rejectDistinctMatches - ) -> SelectorElementMatch { + ) throws -> SelectorElementMatch { let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines) guard !value.isEmpty else { return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) @@ -198,13 +200,17 @@ extension RunnerTests { return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } - let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex + let matches = try elementQueries(app: app, matching: .any) + .flatMap { $0.matching(predicate).allElementsBoundByIndex } .filter(\.exists) - let facts = matches.map { element in - SelectorCandidateFacts( - isHittable: element.isHittable, - hasTappableFrame: hasTappableFrame(app: app, element: element), - containsExpectedPoint: expectedPoint.map(element.frame.contains) ?? true + let facts = try matches.map { element in + let isHittable = element.isHittable + let containsExpectedPoint = expectedPoint.map(element.frame.contains) ?? true + return SelectorCandidateFacts( + isHittable: isHittable, + hasTappableFrame: allowNonHittableFallback && !isHittable && containsExpectedPoint + ? try hasTappableFrame(app: app, element: element) : false, + containsExpectedPoint: containsExpectedPoint ) } switch classifyDirectSelectorCandidates( @@ -230,13 +236,16 @@ extension RunnerTests { // with no frame at all cannot be coordinate-tapped, otherwise the decision // is the shared TapPointPolicy center-in-frame rule (golden parity table // with the TS twin). app.frame is the frame source here — replay taps - // resolved bounds Maestro-style, so the union frame is intentional. - private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { + // resolved bounds Maestro-style, so the union frame is intentional. A + // lit-panel request uses its booked window, because app.frame names one + // panel (ADR 0025). + private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) throws -> Bool { + let windowFrame = try requestLitPanel == nil ? app.frame : requireRequestLitPanelSurface(app: app).frame let frame = element.frame if frame.isEmpty { return false } - return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: app.frame) + return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: windowFrame) } // The tappable on-screen viewport. app.frame is unsuitable: it unions @@ -258,10 +267,12 @@ extension RunnerTests { /// The read that remains is not atomic: a window vanishing while it is the one being resolved still /// records, and the chosen window is read again on its way to a display ID /// (`RunnerResolveWindowDisplayID`). - func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) { - guard let resolved = resolveRunnerWindowElement(app: app) else { - return (nil, app.frame) + func resolveRunnerWindow(app: XCUIApplication) throws -> (window: XCUIElement?, frame: CGRect) { + if requestLitPanel != nil { + let booked = try requireRequestLitPanelSurface(app: app) + return (booked.window, booked.frame) } + guard let resolved = resolveRunnerWindowElement(app: app) else { return (nil, app.frame) } return (resolved.window, resolved.frame) } @@ -269,6 +280,10 @@ extension RunnerTests { /// back to `app.frame`: when the windows query fails because XCTest cannot serialize the app's /// tree, that read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { + if requestLitPanel != nil { + guard bookRequestLitPanelSurface(app: app) == nil, let booked = litPanelState.surface else { return nil } + return (booked.window, booked.frame) + } for window in app.windows.allElementsBoundByIndex where window.exists { let frame = window.frame if !frame.isEmpty { @@ -278,17 +293,17 @@ extension RunnerTests { return nil } - func onScreenWindowFrame(app: XCUIApplication) -> CGRect { - resolveRunnerWindow(app: app).frame + func onScreenWindowFrame(app: XCUIApplication) throws -> CGRect { + try resolveRunnerWindow(app: app).frame } - func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { + func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response { // querySelector is a read — it backs get/is/wait and the offscreen-refusal // double-check, none of which mutate. The fail-closed raw-match rule exists // to stop a mutation acting on an unseen duplicate; applying it here would // instead turn a decorative non-hittable duplicate into an AMBIGUOUS_MATCH // for readers that previously resolved the hittable element. - let match = findElement( + let match = try findElement( app: app, selectorKey: selectorKey, selectorValue: selectorValue, @@ -350,9 +365,9 @@ extension RunnerTests { return left.elementType.rawValue < right.elementType.rawValue } - func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { + func readTextAt(app: XCUIApplication, x: Double, y: Double) throws -> String? { let point = CGPoint(x: x, y: y) - let textInputCandidates = textInputCandidatesAt(app: app, point: point, contentTranslation: liveContentTranslation(app: app)) + let textInputCandidates = try textInputCandidatesAt(app: app, point: point, contentTranslation: liveContentTranslation(app: app)) .map(\.element) for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { @@ -360,7 +375,8 @@ extension RunnerTests { } } - let candidates = app.descendants(matching: .any).allElementsBoundByIndex + let candidates = try elementQueries(app: app, matching: .any) + .flatMap { $0.allElementsBoundByIndex } .filter { element in element.exists && !element.frame.isEmpty && element.frame.contains(point) } @@ -405,16 +421,16 @@ extension RunnerTests { } } - func tapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + func tapAt(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "tap") { return outcome } - return performCoordinateTap(app: app, x: x, y: y) + return try performCoordinateTap(app: app, x: x, y: y) } func mouseClickAt(app: XCUIApplication, x: Double, y: Double, button: String) throws { #if os(macOS) - let coordinate = interactionCoordinate(app: app, x: x, y: y) + let coordinate = try interactionCoordinate(app: app, x: x, y: y) switch button { case "primary": coordinate.tap() @@ -447,7 +463,7 @@ extension RunnerTests { durationMs: durationMs ) - let coordinate = interactionCoordinate(app: app, x: x, y: y) + let coordinate = try interactionCoordinate(app: app, x: x, y: y) let interval = desktopScrollEventIntervalSeconds(durationMs: durationMs, eventCount: events.count) for (index, deltas) in events.enumerated() { // Keep desktop scrolling on XCTest's coordinate API so macOS owns wheel synthesis, natural @@ -521,21 +537,21 @@ extension RunnerTests { } } - func doubleTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + func doubleTapAt(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "double tap") { guard case .performed = outcome else { return outcome } sleepFor(0.1) _ = pressTvRemote(.select) return .performed } - return performCoordinateDoubleTap(app: app, x: x, y: y) + return try performCoordinateDoubleTap(app: app, x: x, y: y) } - func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome { + func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) throws -> RunnerInteractionOutcome { if let outcome = longSelectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), duration: duration) { return outcome } - return performCoordinateLongPress(app: app, x: x, y: y, duration: duration) + return try performCoordinateLongPress(app: app, x: x, y: y, duration: duration) } func dragAt( @@ -545,7 +561,7 @@ extension RunnerTests { x2: Double, y2: Double, holdDuration: TimeInterval - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { // tvOS has no coordinate drag. Preserve the direction as a focus move. let dx = x2 - x let dy = y2 - y @@ -555,41 +571,41 @@ extension RunnerTests { if pressTvRemote(button) { return .performed } - return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration) + return try performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration) } - private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." ) #else - interactionCoordinate(app: app, x: x, y: y).tap() + try interactionCoordinate(app: app, x: x, y: y).tap() return .performed #endif } - private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." ) #else - interactionCoordinate(app: app, x: x, y: y).doubleTap() + try interactionCoordinate(app: app, x: x, y: y).doubleTap() return .performed #endif } - private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome { + private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element", hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then long-select it." ) #else - interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration) + try interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration) return .performed #endif } @@ -601,22 +617,22 @@ extension RunnerTests { x2: Double, y2: Double, holdDuration: TimeInterval - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate drag is not supported on tvOS", hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead." ) #else - let start = interactionCoordinate(app: app, x: x, y: y) - let end = interactionCoordinate(app: app, x: x2, y: y2) + let start = try interactionCoordinate(app: app, x: x, y: y) + let end = try interactionCoordinate(app: app, x: x2, y: y2) start.press(forDuration: holdDuration, thenDragTo: end) return .performed #endif } #if !os(tvOS) - private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { + private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) throws -> XCUICoordinate { #if os(iOS) // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the // resolved window: callers pass screen points, and `app.frame` starts at the screen origin even @@ -624,11 +640,17 @@ extension RunnerTests { // window is not always the one under the finger — a SpringBoard `alert accept` can live in an // alert window while the wallpaper or status-bar window qualifies first, so a window-relative // anchor misses the button. Reference frames and the synthesized display ID still resolve - // through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display. - let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) - return origin.withOffset(CGVector(dx: x, dy: y)) + // through `resolveRunnerWindow`. A foldable anchors at the lit panel's window instead, because + // the app origin names the outer panel whichever one is lit. + if requestLitPanel == nil { + let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) + return origin.withOffset(CGVector(dx: x, dy: y)) + } + let booked = try requireRequestLitPanelSurface(app: app) + let origin = booked.window.coordinate(withNormalizedOffset: .zero) + return origin.withOffset(CGVector(dx: x - booked.frame.minX, dy: y - booked.frame.minY)) #else - let resolved = resolveRunnerWindow(app: app) + let resolved = try resolveRunnerWindow(app: app) let root = resolved.window ?? app let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) let offsetX = x - Double(resolved.frame.origin.x) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index f6e4d6d9bb..7d08d239fe 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -52,7 +52,7 @@ extension RunnerTests { /// session window falls to the system surface's window, which is what the home screen is. macOS /// keeps the host display it always recorded. func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? { - switch captureRunnerFrameResult(app: app) { + switch captureRunnerFrameResult(app: app, litPanel: requestLitPanel) { case .success(let captured): return captured.image case .failure: @@ -65,10 +65,11 @@ extension RunnerTests { /// typed code rather than a message. The ongoing pump reads the same result and ignores a refusal /// the way it ignored the `nil` it used to get; only a frame that must exist owes a reason (#2728). func captureRunnerFrameResult( - app: XCUIApplication + app: XCUIApplication, + litPanel: RunnerLitPanel? ) -> Result { #if os(iOS) - return captureObservedScreen(app: app) + return captureObservedScreen(app: app, litPanel: litPanel) #else var outcome: Result = .failure( .unrenderableImage diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index bd58bed816..9cb03978e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -230,6 +230,13 @@ struct CoordinateTapTextInputEvidence: Codable { let textInputRects: [RunnerGestureViewport] } +/// The wire spelling of the lit panel the host names for a multi-panel device (ADR 0025). +struct RunnerLitPanelFact: Codable { + let displayId: UInt + let widthPt: Double + let heightPt: Double +} + struct Command: Codable { let command: CommandType let commandId: String? @@ -269,6 +276,7 @@ struct Command: Codable { let inlineScreenshot: Bool? let synthesized: Bool? let coordinateTapTextInputEvidence: CoordinateTapTextInputEvidence? + let litPanel: RunnerLitPanelFact? let steps: [SequenceStep]? } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 4bb91cab25..f35f0d16f6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -34,10 +34,10 @@ extension RunnerTests { } @MainActor - func tapInAppBackControl(app: XCUIApplication) -> InAppBackOutcome { + func tapInAppBackControl(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) if let back = macOSNavigationBackElement(app: app) { - tapElementCenter(app: app, element: back) + try tapElementCenter(app: app, element: back) return .performed } return .unavailable @@ -46,7 +46,8 @@ extension RunnerTests { return .performed #else let rejectedQueriesBefore = navigationRejectedQueryCount() - let buttons = app.navigationBars.buttons.allElementsBoundByIndex + let buttons = try elementQueries(app: app, matching: .navigationBar) + .flatMap { $0.buttons.allElementsBoundByIndex } if let back = buttons.first(where: { $0.isHittable }) { // `isHittable` promises a hittable point, not a hittable center: a bar button partly under an // overlay keeps the element tap, which aims at the point XCTest computed. @@ -57,20 +58,20 @@ extension RunnerTests { || isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) { NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", mainOwned.bundleId ?? "") - } else if let frame = topNavigationBackFrame(app: app) { - return tapResolvedNavigationBackControl(app: app, frame: frame) + } else if let frame = try topNavigationBackFrame(app: app) { + return try tapResolvedNavigationBackControl(app: app, frame: frame) } - return tapTopLeadingNavigationFallback( + return try tapTopLeadingNavigationFallback( app: app, windowQueryRejected: navigationRejectedQueryCount() != rejectedQueriesBefore ) #endif } - private func tapElementCenter(app: XCUIApplication, element: XCUIElement) { + private func tapElementCenter(app: XCUIApplication, element: XCUIElement) throws { let frame = element.frame if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = try tapAt(app: app, x: frame.midX, y: frame.midY) return } #if !os(tvOS) @@ -79,10 +80,10 @@ extension RunnerTests { } @MainActor - private func tapResolvedNavigationBackControl(app: XCUIApplication, frame: CGRect) -> InAppBackOutcome { + private func tapResolvedNavigationBackControl(app: XCUIApplication, frame: CGRect) throws -> InAppBackOutcome { guard SnapshotGeometry.isPositiveFinite(frame) else { return .unavailable } - let context = synthesizedCoordinateContext(app: app) - guard dispatchNavigationTap(app: app, point: CGPoint(x: frame.midX, y: frame.midY), context: context) else { + let context = try synthesizedCoordinateContext(app: app) + guard try dispatchNavigationTap(app: app, point: CGPoint(x: frame.midX, y: frame.midY), context: context) else { return .unavailable } return .performed @@ -95,27 +96,29 @@ extension RunnerTests { app: XCUIApplication, point: CGPoint, context: SynthesizedCoordinateContext? - ) -> Bool { - switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { - synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) + ) throws -> Bool { + switch try performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { + try synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) }) { case .performed: return true case .refused: return false case .xctestFallback: - let fallback = performGesture(app) { - tapAt(app: app, x: point.x, y: point.y) + let fallback = try performGesture(app) { + try tapAt(app: app, x: point.x, y: point.y) } if case .performed = fallback.outcome { return true } return false } } - private func topNavigationBackFrame(app: XCUIApplication) -> CGRect? { + private func topNavigationBackFrame(app: XCUIApplication) throws -> CGRect? { #if os(iOS) - let window = onScreenWindowFrame(app: app) - let candidates = app.buttons.matching(Self.navigationBackPredicate()).allElementsBoundByIndex.compactMap { + let window = try onScreenWindowFrame(app: app) + let candidates = try elementQueries(app: app, matching: .button).flatMap { + $0.matching(Self.navigationBackPredicate()).allElementsBoundByIndex + }.compactMap { element -> (frame: CGRect, rank: Int)? in guard element.exists, element.isHittable else { return nil } let frame = element.frame @@ -219,9 +222,9 @@ extension RunnerTests { private func tapTopLeadingNavigationFallback( app: XCUIApplication, windowQueryRejected: Bool - ) -> InAppBackOutcome { + ) throws -> InAppBackOutcome { #if os(iOS) - guard let context = synthesizedCoordinateContext( + guard let context = try synthesizedCoordinateContext( app: app, windowQueryRejected: windowQueryRejected ), let point = Self.topLeadingNavigationFallbackPoint( @@ -231,7 +234,7 @@ extension RunnerTests { return .unavailable } let before = captureNavigationFallbackVisualState(app: app, context: context) - guard dispatchNavigationTap(app: app, point: point, context: context) else { + guard try dispatchNavigationTap(app: app, point: point, context: context) else { return .unavailable } return verifyNavigationFallbackOutcome(app: app, before: before, context: context) @@ -255,9 +258,10 @@ extension RunnerTests { ) -> Result { #if os(iOS) return Self.navigationVisualSample( - from: context.resolvedWindow == nil + // A lit-panel request never takes the sole-screen route: a foldable has no sole screen. + from: context.resolvedWindow == nil && requestLitPanel == nil ? captureSoleAppScreen(app: app) - : captureResolvedAppScreen(app: app), + : captureResolvedAppScreen(app: app, litPanel: requestLitPanel), encoding: { runnerPngData(for: $0.image) } ) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift index 340e3278bf..4b1ae239cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -38,9 +38,9 @@ extension RunnerTests { message: String, context: SynthesizedCoordinateContext, releaseBehavior: ScrollReleaseBehavior? - ) -> Response { + ) throws -> Response { #if os(iOS) - return executeDragGesture( + return try executeDragGesture( activeApp: activeApp, x: x, y: y, @@ -52,7 +52,7 @@ extension RunnerTests { synthesized: (profile: scrollDragProfile(releaseBehavior: releaseBehavior), policyKind: .scroll) ) #else - return executeDragGesture( + return try executeDragGesture( activeApp: activeApp, x: x, y: y, @@ -77,7 +77,7 @@ extension RunnerTests { message: String, synthesizedContext: SynthesizedCoordinateContext? = nil, synthesized: (profile: SynthesizedDragProfile, policyKind: SynthesizedGesturePolicyKind)? = nil - ) -> Response { + ) throws -> Response { let durationMs = durationMs ?? runnerDefaultDragDurationMs let commandName = dragCommandName(message: message) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { @@ -86,7 +86,7 @@ extension RunnerTests { error: ErrorPayload(code: "INVALID_ARGS", message: "\(commandName) requires finite coordinates") ) } - if let synthesized, let synthesizedResponse = executeSynthesizedDragGesture( + if let synthesized, let synthesizedResponse = try executeSynthesizedDragGesture( activeApp: activeApp, x: x, y: y, @@ -100,7 +100,7 @@ extension RunnerTests { ) { return synthesizedResponse } - return executeCoordinateDrag( + return try executeCoordinateDrag( activeApp: activeApp, x: x, y: y, @@ -124,17 +124,17 @@ extension RunnerTests { holdDuration: TimeInterval, message: String, fallback: GestureFallback? = nil - ) -> Response { - let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) - let dragFrame = resolvedDragVisualizationFrame( + ) throws -> Response { + let dragPoints = try keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) + let dragFrame = try resolvedDragVisualizationFrame( app: activeApp, x: dragPoints.x, y: dragPoints.y, x2: dragPoints.x2, y2: dragPoints.y2 ) - let (timing, outcome) = performGesture(activeApp) { - dragAt( + let (timing, outcome) = try performGesture(activeApp) { + try dragAt( app: activeApp, x: dragPoints.x, y: dragPoints.y, @@ -161,10 +161,10 @@ extension RunnerTests { context: SynthesizedCoordinateContext?, policyKind: SynthesizedGesturePolicyKind, profile: SynthesizedDragProfile - ) -> Response? { + ) throws -> Response? { #if os(iOS) - let context = context ?? synthesizedCoordinateContext(app: activeApp) - guard let plan = axFreeSynthesizedDragPlan( + let context = try context ?? synthesizedCoordinateContext(app: activeApp) + guard let plan = try axFreeSynthesizedDragPlan( app: activeApp, x: x, y: y, @@ -191,8 +191,8 @@ extension RunnerTests { referenceWidth: Double(plan.referenceFrame.width), referenceHeight: Double(plan.referenceFrame.height) ) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { - synthesizedDragAt( + switch try performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { + try synthesizedDragAt( app: activeApp, x: plan.points.x, y: plan.points.y, @@ -206,7 +206,7 @@ extension RunnerTests { case .performed(let timing): return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) case .xctestFallback(let fallbackMessage, let hint): - return executeCoordinateDrag( + return try executeCoordinateDrag( activeApp: activeApp, x: plan.points.x, y: plan.points.y, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 586cdc038c..11b714522d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -20,7 +20,7 @@ extension RunnerTests { } @MainActor - func executeSequence(command: Command, activeApp: XCUIApplication) -> Response { + func executeSequence(command: Command, activeApp: XCUIApplication) throws -> Response { guard let steps = command.steps, !steps.isEmpty else { return sequenceInvalidArgs("sequence requires at least one step") } @@ -35,18 +35,18 @@ extension RunnerTests { } } - let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) + let synthesizedContext = try synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) // Touch frame resolves from the first step's coords so recording-gestures works unchanged. A // synthesized context already names the frame, so the app tree is not read for it again. let firstStep = steps[0] - let firstFrame = (firstStep.x != nil && firstStep.y != nil) + let firstFrame = try (firstStep.x != nil && firstStep.y != nil) ? synthesizedContext.map { touchVisualizationFrame(x: firstStep.x!, y: firstStep.y!, context: $0) } ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil - let execution = assembleSequenceExecution(steps: steps) { _, step in - performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) + let execution = try assembleSequenceExecution(steps: steps) { _, step in + try performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) } return sequenceResponse(execution: execution, touchFrame: firstFrame) } @@ -58,8 +58,8 @@ extension RunnerTests { /// results.count == completedSteps + (failedStepIndex != nil ? 1 : 0). func assembleSequenceExecution( steps: [SequenceStep], - perform: (Int, SequenceStep) -> SequenceStepOutcome - ) -> SequenceExecutionResult { + perform: (Int, SequenceStep) throws -> SequenceStepOutcome + ) rethrows -> SequenceExecutionResult { var results: [SequenceStepResult] = [] var completedSteps = 0 var failedStepIndex: Int? @@ -67,7 +67,7 @@ extension RunnerTests { var gestureEndUptimeMs: Double? for (index, step) in steps.enumerated() { - let stepOutcome = perform(index, step) + let stepOutcome = try perform(index, step) if gestureStartUptimeMs == nil { gestureStartUptimeMs = stepOutcome.gestureStartUptimeMs } @@ -144,13 +144,13 @@ extension RunnerTests { _ step: SequenceStep, activeApp: XCUIApplication, synthesizedContext: SynthesizedCoordinateContext? = nil - ) -> SequenceStepOutcome { + ) throws -> SequenceStepOutcome { let x = step.x ?? 0 let y = step.y ?? 0 let longPressDurationMs = min(max(step.durationMs ?? 800, 16), 10000) if let policyKind = synthesizedPolicyKind(forSequenceStep: step) { - switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { - synthesizedSequenceStep( + switch try performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { + try synthesizedSequenceStep( policyKind, app: activeApp, x: x, @@ -171,15 +171,15 @@ extension RunnerTests { break } } - let (timing, outcome) = performGesture(activeApp) { + let (timing, outcome) = try performGesture(activeApp) { switch step.kind { case "doubleTap": // doubleTapAt per step, matching the behavior of the retired tapSeries doubleTap path. - return doubleTapAt(app: activeApp, x: x, y: y) + return try doubleTapAt(app: activeApp, x: x, y: y) case "longPress": - return longPressAt(app: activeApp, x: x, y: y, duration: longPressDurationMs / 1000.0) + return try longPressAt(app: activeApp, x: x, y: y, duration: longPressDurationMs / 1000.0) default: - return tapAt(app: activeApp, x: x, y: y) + return try tapAt(app: activeApp, x: x, y: y) } } return finishedSequenceStep(step, timing: timing, outcome: outcome) @@ -193,11 +193,11 @@ extension RunnerTests { y: Double, longPressDurationMs: Double, context: SynthesizedCoordinateContext? - ) -> RunnerInteractionOutcome? { + ) throws -> RunnerInteractionOutcome? { switch policyKind { #if os(iOS) case .coordinateLongPress: - return synthesizedLongPressAt( + return try synthesizedLongPressAt( app: app, x: x, y: y, @@ -206,7 +206,7 @@ extension RunnerTests { ) #endif default: - return synthesizedTapAt(app: app, x: x, y: y, context: context) + return try synthesizedTapAt(app: app, x: x, y: y, context: context) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 035d676c9e..82f6d386cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -31,6 +31,8 @@ extension RunnerTests { * (#2660). Nil only where the platform has no iOS keyboard to measure. */ let keyboardBand: RunnerKeyboardBandFact? + /// The lit panel's window the request booked; set only on a multi-panel request (ADR 0025). + var litPanelSurface: LitPanelSurface? = nil } struct SnapshotEvaluation { @@ -191,11 +193,11 @@ extension RunnerTests { ) throws -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { var outcome = SnapshotTierOutcome.completed var cachedCollapsedTabCandidates: [CollapsedTabCandidate]? - func collapsedTabCandidatesOnce() -> [CollapsedTabCandidate] { + func collapsedTabCandidatesOnce() throws -> [CollapsedTabCandidate] { if let cachedCollapsedTabCandidates { return cachedCollapsedTabCandidates } - let candidates = collapsedTabCandidates(context: context, processingDeadline: processingDeadline) + let candidates = try collapsedTabCandidates(context: context, processingDeadline: processingDeadline) if candidates == nil { outcome = .deadlineExhausted } cachedCollapsedTabCandidates = candidates ?? [] return candidates ?? [] @@ -204,18 +206,18 @@ extension RunnerTests { // Acquisition serializes reported frames only; no coordinate space or fold decision is carried // down the walk (#2661). Its sole bounds are raw traversal depth (nil for a regular capture) and // the node cap; `SnapshotPresentation` folds and cuts the normalized array. - var nodes: [RawAXNode] = [] - nodes.append( - makeSnapshotNode( - snapshot: context.rootSnapshot, - evaluation: evaluateSnapshot(context.rootSnapshot), - depth: 0, - index: 0, - parentIndex: nil - ) + var root = makeSnapshotNode( + snapshot: context.rootSnapshot, + evaluation: evaluateSnapshot(context.rootSnapshot), + depth: 0, + index: 0, + parentIndex: nil ) + Self.publishLitPanelRoot(&root, surface: context.litPanelSurface) + let rootChildren = try Self.litPanelRootChildren(context.rootSnapshot, surface: context.litPanelSurface) + var nodes: [RawAXNode] = [root] if Self.canDescendAtRawDepth(0, hint: hint) { - appendCollapsedTabFallbackNodes( + try appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: context.rootSnapshot, resolveCandidates: collapsedTabCandidatesOnce, @@ -227,7 +229,7 @@ extension RunnerTests { var seen = Set() var stack: [SnapshotTraversalEntry] = [] if Self.canDescendAtRawDepth(0, hint: hint) { - stack = context.rootSnapshot.children.map { + stack = rootChildren.map { SnapshotTraversalEntry(snapshot: $0, depth: 1, parentIndex: 0) } } @@ -272,7 +274,7 @@ extension RunnerTests { throw regularSnapshotTooLargeFailure(nodeCount: nodes.count) } if Self.canDescendAtRawDepth(depth, hint: hint) { - appendCollapsedTabFallbackNodes( + try appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: snapshot, resolveCandidates: collapsedTabCandidatesOnce, @@ -294,6 +296,44 @@ extension RunnerTests { ) } + /// A lit-panel snapshot's root, raw or regular, is the booked box (ADR 0025). + static func publishLitPanelRoot(_ root: inout RawAXNode, surface: LitPanelSurface?) { + if let surface { + root.rect = SnapshotRect(surface.frame) + } + } + + /// A lit-panel snapshot descends only into windows proven on the lit display; none refuses. + static func litPanelRootChildren( + _ root: XCUIElementSnapshot, + surface: LitPanelSurface? + ) throws -> [XCUIElementSnapshot] { + guard let surface else { return root.children } + let lit = windowsOnLitDisplay(root.children, litDisplayID: surface.displayID) { snapshotDisplayID($0) } + if lit.isEmpty { + throw RunnerAppScreenCaptureFailure.unresolvedScreen + } + return lit + } + + /// A window that reports no display is dropped: it may be on the dark panel. + static func windowsOnLitDisplay( + _ windows: [Window], + litDisplayID: UInt, + displayID: (Window) -> UInt? + ) -> [Window] { + windows.filter { displayID($0) == litDisplayID } + } + + /// The display a window snapshot reports, or nil when the runtime names none. + static func snapshotDisplayID(_ snapshot: XCUIElementSnapshot) -> UInt? { + var displayID: UInt? + _ = RunnerObjCExceptionCatcher.catchException({ + displayID = ((snapshot as AnyObject).value(forKey: "displayID") as? NSNumber)?.uintValue + }) + return displayID.flatMap { $0 == 0 ? nil : $0 } + } + // See `snapshotFast` above: the single production entry point, no unit-test overload. func snapshotRaw(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) @@ -409,7 +449,8 @@ extension RunnerTests { func walk( _ snapshot: XCUIElementSnapshot, depth: Int, - parentIndex: Int? + parentIndex: Int?, + children: [XCUIElementSnapshot]? = nil ) throws { if let limit = hint.rawTraversalDepth, depth > limit { return } @@ -428,8 +469,7 @@ extension RunnerTests { ) ) - let children = snapshot.children - for child in children { + for child in children ?? snapshot.children { try walk( child, depth: depth + 1, @@ -441,8 +481,12 @@ extension RunnerTests { try walk( context.rootSnapshot, depth: 0, - parentIndex: nil + parentIndex: nil, + children: try Self.litPanelRootChildren(context.rootSnapshot, surface: context.litPanelSurface) ) + if !nodes.isEmpty { + Self.publishLitPanelRoot(&nodes[0], surface: context.litPanelSurface) + } return SnapshotAcquisition( hint: hint, nodes: nodes, @@ -457,7 +501,7 @@ extension RunnerTests { app: XCUIApplication, hint: CaptureHint, sliceDeadline deadline: Date - ) -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { + ) throws -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { var nodes: [RawAXNode] = [ interactiveRootNode(rect: .zero) ] @@ -474,7 +518,7 @@ extension RunnerTests { ) } - let viewport = safeSnapshotViewport(app: app, readingOrientation: false) + let viewport = try safeSnapshotViewport(app: app, readingOrientation: false) var seen = Set() var candidates: [RawAXNode] = [] let flatElements = flatInteractiveElements(app: app, deadline: deadline) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 6f31243dea..17f409cd26 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -36,7 +36,8 @@ extension RunnerTests { app: XCUIApplication, hint: CaptureHint, captureDeadline: Date = .distantFuture, - treeCaptureSliceBudgetOverride: TimeInterval? = nil + treeCaptureSliceBudgetOverride: TimeInterval? = nil, + litPanelSurface: LitPanelSurface? = nil ) throws -> SnapshotTraversalContext? { let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow)) @@ -54,7 +55,7 @@ extension RunnerTests { timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)), timeoutError: snapshotMainThreadTimeoutError("reading snapshot viewport") ) { - self.safeSnapshotViewport(app: app, readingOrientation: true) + try self.safeSnapshotViewport(app: app, readingOrientation: true) } // Use the same tree for keyboard coverage so a later dismissal cannot invalidate the band. let keyboardBand = captureKeyboardBandFact(snapshot: rootSnapshot, viewport: viewport, deadline: captureDeadline) @@ -64,7 +65,8 @@ extension RunnerTests { rootSnapshot: rootSnapshot, rootReadDuration: rootReadDuration, viewport: viewport, - keyboardBand: keyboardBand + keyboardBand: keyboardBand, + litPanelSurface: litPanelSurface ) } @@ -136,15 +138,18 @@ extension RunnerTests { /// The viewport as a declared fact; a read that raises is `.missing(reason: .notProvided)` (#2891). /// `readingOrientation` reads the interface orientation in the same hop, for tiers whose frames can /// arrive in the device's native space; without it the viewport cannot anchor a rotation. - func safeSnapshotViewport(app: XCUIApplication, readingOrientation: Bool) -> SnapshotViewport { - safely("SNAPSHOT_VIEWPORT", .missing(reason: .notProvided)) { - .reported( - box: snapshotAppFrame(app: app), - interfaceOrientation: readingOrientation - ? capturedInterfaceOrientation(app: app) - : RunnerInterfaceOrientation.unknown - ) + func safeSnapshotViewport(app: XCUIApplication, readingOrientation: Bool) throws -> SnapshotViewport { + let result: Result = safely("SNAPSHOT_VIEWPORT", .success(.missing(reason: .notProvided))) { + Result { + .reported( + box: try snapshotAppFrame(app: app), + interfaceOrientation: readingOrientation + ? capturedInterfaceOrientation(app: app) + : RunnerInterfaceOrientation.unknown + ) + } } + return try result.get() } private func describeSnapshotError(_ error: Error) -> String { @@ -257,9 +262,9 @@ extension RunnerTests { return placeholderText(element.placeholderValue) } - private func snapshotAppFrame(app: XCUIApplication) -> CGRect { + private func snapshotAppFrame(app: XCUIApplication) throws -> CGRect { #if os(iOS) - return onScreenWindowFrame(app: app) + return try onScreenWindowFrame(app: app) #else return app.frame #endif @@ -294,11 +299,11 @@ extension RunnerTests { func appendCollapsedTabFallbackNodes( to nodes: inout [RawAXNode], containerSnapshot: XCUIElementSnapshot, - resolveCandidates: () -> [CollapsedTabCandidate], + resolveCandidates: () throws -> [CollapsedTabCandidate], depth: Int, parentIndex: Int - ) { - let fallbackNodes = collapsedTabFallbackNodes( + ) throws { + let fallbackNodes = try collapsedTabFallbackNodes( for: containerSnapshot, resolveCandidates: resolveCandidates, startingIndex: nodes.count, @@ -331,15 +336,14 @@ extension RunnerTests { let selected: Bool } - /// The live elements any collapsed tab container in this capture may adopt, read in one XCTest - /// query: `collapsedTabCandidatePredicate` copies each match's attributes while XCTest evaluates - /// it, so no match costs a further round trip. The query starts only while the time left before - /// `processingDeadline` covers what this capture's root tree read took; nil means it did not start. + /// Live controls collapsed tab containers may adopt, queried under the capture's proven-lit roots. + /// The predicate copies attributes during each query without further round trips. Queries start + /// only while the remaining slice covers the root read's duration; nil means a query could not start. @MainActor func collapsedTabCandidates( context: SnapshotTraversalContext, processingDeadline: Date - ) -> [CollapsedTabCandidate]? { + ) throws -> [CollapsedTabCandidate]? { let containerFrames = Self.snapshotDescendants(of: context.rootSnapshot) .filter(isCollapsedTabContainer) .map(\.frame) @@ -347,19 +351,26 @@ extension RunnerTests { let predicate = Self.collapsedTabCandidatePredicate(containerFrames: containerFrames) { attributes in candidates.append(self.collapsedTabCandidate(attributes)) } - guard - Self.xctestQueryCanStart( - deadline: processingDeadline, - now: Date(), - expectedCost: context.rootReadDuration - ) - else { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_QUERY_NOT_STARTED") - return nil + func canStartQuery() -> Bool { + guard Self.xctestQueryCanStart(deadline: processingDeadline, now: Date(), expectedCost: context.rootReadDuration) else { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_QUERY_NOT_STARTED") + return false + } + return true + } + guard canStartQuery() else { return nil } + let queries = try elementQueries( + root: context.queryRoot, + matching: .any, + litDisplayID: context.litPanelSurface?.displayID + ) + var matches: [XCUIElement] = [] + for query in queries { + guard canStartQuery() else { return nil } + matches += snapshotElementsQuery { + query.matching(predicate).allElementsBoundByIndex + }.elements } - let matches = snapshotElementsQuery { - context.queryRoot.descendants(matching: .any).matching(predicate).allElementsBoundByIndex - }.elements return matches.isEmpty ? [] : candidates } @@ -429,18 +440,18 @@ extension RunnerTests { private func collapsedTabFallbackNodes( for containerSnapshot: XCUIElementSnapshot, - resolveCandidates: () -> [CollapsedTabCandidate], + resolveCandidates: () throws -> [CollapsedTabCandidate], startingIndex: Int, depth: Int, parentIndex: Int - ) -> [RawAXNode] { + ) throws -> [RawAXNode] { guard isCollapsedTabContainer(containerSnapshot) else { return [] } // Candidates are read from live elements, which XCTest reports in the app's own space, and a // collapsed tab container sits under the app's own window. `SnapshotGeometrySpace.normalized` keys // rotation on ancestry, so these nodes never enter a declared native space and need no special // case here; `isCollapsedTabCandidate` compares reported frames to each other. let containerFrame = containerSnapshot.frame - let candidates = resolveCandidates().compactMap { candidate in + let candidates = try resolveCandidates().compactMap { candidate in collapsedTabCandidateNode( candidate: candidate, containerSnapshot: containerSnapshot, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index c47b77842a..dadf4b0128 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -162,8 +162,11 @@ extension RunnerTests { preferredBackend: options.preferredBackend ) #endif + // Only the tree tier can prove which panel a window is on, so a multi-panel request runs it + // alone; preparation already refused a pin to another backend. + let surfacePlan = target.litPanelSurface == nil ? plan : [.recursiveTree] let effective = Self.effectiveSnapshotCapturePlan( - plan, + surfacePlan, xCTestChannelPenalized: xCTestChannelPenalized, xCTestRecoveryProbeDue: xCTestRecoveryProbeDue, availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)), @@ -344,7 +347,8 @@ extension RunnerTests { app: app, hint: hint, captureDeadline: deadline, - treeCaptureSliceBudgetOverride: step.treeCaptureSliceBudgetOverride + treeCaptureSliceBudgetOverride: step.treeCaptureSliceBudgetOverride, + litPanelSurface: target.litPanelSurface ) else { return (nil, .completed) @@ -373,7 +377,7 @@ extension RunnerTests { timeout: max(0.1, sliceDeadline.timeIntervalSinceNow), timeoutError: self.snapshotMainThreadTimeoutError("running query-sweep snapshot") ) { - self.querySweepSnapshotAcquisition( + try self.querySweepSnapshotAcquisition( app: app, hint: hint, sliceDeadline: sliceDeadline diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift index 0096953434..7a9b5eaaaf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -12,6 +12,8 @@ struct SnapshotCaptureTarget { let app: XCUIApplication let bundleId: String? let processIdentifier: Int? + /// The lit panel's window the request booked; set only on a multi-panel request. + var litPanelSurface: LitPanelSurface? = nil } /// What snapshot command preparation hands the off-main capture. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index b4063d9b0e..471827e1c0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -18,14 +18,22 @@ extension RunnerTests { case .response(let response): return .response(response) case .context(let context): + if let refusal = self.bookRequestLitPanelSurface(app: context.app) { + return .response(refusal.response) + } + if self.requestLitPanel != nil, let refusal = Self.litPanelBackendPinRefusal(command) { + return .response(refusal) + } + var target = context.observation == nil + ? self.takeSnapshotCaptureTarget(app: context.app) + : SnapshotCaptureTarget( + app: context.app, + bundleId: command.appBundleId?.trimmedNonEmpty, + processIdentifier: Self.processIdentifier(of: context.app) + ) + target.litPanelSurface = self.litPanelState.surface return .capture( - context.observation == nil - ? self.takeSnapshotCaptureTarget(app: context.app) - : SnapshotCaptureTarget( - app: context.app, - bundleId: command.appBundleId?.trimmedNonEmpty, - processIdentifier: Self.processIdentifier(of: context.app) - ), + target, systemSurface: context.systemSurface, observation: context.observation ) @@ -128,6 +136,8 @@ extension RunnerTests { } setNeedsPostSnapshotInteractionDelay() return Response(ok: true, data: payload) + } catch let refusal as RunnerAppScreenCaptureFailure { + return refusal.response } catch let failure as SnapshotCaptureFailure { invalidateCachedTargetAfterSnapshotFailure() return Response( @@ -159,6 +169,25 @@ extension RunnerTests { } } + /// A multi-panel snapshot runs only the tree backend, the one that proves which panel a window is + /// on, so a pin to another backend, or `customActions` implying one, is refused as a request. + static func litPanelBackendPinRefusal(_ command: Command) -> Response? { + guard let pinned = presentationOptions(from: command).preferredBackend, + pinned != SnapshotBackendKind.recursiveTree.rawValue + else { + return nil + } + let cause = command.preferredBackend == nil ? "customActions" : "the \(pinned) backend pin" + return Response( + ok: false, + error: ErrorPayload( + code: "UNSUPPORTED_OPERATION", + message: "a multi-panel snapshot runs only the tree backend, so \(cause) cannot be served", + hint: "Snapshot without \(cause): only the tree backend proves which panel a window is on (ADR 0025)." + ) + ) + } + func invalidateCachedTargetAfterSnapshotFailure() { applyMainOwnedSnapshotState("target_invalidation") { self.invalidateCachedTarget(reason: "ax_snapshot_failure") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift index 8417e035c2..7947052a5e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift @@ -73,14 +73,14 @@ extension RunnerTests { app: XCUIApplication, target: TextEntryTarget, expectedText: String - ) -> SynthesizedTextCommitOutcome { + ) throws -> SynthesizedTextCommitOutcome { guard !expectedText.contains("\n"), !expectedText.contains("\r") else { return .unobservable } let waitStartedAt = Date() NSLog("[DEBUG-1874] wait start expectedLen=%ld route=replacement", expectedText.count) - let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) - let outcome = Self.awaitSynthesizedReplacementCommitOutcome( + let ingredients = try synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) + let outcome = try Self.awaitSynthesizedReplacementCommitOutcome( expectedText: expectedText, placeholder: ingredients.placeholder, fieldIsUnreadable: ingredients.fieldIsUnreadable, @@ -132,13 +132,13 @@ extension RunnerTests { app: XCUIApplication, target: TextEntryTarget, expectedText: String - ) -> ( + ) throws -> ( placeholder: String?, fieldIsUnreadable: Bool, - observe: () -> TextValueRead, + observe: () throws -> TextValueRead, waitForNextObservation: () -> Void ) { - let field = resolveTextEntryInput(app: app, target: target)?.attributes + let field = try resolveTextEntryInput(app: app, target: target)?.attributes let placeholder = field?.placeholderValue let waitStartedAt = Date() return ( @@ -146,7 +146,7 @@ extension RunnerTests { fieldIsUnreadable: textValueRead(for: field) == .unreadable, observe: { let observed = self.textValueRead( - for: self.resolveTextEntryInput(app: app, target: target)?.attributes, + for: try self.resolveTextEntryInput(app: app, target: target)?.attributes, treatingPlaceholderAsEmpty: true ) var observedText: String? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 99a1a4838c..df29a9509d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -115,11 +115,24 @@ extension RunnerTests { func synthesizedSequenceCoordinateContext( steps: [SequenceStep], app: XCUIApplication - ) -> SynthesizedCoordinateContext? { + ) throws -> SynthesizedCoordinateContext? { guard steps.lazy.contains(where: { synthesizedPolicyKind(forSequenceStep: $0) != nil }) else { return nil } - return synthesizedCoordinateContext(app: app) + return try synthesizedCoordinateContext(app: app) + } + + @MainActor + func performSynthesizedTapGesture( + app: XCUIApplication, + x: Double, + y: Double, + context: SynthesizedCoordinateContext? = nil + ) throws -> SynthesizedGestureAttempt { + let context = try context ?? synthesizedCoordinateContext(app: app) + return try performSynthesizedGesture(app, kind: .coordinateTap, context: context) { + try synthesizedTapAt(app: app, x: x, y: y, context: context) + } } /// `context` is nil when no window frame resolved; `kind`'s fallback decision then reads the @@ -129,10 +142,10 @@ extension RunnerTests { _ app: XCUIApplication, kind: SynthesizedGesturePolicyKind, context: SynthesizedCoordinateContext?, - synthesize: () -> RunnerInteractionOutcome? - ) -> SynthesizedGestureAttempt { + synthesize: () throws -> RunnerInteractionOutcome? + ) throws -> SynthesizedGestureAttempt { var outcome: RunnerInteractionOutcome? - let timing = measureGesture { outcome = synthesize() } + let timing = try measureGesture { outcome = try synthesize() } guard let outcome else { logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false) return .refused(timing: timing, message: "synthesized gesture deadline expired before dispatch", hint: nil) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 7a20a5de0f..4b11d38245 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -12,7 +12,7 @@ extension RunnerTests { durationMs: Double, profile: SynthesizedDragProfile, context: SynthesizedCoordinateContext? = nil - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { #if os(iOS) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { return .unsupported( @@ -21,7 +21,7 @@ extension RunnerTests { ) } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard let context = context ?? synthesizedCoordinateContext(app: app) else { + guard let context = try context ?? synthesizedCoordinateContext(app: app) else { return .unsupported( message: "synthesized coordinate drag could not resolve an app window with a finite screen frame", hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." @@ -99,7 +99,7 @@ extension RunnerTests { y: Double, context: SynthesizedCoordinateContext? = nil, deadline: Date? = nil - ) -> RunnerInteractionOutcome? { + ) throws -> RunnerInteractionOutcome? { #if os(iOS) guard x.isFinite, y.isFinite else { return .unsupported( @@ -113,7 +113,7 @@ extension RunnerTests { x: x, y: y, orientation: orientation, - context: context ?? synthesizedCoordinateContext(app: app), + context: try context ?? synthesizedCoordinateContext(app: app), gesture: .tap ) guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { @@ -174,7 +174,7 @@ extension RunnerTests { y: Double, durationMs: Double, context: SynthesizedCoordinateContext? - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { guard x.isFinite, y.isFinite, durationMs.isFinite, durationMs > 0 else { return .unsupported( message: "synthesized coordinate long press requires finite coordinates and a positive duration", @@ -227,7 +227,7 @@ extension RunnerTests { y: Double, x2: Double, y2: Double - ) -> DragPoints { + ) throws -> DragPoints { let original = DragPoints(x: x, y: y, x2: x2, y2: y2) #if os(iOS) guard let keyboardFrame = visibleKeyboardFrame(app: app) else { @@ -245,7 +245,7 @@ extension RunnerTests { return original } - let appFrame = onScreenWindowFrame(app: app) + let appFrame = try onScreenWindowFrame(app: app) guard !appFrame.isEmpty else { return original } @@ -282,9 +282,9 @@ extension RunnerTests { #endif } - func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame { + func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) throws -> TouchVisualizationFrame { let appFrame = app.frame - let referenceFrame = resolvedTouchReferenceFrame(app: app, appFrame: appFrame) + let referenceFrame = try resolvedTouchReferenceFrame(app: app, appFrame: appFrame) let originX = appFrame.isEmpty ? referenceFrame.minX : appFrame.minX let originY = appFrame.isEmpty ? referenceFrame.minY : appFrame.minY return TouchVisualizationFrame( @@ -316,9 +316,9 @@ extension RunnerTests { y: Double, x2: Double, y2: Double - ) -> DragVisualizationFrame { - let start = resolvedTouchVisualizationFrame(app: app, x: x, y: y) - let end = resolvedTouchVisualizationFrame(app: app, x: x2, y: y2) + ) throws -> DragVisualizationFrame { + let start = try resolvedTouchVisualizationFrame(app: app, x: x, y: y) + let end = try resolvedTouchVisualizationFrame(app: app, x: x2, y: y2) return DragVisualizationFrame( x: start.x, y: start.y, @@ -329,8 +329,8 @@ extension RunnerTests { ) } - func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect { - let resolved = resolveRunnerWindow(app: app) + func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) throws -> CGRect { + let resolved = try resolveRunnerWindow(app: app) if resolved.window != nil { return frameAvoidingKeyboard(app: app, frame: resolved.frame) } @@ -344,9 +344,9 @@ extension RunnerTests { /// context its record is routed through, so a tree with no window whose app provably fills its /// sole screen plans on that screen, the same route a synthesized tap or scroll takes. @MainActor - func gestureViewportFrame(app: XCUIApplication) -> CGRect { + func gestureViewportFrame(app: XCUIApplication) throws -> CGRect { #if os(iOS) - guard let context = synthesizedCoordinateContext(app: app) else { + guard let context = try synthesizedCoordinateContext(app: app) else { return frameAvoidingKeyboard(app: app, frame: app.frame) } // The sole-screen route means the windows query just failed, so a keyboard probe would too. @@ -354,7 +354,7 @@ extension RunnerTests { ? context.referenceFrame : frameAvoidingKeyboard(app: app, frame: context.referenceFrame) #else - return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + return try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) #endif } @@ -389,16 +389,16 @@ extension RunnerTests { x2: Double, y2: Double, context: SynthesizedCoordinateContext? = nil - ) -> SynthesizedDragPlan? { + ) throws -> SynthesizedDragPlan? { #if os(iOS) - let context = context ?? synthesizedCoordinateContext(app: app) + let context = try context ?? synthesizedCoordinateContext(app: app) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite, let context else { return nil } let points = context.allowsKeyboardProbe - ? keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) + ? try keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) : DragPoints(x: x, y: y, x2: x2, y2: y2) return SynthesizedDragPlan( points: points, @@ -449,11 +449,18 @@ extension RunnerTests { func synthesizedCoordinateContext( app: XCUIApplication, windowQueryRejected: Bool = false - ) -> SynthesizedCoordinateContext? { + ) throws -> SynthesizedCoordinateContext? { #if os(iOS) var health = mainOwned.accessibilityHealth let window: XCUIElement? let referenceFrame: CGRect + if requestLitPanel != nil { + let booked = try requireRequestLitPanelSurface(app: app) + guard let context = SynthesizedCoordinateContext( + validating: booked.frame, resolvedWindow: booked.window, accessibilityHealth: health + ) else { throw RunnerAppScreenCaptureFailure.unresolvedScreen } + return context + } // A tree that just rejected a query skips the windows query only where the sole screen answers // without it; anywhere else that query is still the only route. let rejectedTreeSoleScreenFrame = windowQueryRejected ? soleScreenReferenceFrame(app: app) : nil @@ -512,14 +519,20 @@ extension RunnerTests { } /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a - /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the - /// resolved window `fallback`, which names its panel (ADR 0025). When the device's only screen + /// placed iPad window's smaller box is not the basis (#3899). A named lit panel supplies its full + /// screen box independently of the booked window translation (ADR 0025). When the device's only screen /// will not report its size, `fallback` is the basis only while the app provably fills that /// screen; otherwise this is nil and the gesture refuses before it dispatches. Portrait and an /// unknown orientation leave the point as it is, so they keep `fallback` without a read; landscape /// swaps the box's sides and upside-down mirrors the point across them, so both read it. @MainActor func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { + if let panel = requestLitPanel { + return Self.soleScreenReferenceFrame( + portraitSize: CGSize(width: panel.widthPt, height: panel.heightPt), + interfaceOrientation: interfaceOrientation + ) ?? fallback + } guard XCUIScreen.screens.count == 1 else { return fallback } switch interfaceOrientation { case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight, @@ -657,9 +670,9 @@ extension RunnerTests { } } - func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? { + func swipe(app: XCUIApplication, direction: String) throws -> DragVisualizationFrame? { if performTvRemoteSwipeIfAvailable(direction: direction) { - let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + let frame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) let midX = frame.midX let midY = frame.midY return DragVisualizationFrame( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 16e7020d20..6ab772d269 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -261,12 +261,12 @@ extension RunnerTests { /// Clears until the field reads empty, at most `maxClearPassCount` passes. A read that cannot /// answer ends the clear, so the post never types over text it could not see removed. static func clearUntilEmpty( - clearOnce: () -> SynthesizedStepDispatch, - read: () -> TextValueRead - ) -> SynthesizedClearOutcome { + clearOnce: () throws -> SynthesizedStepDispatch, + read: () throws -> TextValueRead + ) rethrows -> SynthesizedClearOutcome { for pass in 1...SynthesizedTextPlan.Step.maxClearPassCount { - guard clearOnce() == .posted else { return .stopped } - switch read() { + guard try clearOnce() == .posted else { return .stopped } + switch try read() { case .text(let value) where value.isEmpty: return .cleared case .text: @@ -300,22 +300,22 @@ extension RunnerTests { func runSynthesizedTextPlan( _ plan: SynthesizedTextPlan, text: String, - post: (_ characters: String, _ step: SynthesizedTextPlan.Step) -> SynthesizedStepDispatch, - waitAfterWarmupCharacter: (_ characters: String) -> Void, + post: (_ characters: String, _ step: SynthesizedTextPlan.Step) throws -> SynthesizedStepDispatch, + waitAfterWarmupCharacter: (_ characters: String) throws -> Void, didPostStep: (_ step: SynthesizedTextPlan.Step) -> Void = { _ in } - ) -> SynthesizedPlanRun { + ) rethrows -> SynthesizedPlanRun { let characters = Array(text) var postedCount = 0 for step in plan.steps { let nextCount = postedCount + step.characterCount - if post(String(characters[postedCount.. 0 { sleepFor(step.pauseAfterSeconds) } @@ -326,7 +326,7 @@ extension RunnerTests { @MainActor func runSynthesizedReplacementRoute( _ request: SynthesizedReplacementRequest - ) -> SynthesizedReplacementRouteOutcome { + ) throws -> SynthesizedReplacementRouteOutcome { #if os(iOS) NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder-replacement") let plan = Self.synthesizedTextPlan( @@ -355,11 +355,11 @@ extension RunnerTests { var clearOutcome: SynthesizedClearOutcome? // A private synthesis channel that is gone mid-plan leaves the command the same // point-and-focus fallback it had before the plan existed. - let synthesisAvailable = runSynthesizedTextPlan( + let synthesisAvailable = try runSynthesizedTextPlan( plan, text: request.text, post: { slice, step in - func enter(_ text: String, replacingExistingText: Bool) -> SynthesizedStepDispatch { + func enter(_ text: String, replacingExistingText: Bool) throws -> SynthesizedStepDispatch { switch request.synthesizer.enterText( app: request.app, text: text, @@ -379,18 +379,18 @@ extension RunnerTests { } if step.replacesExistingText { var passes = 0 - let outcome = Self.clearUntilEmpty( + let outcome = try Self.clearUntilEmpty( clearOnce: { passes += 1 - return enter(XCUIKeyboardKey.delete.rawValue, replacingExistingText: true) + return try enter(XCUIKeyboardKey.delete.rawValue, replacingExistingText: true) }, - read: { self.clearedFieldRead(app: request.app, target: request.target) } + read: { try self.clearedFieldRead(app: request.app, target: request.target) } ) clearOutcome = outcome NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_CLEAR passes=%d outcome=%@", passes, String(describing: outcome)) guard outcome == .cleared || outcome == .unverified else { return .stop } } - return enter(slice, replacingExistingText: false) + return try enter(slice, replacingExistingText: false) }, // A replacement never peels a warmup character, so no step asks for a read-back. waitAfterWarmupCharacter: { _ in } @@ -418,7 +418,7 @@ extension RunnerTests { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable") guard let point = request.target.refreshPoint else { return .notApplicable } return .fallback( - focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y) + try focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y) ) } // The private synthesize call returns at post time, not commit time, and this route never @@ -426,7 +426,7 @@ extension RunnerTests { // still-in-flight character at all. Wait here, on the same request.target // (element nil, refreshPoint set) that gated this route, so each poll re-resolves via the // refresh point rather than trusting a stale element handle. - let commit = awaitSynthesizedReplacementCommit( + let commit = try awaitSynthesizedReplacementCommit( app: request.app, target: request.target, expectedText: request.text @@ -523,9 +523,9 @@ extension RunnerTests { stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout, ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling, now: () -> Date = { Date() }, - observe: () -> TextValueRead, + observe: () throws -> TextValueRead, waitForNextObservation: () -> Void - ) -> SynthesizedTextCommitOutcome { + ) rethrows -> SynthesizedTextCommitOutcome { // An unreadable field never exposes its value, so every read could only expire the wait. The // element route leaves such a field unverified rather than failed; so does this one. A masked // value appears only once the burst lands, so a read that turns unreadable mid-wait ends it @@ -545,7 +545,7 @@ extension RunnerTests { // landed during the final poll sleep — the exact loaded-host timing this wait exists for. while true { let observedText: String? - switch observe() { + switch try observe() { case .unreadable: return .unobservable case .text(let value): diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 3d3d4d3987..4bc51d69cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -286,28 +286,28 @@ extension RunnerTests { struct TextEntryReader { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? - private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? + private let resolve: (TextEntryTarget) throws -> ResolvedTextEntryInput? - init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) { + init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) throws -> ResolvedTextEntryInput?) { self.target = target.withResolvedInput(target.postFocusRead) lastRead = target.postFocusRead self.resolve = resolve } - mutating func input() -> ResolvedTextEntryInput? { + mutating func input() throws -> ResolvedTextEntryInput? { if let lastRead { return lastRead } - let read = resolve(target) + let read = try resolve(target) target = target.withResolvedInput(read) lastRead = read return read } /// Runs `dispatch`, which posts to the app or pauses, so the next `input()` resolves again. - mutating func dispatch(_ dispatch: () -> T) -> T { + mutating func dispatch(_ dispatch: () throws -> T) rethrows -> T { lastRead = nil - return dispatch() + return try dispatch() } } @@ -395,8 +395,8 @@ extension RunnerTests { /// left, delete bursts run from the caret, and a value longer than one burst is read again after /// each burst until it reads empty, stops shortening, or has had at least one burst more than it /// needs. - func clearTextInput(app: XCUIApplication, reader: inout TextEntryReader) { - guard let input = reader.input() else { + func clearTextInput(app: XCUIApplication, reader: inout TextEntryReader) throws { + guard let input = try reader.input() else { return } // Skip the clear (edge-tap, select all, delete bursts) ONLY when we can confirm the @@ -415,10 +415,10 @@ extension RunnerTests { var valueLength = normalizedElementText(input.attributes.value).count // The tap also focuses the field, which a clear can reach with focus unconfirmed. #if !os(tvOS) - reader.dispatch { moveCaretToEnd(app: app, input: input) } + try reader.dispatch { try moveCaretToEnd(app: app, input: input) } #endif // An input that stops answering once focused (#3060) cannot be read back, so it gets one burst. - guard reader.input() != nil else { + guard try reader.input() != nil else { let burst = Self.textClearBurst(valueLength: valueLength) reader.dispatch { input.element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: burst)) } return @@ -430,9 +430,9 @@ extension RunnerTests { // bursts alone. if existing != nil { var passes = 0 - let outcome = Self.clearUntilEmpty( + let outcome = try Self.clearUntilEmpty( clearOnce: { - guard let input = reader.input() else { + guard let input = try reader.input() else { return .stop } passes += 1 @@ -443,13 +443,13 @@ extension RunnerTests { return .posted }, read: { - textValueRead(for: reader.input()?.attributes, treatingPlaceholderAsEmpty: true) + textValueRead(for: try reader.input()?.attributes, treatingPlaceholderAsEmpty: true) } ) NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_CLEAR route=element passes=%d outcome=%@", passes, String(describing: outcome)) // Text left after every pass goes to the bursts; an input that no longer resolves is not typed into. guard outcome == .notCleared, - let remaining = reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }) + let remaining = try reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }) else { return } @@ -457,14 +457,14 @@ extension RunnerTests { } #endif for _ in 0...(valueLength / Self.maxTextClearBurst + 1) { - guard let input = reader.input() else { + guard let input = try reader.input() else { return } let burst = Self.textClearBurst(valueLength: valueLength) reader.dispatch { input.element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: burst)) } // A burst that covers the value needs no read. guard burst < valueLength + Self.textClearSlack, - let remaining = reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }), + let remaining = try reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }), Self.textClearContinues(clearedLength: valueLength, remainingLength: remaining.count) else { return @@ -516,10 +516,11 @@ extension RunnerTests { /// Resolves the target's input, with the snapshot the resolution read it through. A bound target /// accepts only an input whose snapshot carries its bound identity, so every post, read-back, - /// verification poll, and repair refuses an input that took the bound one's place. - func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { + /// verification poll, and repair refuses an input that took the bound one's place. Every search runs + /// under the element search's roots, so a lit-panel refusal propagates. + func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) throws -> ResolvedTextEntryInput? { guard let boundIdentity = target.boundIdentity else { - return resolveUnboundTextEntryInput(app: app, target: target) + return try resolveUnboundTextEntryInput(app: app, target: target) } func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? { guard let input = ResolvedTextEntryInput(reading: candidate, contentTranslation: target.contentTranslation), @@ -538,14 +539,14 @@ extension RunnerTests { } if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint, - case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), + case .matches(let candidates) = try probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), let match = candidates.first(where: { $0.identity == boundIdentity }) { return match } if let focused = focusedTextInput(app: app), let input = boundInput(focused) { return input } - if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity), + if let byIdentifier = try boundTextEntryInputQuery(app: app, identity: boundIdentity), let input = boundInput(byIdentifier) { return input } @@ -554,12 +555,14 @@ extension RunnerTests { /// Whether a bound target's input is proven gone: XCTest found no match for it, or a different /// input answers in its place, and nothing at its point carries its identity. A failed probe - /// proves nothing. Unbound targets and unidentified inputs without an element cannot be proven - /// gone. - func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) -> Bool { - guard let boundIdentity = target.boundIdentity, - let query = boundTextEntryInputQuery(app: app, identity: boundIdentity) ?? target.element - else { + /// proves nothing; a lit-panel refusal propagates. Unbound targets and unidentified inputs + /// without an element cannot be proven gone. + func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) throws -> Bool { + guard let boundIdentity = target.boundIdentity else { + return false + } + let byIdentifier = try boundTextEntryInputQuery(app: app, identity: boundIdentity) + guard let query = byIdentifier ?? target.element else { return false } switch probeTextEntryInput(query) { @@ -573,7 +576,7 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { + switch try probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { case .absent: return true case .matches(let candidates): @@ -583,12 +586,15 @@ extension RunnerTests { } } - /// The app-wide query for an input whose identifier is unique enough to search by. - private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) -> XCUIElement? { + /// The query, under the element search's roots, for an input whose identifier is distinctive. + private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) throws -> XCUIElement? { guard identity.isDistinguishable else { return nil } - return app.descendants(matching: identity.elementType).matching(identifier: identity.identifier).element + let queries = try elementQueries(app: app, matching: identity.elementType).map { + $0.matching(identifier: identity.identifier) + } + return (queries.count == 1 ? queries.first : queries.first { $0.count > 0 })?.element } /// Snapshots one candidate: its identity, a proven no-match, or a failure that proves nothing. @@ -606,7 +612,7 @@ extension RunnerTests { } /// Resolution for a target not bound to an input yet: the first element that exists. - private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { + private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) throws -> ResolvedTextEntryInput? { let read = { (element: XCUIElement) in ResolvedTextEntryInput(reading: element, contentTranslation: target.contentTranslation) } @@ -617,13 +623,13 @@ extension RunnerTests { return input } if let refreshPoint = target.refreshPoint, - let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y, contentTranslation: target.contentTranslation) { + let refreshed = try textInputCandidatesAt(app: app, point: refreshPoint, contentTranslation: target.contentTranslation).first { return refreshed } return focusedTextInput(app: app).flatMap(read) } - private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) { + private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) throws { #if os(tvOS) return #else @@ -642,7 +648,7 @@ extension RunnerTests { guard let frame = input.frameOnScreen, !frame.isEmpty else { return } - _ = tapAt( + _ = try tapAt( app: app, x: frame.minX + max(2, frame.width - 4), y: frame.minY + max(2, frame.height / 2) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 0f598bcb68..fca641cc12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -104,7 +104,7 @@ extension RunnerTests { } @MainActor - func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { + func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) throws -> TextEntryTarget { let contentTranslation = liveContentTranslation(app: app) guard let x, let y else { let softwareKeyboardVisible = isKeyboardVisible(app: app) @@ -123,7 +123,7 @@ extension RunnerTests { contentTranslation: contentTranslation ) } - let focused = waitForTextEntryReadiness( + let focused = try waitForTextEntryReadiness( app: app, target: TextEntryTarget( element: focusedTextInput(app: app), @@ -136,12 +136,12 @@ extension RunnerTests { } let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) + let input = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first let target = input?.element let requestedPoint = CGPoint(x: x, y: y) // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint - _ = tapAt(app: app, x: tapPoint.x, y: tapPoint.y) + _ = try tapAt(app: app, x: tapPoint.x, y: tapPoint.y) // A visible keyboard is not enough evidence for app.typeText, because focus may still // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. @@ -171,7 +171,7 @@ extension RunnerTests { ) let element = stabilized.focusConfirmed ? readyTarget.element - : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) + : (try waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) return focusedTextEntryTarget(element, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index b30d584165..fa7f9da807 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -30,9 +30,9 @@ extension RunnerTests { app: XCUIApplication, target: TextEntryTarget, timeout: TimeInterval = TextEntryTiming.readinessTimeout - ) -> XCUIElement? { + ) throws -> XCUIElement? { #if os(iOS) - let candidate = resolveTextEntryInput(app: app, target: target)?.element + let candidate = try resolveTextEntryInput(app: app, target: target)?.element let keyboardVisibleAtEntry = isKeyboardVisible(app: app) let deadline = Date().addingTimeInterval(timeout) var hardwareKeyboardFallback = Date().addingTimeInterval( @@ -67,7 +67,7 @@ extension RunnerTests { } return candidate #else - return resolveTextEntryInput(app: app, target: target)?.element + return try resolveTextEntryInput(app: app, target: target)?.element #endif } @@ -85,7 +85,7 @@ extension RunnerTests { /// `ResolvedTextEntryInput` already reads elements this way. It is read again after the /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. - func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { + func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) throws { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return @@ -95,18 +95,18 @@ extension RunnerTests { }) else { return } - retapAt(read: settled, app: app, element: element) + try retapAt(read: settled, app: app, element: element) #endif } /// The readiness repeat, at the read's screen box; a read that names none taps nothing (#3899). - func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { + func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) throws { guard let frame = read.frameOnScreen, !frame.isEmpty else { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=%@", read.frameOnScreen == nil ? "no-screen-frame" : "empty-frame") return } NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP x=%.1f y=%.1f", frame.midX, frame.midY) - _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = try tapAt(app: app, x: frame.midX, y: frame.midY) _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index be057cc408..2106ac54c3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -20,18 +20,15 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt(app: XCUIApplication, x: Double, y: Double, contentTranslation: TextEntryContentTranslation) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first - } - - func textInputCandidatesAt(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> [ResolvedTextEntryInput] { - safely("TEXT_INPUT_AT_POINT", []) { - queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) + func textInputCandidatesAt(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) throws -> [ResolvedTextEntryInput] { + let queries = try textInputQueries(app: app) + return safely("TEXT_INPUT_AT_POINT", []) { + queryTextInputs(queries: queries, point: point, contentTranslation: contentTranslation, shouldStop: { false }) } } - func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: liveContentTranslation(app: app)) { + func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) throws -> XCUIElement? { + switch try probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: liveContentTranslation(app: app)) { case .matches(let inputs): return inputs.first?.element case .absent: @@ -58,8 +55,8 @@ extension RunnerTests { x: Double, y: Double, contentTranslation: TextEntryContentTranslation - ) -> CoordinateTextInputResolution { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { + ) throws -> CoordinateTextInputResolution { + switch try probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): guard let input = inputs.first, let identity = input.identity else { return .unavailable } return .input(TextInputAtPoint(element: input.element, identity: identity)) @@ -84,23 +81,32 @@ extension RunnerTests { } /// What `target`'s input reads between a replacement's clear passes, read inside the same issue - /// containment, with a placeholder read as empty. A read that cannot answer is `.unavailable`. - func clearedFieldRead(app: XCUIApplication, target: TextEntryTarget) -> TextValueRead { + /// containment, with a placeholder read as empty. A read that cannot answer is `.unavailable`; + /// a lit-panel refusal propagates. + func clearedFieldRead(app: XCUIApplication, target: TextEntryTarget) throws -> TextValueRead { + var refusal: Error? let probed = containingTextInputProbeIssues(fallback: TextValueRead.unavailable) { _ in - textValueRead(for: resolveTextEntryInput(app: app, target: target)?.attributes, treatingPlaceholderAsEmpty: true) + do { + return textValueRead(for: try resolveTextEntryInput(app: app, target: target)?.attributes, treatingPlaceholderAsEmpty: true) + } catch { + refusal = error + return .unavailable + } } + if let refusal { throw refusal } guard case .success(let read) = probed else { return .unavailable } return read } - func probeTextInputs(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> TextInputProbeOutcome { + func probeTextInputs(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) throws -> TextInputProbeOutcome { + let queries = try textInputQueries(app: app) switch containingTextInputProbeIssues(fallback: TextInputProbeOutcome.unavailable(.emptyTree), { shouldStop in var treeReadable = app.descendants(matching: .any).firstMatch.exists #if AGENT_DEVICE_RUNNER_UNIT_TESTS if textInputProbeBlindForTesting { treeReadable = false } #endif guard treeReadable else { return .unavailable(.emptyTree) } - let inputs = queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) + let inputs = queryTextInputs(queries: queries, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) return inputs.isEmpty ? .absent : .matches(inputs) }) { case .success(let outcome): @@ -171,14 +177,20 @@ extension RunnerTests { return true } + /// Resolved before the probe's issue containment, so a lit-panel refusal is not read as no input. + private func textInputQueries(app: XCUIApplication) throws -> [XCUIElementQuery] { + let inputTypes: [XCUIElement.ElementType] = [.textField, .secureTextField, .searchField, .textView] + return try inputTypes.flatMap { try elementQueries(app: app, matching: $0) } + } + private func queryTextInputs( - app: XCUIApplication, + queries: [XCUIElementQuery], point: CGPoint, contentTranslation: TextEntryContentTranslation, shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] - for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] { + for query in queries { if shouldStop() { break } let candidates = query.allElementsBoundByIndex #if AGENT_DEVICE_RUNNER_UNIT_TESTS diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 142f9c4415..7adcbac550 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -13,9 +13,9 @@ extension RunnerTests { xCTestChannelPenalized: Bool = false, synthesizer: any TextEntrySynthesizing, commandId: String? = nil - ) -> TextEntryResult { + ) throws -> TextEntryResult { let totalStartedAt = Date() - let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) } + let resolve = { (target: TextEntryTarget) in try self.resolveTextEntryInput(app: app, target: target) } var reader = TextEntryReader(target: target, resolve: resolve) guard !text.isEmpty else { // Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing @@ -23,7 +23,7 @@ extension RunnerTests { // place while the response reported "typed". Append (`type`) and unrepaired entry stay // no-ops, which is what an empty payload means for them. if repairMode == .replacement { - guard reader.input() != nil else { + guard try reader.input() != nil else { // No resolvable input means no clear happened — including the synthesized // first-responder route, whose target carries no element. Success here would claim a // clear the runner never performed. @@ -36,10 +36,10 @@ extension RunnerTests { failure: .notFocused ) } - clearTextInput(app: app, reader: &reader) + try clearTextInput(app: app, reader: &reader) // nil means the value is unreadable (secure fields), not "not empty": leave `verified` // nil there rather than reporting a mismatch the runner cannot actually observe. - let observed = editableTextValue(for: reader.input()?.attributes, treatingPlaceholderAsEmpty: true) + let observed = editableTextValue(for: try reader.input()?.attributes, treatingPlaceholderAsEmpty: true) logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode) return TextEntryResult( verified: observed.map { $0.isEmpty }, @@ -58,7 +58,7 @@ extension RunnerTests { xCTestChannelPenalized: xCTestChannelPenalized ) { - switch runSynthesizedReplacementRoute( + switch try runSynthesizedReplacementRoute( SynthesizedReplacementRequest( app: app, target: target, @@ -84,7 +84,7 @@ extension RunnerTests { softwareKeyboardVisible: isKeyboardVisible(app: app) ) let initialResolveStartedAt = Date() - let initialTarget = reader.input() + let initialTarget = try reader.input() let currentText = editableTextValue(for: initialTarget?.attributes, treatingPlaceholderAsEmpty: true) let initialText = repairMode == .append ? currentText : nil let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText) @@ -125,7 +125,7 @@ extension RunnerTests { } if currentText == nil || currentText?.isEmpty == false { let clearStartedAt = Date() - clearTextInput(app: app, reader: &reader) + try clearTextInput(app: app, reader: &reader) logTextEntryPhase( commandId: commandId, phase: "clear", @@ -134,7 +134,7 @@ extension RunnerTests { mode: repairMode ) } - entryBaseline = textEntryObservation(for: reader.input()) + entryBaseline = textEntryObservation(for: try reader.input()) } // The shape this command posts, decided once: a spaced per-character plan, a peeled warmup plus @@ -170,10 +170,10 @@ extension RunnerTests { return nil } - func typeIntoCurrentTarget(_ value: String) -> TextEntryFailure? { + func typeIntoCurrentTarget(_ value: String) throws -> TextEntryFailure? { #if os(iOS) if shouldUseSynthesizedFirstResponderType { - guard reader.input() != nil else { + guard try reader.input() != nil else { return .notFocused } textEntryRoute = "synthesized-first-responder" @@ -191,7 +191,7 @@ extension RunnerTests { } } #endif - if let currentTarget = reader.input() { + if let currentTarget = try reader.input() { textEntryRoute = "xctest-element" reader.dispatch { currentTarget.element.typeText(value) } return nil @@ -243,7 +243,7 @@ extension RunnerTests { reader.dispatch { sleepFor(TextEntryTiming.pollInterval) } } - func readBackWarmupCharacter(_ peeledCharacter: String) { + func readBackWarmupCharacter(_ peeledCharacter: String) throws { // A route with an element reads the peeled character back and waits up to `warmupValueTimeout` // for the app to accept it. The route that had to be budgeted has no element, so its expected // value is nil and this is the plan's one charged poll. @@ -258,7 +258,7 @@ extension RunnerTests { } let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout) while Date() < deadline { - let observed = textValueRead(for: reader.input()?.attributes) + let observed = textValueRead(for: try reader.input()?.attributes) if !observed.warmupReadIsPending(expectedText: warmupExpectedText) { if observed == .unreadable { pauseAfterWarmupRead() } return @@ -273,12 +273,12 @@ extension RunnerTests { var planFailure: TextEntryResult? var stepStartedAt = Date() let pacedStartedAt = Date() - runSynthesizedTextPlan( + try runSynthesizedTextPlan( synthesizedTypePlan, text: text, post: { slice, _ in stepStartedAt = Date() - if let failure = typeIntoCurrentTarget(slice) { + if let failure = try typeIntoCurrentTarget(slice) { planFailure = dispatchFailureResult(failure) return .stop } @@ -286,7 +286,7 @@ extension RunnerTests { }, waitAfterWarmupCharacter: { peeledCharacter in let warmupStartedAt = Date() - readBackWarmupCharacter(peeledCharacter) + try readBackWarmupCharacter(peeledCharacter) logTextEntryPhase( commandId: commandId, phase: "warmup", @@ -344,9 +344,9 @@ extension RunnerTests { let verifyStartedAt = Date() var result: TextEntryResult if repairMode == .replacement { - result = observeReplacementTextEntry(app: app, reader: &reader, requested: text, baseline: entryBaseline) + result = try observeReplacementTextEntry(app: app, reader: &reader, requested: text, baseline: entryBaseline) } else { - result = verifyAppendTextEntryWithRepairIfNeeded(app: app, reader: &reader, expectedText: expectedText) + result = try verifyAppendTextEntryWithRepairIfNeeded(app: app, reader: &reader, expectedText: expectedText) } logTextEntryPhase( commandId: commandId, @@ -393,12 +393,13 @@ extension RunnerTests { reader: inout TextEntryReader, requested: String, baseline: TextEntryObservation? - ) -> TextEntryResult { + ) throws -> TextEntryResult { var confirmation = ReplacementTextEntryConfirmation(requested: requested, baseline: baseline, startedAt: Date()) while true { - guard let input = reader.input() else { + guard let input = try reader.input() else { let target = reader.target - let failure: TextEntryFailure? = target.boundIdentity != nil && !boundTextEntryInputIsGone(app: app, target: target) + let inputIsGone = try boundTextEntryInputIsGone(app: app, target: target) + let failure: TextEntryFailure? = target.boundIdentity != nil && !inputIsGone ? .commitNotObserved : nil if target.boundIdentity != nil && failure == nil { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY") @@ -422,9 +423,9 @@ extension RunnerTests { app: XCUIApplication, reader: inout TextEntryReader, expectedText: String? - ) -> TextEntryResult { - if reader.input() == nil, reader.target.boundIdentity != nil { - guard boundTextEntryInputIsGone(app: app, target: reader.target) else { + ) throws -> TextEntryResult { + if try reader.input() == nil, reader.target.boundIdentity != nil { + guard try boundTextEntryInputIsGone(app: app, target: reader.target) else { return TextEntryResult( verified: nil, repaired: false, @@ -438,7 +439,7 @@ extension RunnerTests { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY") return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil) } - let initialResult = verifyTextEntry( + let initialResult = try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: false @@ -449,11 +450,11 @@ extension RunnerTests { else { return initialResult } - guard shouldRepairAppendTextEntry( + guard try shouldRepairAppendTextEntry( reader: &reader, expectedText: expectedText ) else { - return verifyTextEntry( + return try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: false @@ -461,14 +462,14 @@ extension RunnerTests { } guard Self.textEntryRepairCanTarget(boundIdentity: reader.target.boundIdentity) else { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_REPAIR_REFUSED reason=unidentified-input") - return verifyTextEntry( + return try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: false ) } - guard let repairTarget = reader.input() else { + guard let repairTarget = try reader.input() else { return initialResult } let observedText = editableTextValue(for: repairTarget.attributes) ?? "" @@ -477,9 +478,9 @@ extension RunnerTests { expectedText.count, observedText.count ) - clearTextInput(app: app, reader: &reader) + try clearTextInput(app: app, reader: &reader) // An input the clear removed or let another take the place of has nowhere for the repair to go. - guard let input = reader.input() else { + guard let input = try reader.input() else { return TextEntryResult( verified: nil, repaired: false, @@ -489,7 +490,7 @@ extension RunnerTests { ) } reader.dispatch { input.element.typeText(expectedText) } - return verifyTextEntry( + return try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: true @@ -503,8 +504,8 @@ extension RunnerTests { reader: inout TextEntryReader, expectedText: String?, repaired: Bool - ) -> TextEntryResult { - let targetInput = reader.input() + ) throws -> TextEntryResult { + let targetInput = try reader.input() guard let expectedText else { return TextEntryResult( verified: nil, @@ -532,7 +533,7 @@ extension RunnerTests { var latestObservedText = observedText while Date() < stableDeadline { reader.dispatch { sleepFor(TextEntryTiming.pollInterval) } - let nextInput = reader.input() + let nextInput = try reader.input() guard let nextObservedText = editableTextValue(for: nextInput?.attributes) else { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } @@ -593,13 +594,13 @@ extension RunnerTests { private func shouldRepairAppendTextEntry( reader: inout TextEntryReader, expectedText: String - ) -> Bool { + ) throws -> Bool { #if os(iOS) var latestObservedText: String? var latestElementType: XCUIElement.ElementType? let deadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow) repeat { - let input = reader.input() + let input = try reader.input() guard let observedText = editableTextValue(for: input?.attributes) else { return false } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift index b768656c87..17a05d9f6a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift @@ -60,7 +60,7 @@ extension RunnerTests { element: XCUIElement, action: String, resolvedFrame: CGRect? = nil - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { if let outcome = selectFocusedTvElement(app: app, element: element, action: action) { return outcome } @@ -72,7 +72,7 @@ extension RunnerTests { // XCUIElement.tap() can fail the whole XCTest after navigation because it // re-resolves the tapped element even after the app removed it. Keep the // selector target semantic, then activate its resolved stable screen point. - return tapAt(app: app, x: frame.midX, y: frame.midY) + return try tapAt(app: app, x: frame.midX, y: frame.midY) } return performElementTap(element) #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index c02e2096c7..7299948cef 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -3,7 +3,7 @@ import AgentDeviceSnapshotPresentation extension RunnerTests { @MainActor - func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response { + func executeTypeCommand(activeApp: XCUIApplication, command: Command) throws -> Response { guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "type requires text")) } @@ -33,7 +33,7 @@ extension RunnerTests { // answer both refuse here, each under its own reason. let contentTranslation = liveContentTranslation(app: activeApp) let inputAtPoint: TextInputAtPoint - switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { + switch try coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { case .input(let input): inputAtPoint = input case .absent: @@ -41,11 +41,8 @@ extension RunnerTests { case .unavailable: return textInputProbeUnavailableResponse() } - let policyKind = SynthesizedGesturePolicyKind.coordinateTap - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - }) { + let context = try synthesizedCoordinateContext(app: activeApp) + switch try performSynthesizedTapGesture(app: activeApp, x: x, y: y, context: context) { case .performed: resolvedCoordinateContext = context resolvedCoordinateTarget = TextEntryTarget( @@ -68,7 +65,7 @@ extension RunnerTests { if let resolvedCoordinateTarget { target = resolvedCoordinateTarget } else { - target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + target = try focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) } NSLog( "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=focus durationMs=%.1f chars=%d mode=%@", @@ -95,7 +92,7 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(message: message)) } } - let textResult = typeTextReliably( + let textResult = try typeTextReliably( app: activeApp, target: target, text: text, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 143ac75555..c4582e58b6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -89,6 +89,7 @@ final class RunnerTests: XCTestCase { let app = XCUIApplication() lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId) let mainOwned = RunnerMainOwnedState() + let litPanelState = RunnerLitPanelState() // Set while serving a command that had to re-activate the bound app, and stamped onto that // command's response before it leaves the execution queue (#2682). var pendingTargetActivation: TargetActivationFactPayload? @@ -202,6 +203,9 @@ final class RunnerTests: XCTestCase { /// Simulates an app tree read that returns no descendants without an issue or exception. var textInputProbeBlindForTesting = false + /// Records an XCTest issue during the lit-panel window walk, as a window that vanishes mid-walk does. + var litPanelWindowWalkIssueForTesting = false + static let injectedTapFailureFlagPathForTesting = "/tmp/agent-device-inject-tap-recorded-failure-for-testing" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 8e0c1c6b4c..dadac43409 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -29,8 +29,34 @@ typedef NS_ENUM(NSUInteger, RunnerApplicationScreenFailure) { RunnerApplicationScreenFailureUnresolvedWindow, // A resolved window whose screen or display identity the XCTest runtime would not answer. RunnerApplicationScreenFailureUnresolvedScreen, + // The application has windows, and every one of them is on a display other than the lit panel. + RunnerApplicationScreenFailureOffLitPanel, }; +// The lit panel (ADR 0025): its display and native box in points, sides swapped on a turned window. +typedef struct { + NSUInteger displayID; + CGFloat widthPt; + CGFloat heightPt; +} RunnerLitPanel; + +// The booking walk (`RunnerResolveLitPanelWindow`) reads at most this many windows: a vanished +// window's read records an XCTest issue (#2995), and SpringBoard lists about forty. +#define AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS 8 + +// Resolves the window on `litPanel`: the first window on that display whose frame is the panel's +// box either way up, else the largest window on that display. An unreadable list, an unplaced +// window, or a list past the cap with no full-panel window in it, is `UnresolvedScreen`. Windows +// that are all on another display are `OffLitPanel`; no window at all is `UnresolvedWindow`. The +// screen is the one the chosen window reports. +FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( + id application, + RunnerLitPanel litPanel, + id _Nullable *window, + CGRect *windowFrame, + id _Nullable *screen +); + // Resolves the shared core against the live XCTest runtime. Returns nil on success, or a // "private XCTest synthesis unavailable: ..." message when a class or selector the // private API is expected to expose is missing. `surface` names the calling feature. @@ -52,6 +78,11 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( RunnerApplicationScreenFailure *failure ); +// Every application window that reports `litPanel`'s display, in list order, uncapped so element +// search sees all of them; the caller holds the walk's XCTest issues. A window that is gone or names +// no display is left out. Nil when the window list itself cannot be read. +FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, NSUInteger litDisplayID); + // Reads the display ID off an already-resolved window, keeping a synthesized record's display and // the caller's booked reference frame on the same window. The frame is still read before the // screen; a window that answers empty or raises fails closed with a refusal message naming the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index b32a285818..f5f0583f18 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -109,6 +109,8 @@ return @"no resolved application window"; case RunnerApplicationScreenFailureUnresolvedScreen: return @"no resolved window display ID"; + case RunnerApplicationScreenFailureOffLitPanel: + return @"every application window is off the lit panel"; case RunnerApplicationScreenFailureNone: return @"application screen resolved"; } @@ -186,6 +188,156 @@ static RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( return RunnerApplicationScreenFailureNone; } +static BOOL RunnerFrameFillsLitPanel(CGRect frame, RunnerLitPanel panel) { + const CGFloat tolerance = 0.5; + if (fabs(frame.origin.x) > tolerance || fabs(frame.origin.y) > tolerance) { + return NO; + } + BOOL upright = fabs(frame.size.width - panel.widthPt) <= tolerance && + fabs(frame.size.height - panel.heightPt) <= tolerance; + BOOL turned = fabs(frame.size.width - panel.heightPt) <= tolerance && + fabs(frame.size.height - panel.widthPt) <= tolerance; + return upright || turned; +} + +// One window: whether it is there, its frame, and the display showing it. `frame` is not asked of a +// window that says it is gone, because that read records an XCTest issue (#2995). +static RunnerApplicationScreenFailure RunnerReadLitPanelCandidate( + id candidate, + CGRect *frame, + id _Nullable *screen, + NSUInteger *displayID +) { + __block CGRect readFrame = CGRectNull; + __block BOOL present = NO; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + NSNumber *exists = [candidate valueForKey:@"exists"]; + present = [exists isKindOfClass:NSNumber.class] && exists.boolValue; + if (present) { + [[candidate valueForKey:@"frame"] getValue:&readFrame size:sizeof(readFrame)]; + } + }]; + *frame = readFrame; + if (exception != nil) { + return RunnerApplicationScreenFailureUnresolvedScreen; + } + if (!present || !AgentDeviceForegroundFrameIsMeasurable(readFrame)) { + return RunnerApplicationScreenFailureUnresolvedWindow; + } + return RunnerReadResolvedWindowScreen(candidate, readFrame, screen, displayID); +} + +// The application's window list, or nil when the query raises or answers no list. +static NSArray *_Nullable RunnerCopyWindowList(id application) { + __block NSArray *windows = nil; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + id elements = [[application valueForKey:@"windows"] valueForKey:@"allElementsBoundByIndex"]; + windows = [elements isKindOfClass:NSArray.class] ? elements : nil; + }]; + return exception == nil ? windows : nil; +} + +RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( + id application, + RunnerLitPanel litPanel, + id _Nullable *window, + CGRect *windowFrame, + id _Nullable *screen +) { + *window = nil; + *windowFrame = CGRectNull; + *screen = nil; + // An unread list is not an app without windows, so it never reaches the system-surface fallback. + NSArray *windows = RunnerCopyWindowList(application); + if (windows == nil) { + return RunnerApplicationScreenFailureUnresolvedScreen; + } + // A window the walk could not place may be the panel's surface, so it refuses every booking. A + // list past the cap refuses only when no full-panel window was found before it. + BOOL unplaced = NO; + BOOL truncated = windows.count > AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS; + BOOL placedAny = NO; + id chosen = nil; + id chosenScreen = nil; + CGRect chosenFrame = CGRectNull; + RunnerApplicationScreenFailure outcome = RunnerApplicationScreenFailureUnresolvedWindow; + NSUInteger count = MIN(windows.count, (NSUInteger)AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS); + for (NSUInteger index = 0; index < count; index += 1) { + id candidate = windows[index]; + CGRect frame = CGRectNull; + id candidateScreen = nil; + NSUInteger displayID = 0; + RunnerApplicationScreenFailure read = + RunnerReadLitPanelCandidate(candidate, &frame, &candidateScreen, &displayID); + if (read == RunnerApplicationScreenFailureUnresolvedWindow) { + continue; + } + if (read != RunnerApplicationScreenFailureNone) { + unplaced = YES; + continue; + } + placedAny = YES; + if (displayID != litPanel.displayID) { + continue; + } + if (RunnerFrameFillsLitPanel(frame, litPanel)) { + chosen = candidate; + chosenScreen = candidateScreen; + chosenFrame = frame; + truncated = NO; + break; + } + if (chosen == nil || + frame.size.width * frame.size.height > chosenFrame.size.width * chosenFrame.size.height) { + chosen = candidate; + chosenScreen = candidateScreen; + chosenFrame = frame; + } + } + if (unplaced || truncated) { + outcome = RunnerApplicationScreenFailureUnresolvedScreen; + } else if (chosen != nil) { + outcome = RunnerApplicationScreenFailureNone; + *window = chosen; + *windowFrame = chosenFrame; + *screen = chosenScreen; + } else if (placedAny) { + outcome = RunnerApplicationScreenFailureOffLitPanel; + } + NSLog( + @"AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=%lu(%gx%g) windows=%lu outcome=%@ frame=(%g,%g %gx%g)", + (unsigned long)litPanel.displayID, + litPanel.widthPt, + litPanel.heightPt, + (unsigned long)windows.count, + RunnerApplicationScreenFailureDescription(outcome), + chosenFrame.origin.x, + chosenFrame.origin.y, + chosenFrame.size.width, + chosenFrame.size.height + ); + return outcome; +} + +NSArray *RunnerCopyLitPanelWindows(id application, NSUInteger litDisplayID) { + NSArray *windows = RunnerCopyWindowList(application); + if (windows == nil) { + return nil; + } + NSMutableArray *lit = [NSMutableArray array]; + for (id candidate in windows) { + CGRect frame = CGRectNull; + id screen = nil; + NSUInteger displayID = 0; + if (RunnerReadLitPanelCandidate(candidate, &frame, &screen, &displayID) == + RunnerApplicationScreenFailureNone && + displayID == litDisplayID) { + [lit addObject:candidate]; + } + } + return lit; +} + BOOL RunnerResolveApplicationScreen( id application, id _Nullable *screen, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift new file mode 100644 index 0000000000..4e832de878 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -0,0 +1,649 @@ +import AgentDeviceSnapshotPresentation +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private final class LitPanelBookingScreen: NSObject { + @objc let displayID: UInt + + init(_ displayID: UInt = 3) { self.displayID = displayID } +} + +@MainActor +private final class LitPanelBookingProbe: NSObject { + static var originalValueForKey: IMP? + static let query = NSObject() + static var windowListReads = 0 + static var maxWindowListReads: Int? + static var window: XCUIElement? + static var darkWindow: XCUIElement? + static var coordinate: XCUICoordinate? + static var windowFrame = CGRect.zero + static var orientation = RunnerInterfaceOrientation.landscapeLeft + static var anchor: NSObject? + static var point = CGPoint.zero + static var tappedPoint: CGPoint? + + @objc class func interfaceOrientation(forApplication application: AnyObject) -> Int { orientation } + + nonisolated override func value(forKey key: String) -> Any? { + MainActor.assumeIsolated { + switch key { + case "windows": return Self.query + case "allElementsBoundByIndex": + Self.windowListReads += 1 + if let max = Self.maxWindowListReads, Self.windowListReads > max { return [] } + return [Self.window!, Self.darkWindow].compactMap { $0 } + case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : 3) + case "exists": return true + case "frame": return NSValue(cgRect: Self.windowFrame) + default: break + } + let original = unsafeBitCast( + Self.originalValueForKey!, + to: (@convention(c) (AnyObject, Selector, NSString) -> AnyObject?).self + ) + return original(self, #selector(NSObject.value(forKey:)), key as NSString) + } + } + + @objc(coordinateWithNormalizedOffset:) + func coordinateWithNormalizedOffset(_ offset: CGVector) -> XCUICoordinate { + Self.anchor = self + Self.point = CGPoint( + x: Self.windowFrame.minX + offset.dx * Self.windowFrame.width, + y: Self.windowFrame.minY + offset.dy * Self.windowFrame.height + ) + return Self.coordinate! + } + + @objc(coordinateWithOffset:) + func coordinateWithOffset(_ offset: CGVector) -> XCUICoordinate { + Self.point.x += offset.dx + Self.point.y += offset.dy + return Self.coordinate! + } + + @objc func tap() { Self.tappedPoint = Self.point } +} + +extension RunnerTests { + @MainActor + func testLitPanelBookingAndCoordinateFallbackShareScreenPointsForPartialWindows() throws { + try withLitPanelBookingProbe { window in + for (frame, orientation, expected) in [ + ( + CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, + CGRect(x: 0, y: 0, width: 951, height: 669) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, + CGRect(x: 0, y: 0, width: 669, height: 951) + ), + ( + CGRect(x: 80, y: 0, width: 180, height: 669), RunnerInterfaceOrientation.landscapeRight, + CGRect(x: 0, y: 0, width: 951, height: 669) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, + CGRect(x: 0, y: 0, width: 669, height: 951) + ), + ] { + LitPanelBookingProbe.windowFrame = frame + LitPanelBookingProbe.orientation = orientation + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.frame, frame) + XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: booked.frame, interfaceOrientation: orientation), expected) + if case .unsupported(let message, _) = try tapAt(app: app, x: 200, y: 150) { XCTFail(message) } + XCTAssertTrue(LitPanelBookingProbe.anchor === window) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) + } + XCTAssertNil(LitPanelBookingProbe.tappedPoint) + + } + } + + @MainActor + func testCollapsedTabCandidatesExcludeOverlappingDarkWindowsAndKeepTheCaptureScope() throws { + try withLitPanelBookingProbe { window in + let darkWindow = XCUIApplication(bundleIdentifier: "com.apple.springboard").windows.firstMatch + LitPanelBookingProbe.darkWindow = darkWindow + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 0, width: 669, height: 951) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"snapshot","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let container = ElementAttributesFixture(.tabBar, CGRect(x: 0, y: 800, width: 669, height: 31)) + let context = SnapshotTraversalContext( + queryRoot: app, + rootSnapshot: ElementAttributesFixture(.application, LitPanelBookingProbe.windowFrame, children: [ + ElementAttributesFixture(.window, LitPanelBookingProbe.windowFrame, children: [container], displayID: 3), + ]), + rootReadDuration: 0, + viewport: .missing(reason: .notProvided), + keyboardBand: nil, + litPanelSurface: try XCTUnwrap(litPanelState.surface) + ) + let lit = ElementAttributesFixture(.button, CGRect(x: 30, y: 800, width: 100, height: 30), label: "Lit") + let second = ElementAttributesFixture(.button, CGRect(x: 170, y: 800, width: 100, height: 30), label: "Second") + let dark = ElementAttributesFixture(.button, lit.frame, label: "Dark") + let appQuery = app.descendants(matching: .any) + let litQuery = window.descendants(matching: .any) + let darkQuery = darkWindow.descendants(matching: .any) + var predicates: [ObjectIdentifier: NSPredicate] = [:] + var restores: [() -> Void] = [] + defer { restores.reversed().forEach { $0() } } + func swap(_ type: AnyClass, _ name: String, _ block: Any) throws { + let method = try XCTUnwrap(class_getInstanceMethod(type, NSSelectorFromString(name))) + let replacement = imp_implementationWithBlock(block) + let original = method_setImplementation(method, replacement) + restores.append { + method_setImplementation(method, original) + imp_removeBlock(replacement) + } + } + try swap(XCUIElement.self, "descendantsMatchingType:", { (root: AnyObject, _: UInt) -> XCUIElementQuery in + root === window ? litQuery : root === darkWindow ? darkQuery : appQuery + } as @convention(block) (AnyObject, UInt) -> XCUIElementQuery) + try swap(XCUIElementQuery.self, "matchingPredicate:", { (query: AnyObject, predicate: NSPredicate) -> XCUIElementQuery in + predicates[ObjectIdentifier(query)] = predicate + return query as! XCUIElementQuery + } as @convention(block) (AnyObject, NSPredicate) -> XCUIElementQuery) + try swap(XCUIElementQuery.self, "allElementsBoundByIndex", { (query: AnyObject) -> NSArray in + let attributes = query === litQuery ? [lit, second] : query === darkQuery ? [dark] : [lit, second, dark] + let matches = attributes.filter { predicates[ObjectIdentifier(query)]?.evaluate(with: $0) == true } + return matches.isEmpty ? [] : [window] + } as @convention(block) (AnyObject) -> NSArray) + let candidates = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) + XCTAssertEqual(candidates.map(\.label), ["Lit", "Second"], "an overlapping control on a dark window must not enter the capture") + let hint = CaptureHint(projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false) + let capture = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: .distantFuture) + let labels = capture.acquisition.nodes.compactMap(\.label) + XCTAssertTrue(labels.contains("Lit")) + XCTAssertTrue(labels.contains("Second")) + XCTAssertFalse(labels.contains("Dark"), "the serialized capture must exclude the overlapping dark control") + endRequestLitPanel(try XCTUnwrap(litPanelState.requestID)) + let pinned = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) + XCTAssertEqual(pinned.map(\.label), ["Lit", "Second"], "the capture's display scope survives a cleared current request") + var unscoped = context + unscoped.litPanelSurface = nil + let all = try XCTUnwrap(try collapsedTabCandidates(context: unscoped, processingDeadline: .distantFuture)) + XCTAssertEqual(all.map(\.label), ["Lit", "Second", "Dark"], "an unscoped capture keeps application-wide candidates") + litPanelWindowWalkIssueForTesting = true + defer { litPanelWindowWalkIssueForTesting = false } + XCTAssertThrowsError(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + } + } + + @MainActor + func testLitPanelBookingUsesBundleIdentityAcrossProxiesAndRequests() throws { + try withLitPanelBookingProbe { window in + let bundleID = try XCTUnwrap(app.value(forKey: "bundleID") as? String) + let sameApp = XCUIApplication(bundleIdentifier: bundleID) + let otherApp = XCUIApplication(bundleIdentifier: "com.apple.springboard") + let secondWindow = otherApp.windows.firstMatch + let command = try runnerCommandFixture(#"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"#) + LitPanelBookingProbe.windowFrame = CGRect(x: 80, y: 0, width: 871, height: 669) + let firstRequest = setRequestLitPanel(from: command) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let firstReads = LitPanelBookingProbe.windowListReads + XCTAssertNil(bookRequestLitPanelSurface(app: sameApp)) + XCTAssertEqual(LitPanelBookingProbe.windowListReads, firstReads, "a second proxy of the same bundle reuses this request's booking") + LitPanelBookingProbe.window = secondWindow + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + XCTAssertNil(bookRequestLitPanelSurface(app: otherApp)) + XCTAssertTrue(try requireRequestLitPanelSurface(app: otherApp).window === secondWindow) + XCTAssertGreaterThan(LitPanelBookingProbe.windowListReads, firstReads) + LitPanelBookingProbe.window = window + let nextRequest = setRequestLitPanel(from: command) + endRequestLitPanel(firstRequest) + XCTAssertNil(bookRequestLitPanelSurface(app: sameApp)) + XCTAssertTrue(try requireRequestLitPanelSurface(app: app).window === window) + XCTAssertEqual(litPanelState.surface?.key.requestID, nextRequest) + endRequestLitPanel(nextRequest) + XCTAssertNil(litPanelState.surface) + XCTAssertNil(requestLitPanel) + } + } + + @MainActor + func testAcceptedRequestEndsItsLitPanelBooking() throws { + try withLitPanelBookingProbe { _ in + let command = try runnerCommandFixture(#"{"command":"uptime","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"#) + _ = try executeAccepted(command: command) + XCTAssertNil(requestLitPanel, "finished commands cannot leave a panel for unrelated main work") + XCTAssertNil(litPanelState.surface) + } + } + + @MainActor + func testDuplicateSelectorRefusesWithoutTryingToBookATapPoint() throws { + app.launchArguments = ["--agent-device-selector-read-regression"] + try withLitPanelBookingProbe { _ in + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 0, width: 669, height: 951) + LitPanelBookingProbe.maxWindowListReads = 1 + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + let match = try findElement(app: app, selectorKey: "id", selectorValue: "agent-device-selector-read-duplicate") + XCTAssertTrue(match.isAmbiguous) + XCTAssertNil(match.element) + XCTAssertEqual(LitPanelBookingProbe.windowListReads, 1, "ambiguity does not require a coordinate booking") + } + } + + @MainActor + func testFailedLitPanelBookingRefusesEveryReferenceFrameConsumer() throws { + try withLitPanelBookingProbe { _ in + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 0, width: 669, height: 951) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + func refuses(_ read: () throws -> T) { + XCTAssertThrowsError(try read()) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) + } + } + refuses { try resolveRunnerWindow(app: app) } + refuses { try onScreenWindowFrame(app: app) } + refuses { try resolvedTouchReferenceFrame(app: app, appFrame: CGRect(x: 0, y: 0, width: 466, height: 678)) } + refuses { try synthesizedCoordinateContext(app: app) } + refuses { try gestureViewportFrame(app: app) } + refuses { try resolvedTouchVisualizationFrame(app: app, x: 20, y: 30) } + refuses { try resolvedDragVisualizationFrame(app: app, x: 20, y: 30, x2: 40, y2: 50) } + refuses { try performSynthesizedTapGesture(app: app, x: 20, y: 30) } + var alertRefusal: Error? + let held = holdingRecordedIssues { + do { + _ = try activateAlertButton( + RunnerAlert(root: app, ownerApp: app, buttons: [app], source: .appAlert, window: nil), + button: app, action: "accept", frame: CGRect(x: 10, y: 10, width: 40, height: 40), deadline: .distantFuture + ) + } catch { alertRefusal = error } + } + XCTAssertEqual(alertRefusal as? RunnerAppScreenCaptureFailure, .offLitPanel) + XCTAssertTrue(held.issues.isEmpty, "the typed refusal must survive the idle/interruption wrappers") + } + } + + @MainActor + private func withLitPanelBookingProbe(_ body: (XCUIElement) throws -> Void) throws { + app.launch() + defer { app.terminate() } + let query = app.windows + let window = query.firstMatch + LitPanelBookingProbe.window = window + LitPanelBookingProbe.coordinate = window.coordinate(withNormalizedOffset: .zero) + let windowClass = try XCTUnwrap(object_getClass(window)) + let coordinateClass = try XCTUnwrap(object_getClass(LitPanelBookingProbe.coordinate!)) + let valueForKey = try XCTUnwrap(class_getInstanceMethod(windowClass, #selector(NSObject.value(forKey:)))) + LitPanelBookingProbe.originalValueForKey = method_getImplementation(valueForKey) + let swaps: [(Method?, Method?)] = [ + (valueForKey, + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(NSObject.value(forKey:)))), + (class_getClassMethod(RunnerSynthesizedGesture.self, #selector(RunnerSynthesizedGesture.interfaceOrientation(forApplication:))), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.interfaceOrientation(forApplication:)))), + (class_getInstanceMethod(windowClass, #selector(XCUIElement.coordinate(withNormalizedOffset:))), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.coordinateWithNormalizedOffset(_:)))), + (class_getInstanceMethod(coordinateClass, #selector(XCUICoordinate.withOffset(_:))), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.coordinateWithOffset(_:)))), + (class_getInstanceMethod(coordinateClass, #selector(XCUICoordinate.tap)), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.tap))), + ] + var originals: [(Method, IMP)] = [] + defer { + for (method, original) in originals.reversed() { method_setImplementation(method, original) } + LitPanelBookingProbe.darkWindow = nil + LitPanelBookingProbe.window = nil + LitPanelBookingProbe.coordinate = nil + LitPanelBookingProbe.anchor = nil + LitPanelBookingProbe.originalValueForKey = nil + LitPanelBookingProbe.maxWindowListReads = nil + litPanelState.panel = nil + litPanelState.surface = nil + } + for (method, stub) in swaps { + let method = try XCTUnwrap(method) + let stub = try XCTUnwrap(stub) + originals.append((method, method_setImplementation(method, method_getImplementation(stub)))) + } + LitPanelBookingProbe.windowListReads = 0 + try body(window) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + /// One runner process can serve a foldable's `tap` and then a single-panel request; the second + /// must not resolve against the first's panel. + func testRequestPanelFollowsTheCommandAndAnUnnamedPanelClearsIt() throws { + setRequestLitPanel(from: try command(namingLitPanel: #"{"displayId":3,"widthPt":669,"heightPt":951}"#)) + XCTAssertEqual(requestLitPanel?.displayID, 3) + XCTAssertEqual(requestLitPanel?.widthPt, 669) + XCTAssertEqual(requestLitPanel?.heightPt, 951) + + setRequestLitPanel(from: try command(namingLitPanel: nil)) + XCTAssertNil(requestLitPanel) + } + + func testRequestPanelTreatsAPanelNamingNoUsableBoxAsNoPanel() throws { + for panel in [ + #"{"displayId":0,"widthPt":669,"heightPt":951}"#, + #"{"displayId":3,"widthPt":0,"heightPt":951}"#, + #"{"displayId":3,"widthPt":669,"heightPt":-951}"#, + ] { + setRequestLitPanel(from: try command(namingLitPanel: panel)) + XCTAssertNil(requestLitPanel, panel) + } + } + + /// Main work a command abandoned runs before the next command's panel is written, so it still + /// reads its own command's panel. + func testRequestLitPanelIsWrittenBehindMainWorkAlreadyQueued() throws { + let unnamed = try command(namingLitPanel: nil) + let named = try command(namingLitPanel: #"{"displayId":3,"widthPt":669,"heightPt":951}"#) + setRequestLitPanel(from: unnamed) + let drained = expectation(description: "main queue drained") + var queuedEarlier: UInt? + var queuedLater: UInt? + DispatchQueue.global().async { + DispatchQueue.main.async { queuedEarlier = self.requestLitPanel?.displayID ?? 0 } + self.setRequestLitPanel(from: named) + DispatchQueue.main.async { + queuedLater = self.requestLitPanel?.displayID ?? 0 + drained.fulfill() + } + } + wait(for: [drained], timeout: 5) + setRequestLitPanel(from: unnamed) + XCTAssertEqual(queuedEarlier, 0) + XCTAssertEqual(queuedLater, 3) + } + + func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() throws { + func refusal(_ fields: String) throws -> ErrorPayload? { + try Self.litPanelBackendPinRefusal(runnerCommandFixture(#"{"command":"snapshot"\#(fields)}"#))?.error + } + XCTAssertEqual(try refusal(#","preferredBackend":"private-ax""#)?.code, "UNSUPPORTED_OPERATION") + XCTAssertEqual(try refusal(#","preferredBackend":"queries""#)?.code, "UNSUPPORTED_OPERATION") + XCTAssertNil(try refusal(#","preferredBackend":"tree""#)) + XCTAssertNil(try refusal("")) + let implied = try XCTUnwrap(refusal(#","customActions":true"#)) + XCTAssertEqual(implied.code, "UNSUPPORTED_OPERATION") + XCTAssertTrue(implied.message.contains("customActions"), implied.message) + XCTAssertFalse(implied.hint?.contains("backend pin") ?? true, implied.hint ?? "") + } + + /// Decoded from JSON, so a renamed wire field fails here. + private func command(namingLitPanel panel: String?) throws -> Command { + let fields = ["\"command\":\"tap\"", panel.map { "\"litPanel\":\($0)" }].compactMap { $0 } + return try JSONDecoder().decode( + Command.self, + from: Data("{\(fields.joined(separator: ","))}".utf8) + ) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + /// A window whose display the runtime does not name may be on the dark panel, so it is dropped. + func testLitPanelSnapshotKeepsOnlyWindowsProvenOnTheLitDisplay() { + let windows: [(name: String, displayID: UInt?)] = [("outer", 1), ("unknown", nil), ("inner", 3)] + let kept = Self.windowsOnLitDisplay(windows, litDisplayID: 3) { $0.displayID } + XCTAssertEqual(kept.map(\.name), ["inner"]) + } + +#if os(iOS) + /// A window that leaves part of the lit panel uncovered, like an app beside the Duo's 80 pt status + /// strip, books its own frame as the reference, and a synthesized turn is about the whole panel. + @MainActor + func testLitPanelBookingTurnsAboutTheWholePanelForAPartialWindow() throws { + let (_, frame) = launchAppOnTheLitPanel() + defer { tearDownLitPanelApp() } + let window = app.windows.firstMatch + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":\#(displayID),"widthPt":\#(frame.width),"heightPt":\#(frame.height + 80)}}"# + )) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let context = try XCTUnwrap(try synthesizedCoordinateContext(app: app)) + XCTAssertEqual(context.referenceFrame, frame, "the booked reference is the window's own frame") + XCTAssertEqual( + synthesizedRotationFrame(app: app, fallback: context.referenceFrame, interfaceOrientation: RunnerInterfaceOrientation.portrait), + CGRect(x: 0, y: 0, width: frame.width, height: frame.height + 80), + "the turn is about the whole lit panel, not the window" + ) + } + /// Launches the app and names its own window's display and box as the lit panel: the + /// `litPanel` JSON member, and the window frame. + @MainActor + private func launchAppOnTheLitPanel() -> (panel: String, frame: CGRect) { + app.launch() + mainOwned.app = app + let window = app.windows.firstMatch + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) + let frame = window.frame + return (#""litPanel":{"displayId":\#(displayID),"widthPt":\#(frame.width),"heightPt":\#(frame.height)}"#, frame) + } + + @MainActor + private func tearDownLitPanelApp() { + litPanelWindowWalkIssueForTesting = false + setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + /// Runs `command` against the launched app while the request names display 999, which none of + /// the app's windows is on. + @MainActor + private func responseWithTheAppOffTheLitPanel( + _ json: String, + _ run: (Command) throws -> Response + ) throws -> Response { + app.launch() + mainOwned.app = app + defer { tearDownLitPanelApp() } + let command = try runnerCommandFixture(json) + setRequestLitPanel(from: command) + return try run(command) + } + + @MainActor + func testCoordinateTapRefusesBeforeDispatchWhenTheAppIsOffTheLitPanel() throws { + var fellBack = true + let response = try responseWithTheAppOffTheLitPanel( + #"{"command":"tap","x":40,"y":120,"synthesized":true,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + ) { command in + let response = try executeOnMainPrepared(command: command, activeApp: app) + fellBack = resolveRunnerWindowElement(app: app) != nil + return response + } + XCTAssertEqual(response.error?.code, "APP_SCREEN_OFF_LIT_PANEL") + XCTAssertFalse(fellBack, "no read in the request may fall back to the app") + } + + /// The app's window is booked on the lit panel, and then a window vanishes during the element + /// search's walk: the issue is held, the read refuses with the typed code instead of finding + /// nothing, and a coordinate `type` refuses before it taps a point with no target. + @MainActor + func testElementReadsRefuseWhenTheLitPanelWindowWalkFails() throws { + let (panel, frame) = launchAppOnTheLitPanel() + defer { tearDownLitPanelApp() } + for json in [ + #"{"command":"readText","x":\#(frame.midX),"y":\#(frame.midY),\#(panel)}"#, + #"{"command":"type","text":"x","x":\#(frame.midX),"y":\#(frame.midY),\#(panel)}"#, + ] { + let command = try runnerCommandFixture(json) + setRequestLitPanel(from: command) + XCTAssertNil(bookRequestLitPanelSurface(app: app), "the window is on the lit panel") + litPanelWindowWalkIssueForTesting = true + let response = try executeOnMainPrepared(command: command, activeApp: app) + litPanelWindowWalkIssueForTesting = false + XCTAssertEqual(response.error?.code, "APP_SCREEN_UNRESOLVED", json) + } + } + + /// A target bound to the regression fixture's field by identifier, refreshed at its center. + @MainActor + private func boundFixtureTarget() throws -> TextEntryTarget { + let identifier = "agent-device-hardware-keyboard-input" + let field = app.textFields[identifier] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + let frame = field.frame + return TextEntryTarget( + element: nil, + refreshPoint: CGPoint(x: frame.midX, y: frame.midY), + prefersFocusedElement: false, + contentTranslation: .none, + boundIdentity: TextEntryInputIdentity(elementType: .textField, identifier: identifier) + ) + } + + /// A bound input re-read on a lit-panel request whose walk fails propagates the refusal, and the + /// input cannot be proven gone. + @MainActor + func testBoundInputReReadRefusesWhenTheLitPanelWalkFails() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + let (panel, _) = launchAppOnTheLitPanel() + defer { + app.launchArguments = [] + tearDownLitPanelApp() + } + let target = try boundFixtureTarget() + let command = try runnerCommandFixture(#"{"command":"type",\#(panel)}"#) + setRequestLitPanel(from: command) + XCTAssertNotNil(try resolveTextEntryInput(app: app, target: target), "the walk answers") + litPanelWindowWalkIssueForTesting = true + XCTAssertThrowsError(try resolveTextEntryInput(app: app, target: target)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + XCTAssertThrowsError(try clearedFieldRead(app: app, target: target), "a clear pass read propagates it too") { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + XCTAssertThrowsError(try boundTextEntryInputIsGone(app: app, target: target)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + } + + /// A bound input read refuses when every window is off the named panel, even if the app-wide + /// query can resolve the same input. + @MainActor + func testBoundInputIdentifierReadRefusesWhenAllWindowsAreOffTheLitPanel() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + mainOwned.app = app + defer { + app.launchArguments = [] + tearDownLitPanelApp() + } + let target = try boundFixtureTarget() + setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"type"}"#)) + XCTAssertNotNil(try resolveTextEntryInput(app: app, target: target), "the app-wide query finds it") + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"type","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + let point = try XCTUnwrap(target.refreshPoint) + XCTAssertThrowsError(try probeTextInputs(app: app, point: point, contentTranslation: .none)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + XCTAssertThrowsError(try resolveTextEntryInput(app: app, target: target)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + } + + /// With a lit panel named, selector and point reads search only the windows on that display; the + /// launched app's windows are all on another one, so neither read may answer. + @MainActor + func testElementReadsSearchOnlyTheLitDisplaysWindows() throws { + _ = try responseWithTheAppOffTheLitPanel( + #"{"command":"tap","x":1,"y":1,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + ) { _ in + XCTAssertThrowsError(try findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner")) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"tap"}"#)) + let match = try findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") + let element = try XCTUnwrap(match.element) + let frame = element.frame + XCTAssertNotNil(try readTextAt(app: app, x: frame.midX, y: frame.midY)) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try readTextAt(app: app, x: frame.midX, y: frame.midY)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + return Response(ok: true) + } + } + + @MainActor + func testCollapsedTabSnapshotReadsProvenLitLiveWindows() throws { + app.launchArguments = ["--agent-device-collapsed-tab-bar", "--agent-device-wide-slider"] + let (panel, _) = launchAppOnTheLitPanel() + defer { tearDownLitPanelApp() } + setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"snapshot",\#(panel)}"#)) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let hint = CaptureHint(projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false) + let context = try XCTUnwrap(try makeSnapshotTraversalContext( + app: app, + hint: hint, + litPanelSurface: try XCTUnwrap(litPanelState.surface) + )) + let container = try XCTUnwrap(Self.snapshotDescendants(of: context.rootSnapshot).first { $0.elementType == .tabBar }) + XCTAssertTrue(container.children.isEmpty, "the live fixture must reach collapsed-tab recovery") + let candidates = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) + XCTAssertTrue(candidates.contains { $0.label == "Left" }) + XCTAssertTrue(candidates.contains { $0.label == "Right" }) + let capture = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: .distantFuture) + let labels = capture.acquisition.nodes.compactMap(\.label) + XCTAssertTrue(labels.contains("Left")) + XCTAssertTrue(labels.contains("Right")) + } + + /// A refused pin is a request refusal: it never reaches the capture plan, so it does not + /// invalidate the cached target the next command relies on. + @MainActor + func testLitPanelSnapshotPinRefusalKeepsTheCachedTarget() throws { + let (panel, _) = launchAppOnTheLitPanel() + mainOwned.bundleId = "com.callstack.agentdevice.runner" + defer { tearDownLitPanelApp() } + let command = try runnerCommandFixture( + #"{"appBundleId":"com.callstack.agentdevice.runner","command":"snapshot","preferredBackend":"private-ax",\#(panel)}"# + ) + setRequestLitPanel(from: command) + let response = try executeSnapshotDispatched(command: command) + XCTAssertEqual(response.error?.code, "UNSUPPORTED_OPERATION") + XCTAssertTrue(mainOwned.app === app, "a refused pin must not invalidate the cached target") + } + + @MainActor + func testSnapshotRefusesWhenTheAppIsOffTheLitPanel() throws { + let response = try responseWithTheAppOffTheLitPanel( + #"{"command":"snapshot","interactiveOnly":true,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + ) { try executeSnapshotDispatched(command: $0) } + XCTAssertEqual(response.error?.code, "APP_SCREEN_OFF_LIT_PANEL") + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index b741c7d931..380c8c9199 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -170,7 +170,7 @@ extension RunnerTests { let alert = try requireResolvedAlert(within: RunnerTests.alertResolutionAllowance) XCTAssertEqual(alert.buttons.map(\.label).last, "Allow", "the fixture keeps the affirmative button last") - let response = handleAlert( + let response = try handleAlert( alert, action: "dismiss", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline) @@ -194,7 +194,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try requireResolvedAlert(within: RunnerTests.alertResolutionAllowance) - let response = handleAlert( + let response = try handleAlert( alert, action: "dismiss", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index f764124a0a..de2db7af2c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -45,15 +45,9 @@ extension RunnerTests { @MainActor func testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let alert = try resolveAlertBeforeTheCommand() - let response = handleAlert(alert, action: "accept", deadline: .distantPast) + defer { tearDownReplacementAlert() } + let alert = try launchReplacementAlert() + let response = try handleAlert(alert, action: "accept", deadline: .distantPast) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") XCTAssertTrue(app.alerts.firstMatch.exists) @@ -77,7 +71,7 @@ extension RunnerTests { } return true } - let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(1)) + let response = try handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(1)) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") XCTAssertTrue(app.alerts.firstMatch.exists) @@ -86,34 +80,22 @@ extension RunnerTests { @MainActor func testAlertActivationAfterDeadlineDoesNotTapTheOriginal() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let alert = try resolveAlertBeforeTheCommand() + defer { tearDownReplacementAlert() } + let alert = try launchReplacementAlert() let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame - let outcome = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantPast) + let outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantPast) XCTAssertNil(outcome, "an expired command must not synthesize a tap") - XCTAssertNil(activateAlertButton(alert, button: button, action: "accept", frame: .zero, deadline: .distantFuture)) + XCTAssertNil(try activateAlertButton(alert, button: button, action: "accept", frame: .zero, deadline: .distantFuture)) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") } @MainActor func testAlertActivationRefusesAnUnreadableHostWindowWithoutTapping() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let resolved = try resolveAlertBeforeTheCommand() + defer { tearDownReplacementAlert() } + let resolved = try launchReplacementAlert() let alert = RunnerAlert( root: resolved.root, ownerApp: resolved.ownerApp, @@ -127,8 +109,8 @@ extension RunnerTests { let options = XCTExpectedFailure.Options() options.issueMatcher = { $0.compactDescription.contains("agent-device-missing-window") } var outcome: RunnerInteractionOutcome? - XCTExpectFailure("The unreadable window is the condition under test", options: options) { - outcome = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) + try XCTExpectFailure("The unreadable window is the condition under test", options: options) { + outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) } guard case .unsupported? = outcome else { @@ -139,39 +121,24 @@ extension RunnerTests { @MainActor func testAlertDeadlineDuringSynthesisPreparationLeavesTheOriginalUntouched() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let alert = try resolveAlertBeforeTheCommand() + defer { tearDownReplacementAlert() } + let alert = try launchReplacementAlert() let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") - let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) - let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector)) - let original = method_getImplementation(method) - method_setImplementation(method, method_getImplementation(stub)) - defer { - AlertTapDeadlineStub.status = nil - AlertTapDeadlineStub.interruptionHandlingDisabled = nil - method_setImplementation(method, original) - } + try withAlertTapStub { + let outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) - let outcome = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) + XCTAssertNil(outcome) + XCTAssertTrue(app.alerts.firstMatch.exists) + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") - XCTAssertNil(outcome) - XCTAssertTrue(app.alerts.firstMatch.exists) - XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") - - AlertTapDeadlineStub.status = RunnerTapSynthesisStatus(rawValue: 999) - let unknown = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) - guard case .unsupported? = unknown else { - return XCTFail("an unknown native status must report failure rather than deadline expiry") + AlertTapDeadlineStub.status = RunnerTapSynthesisStatus(rawValue: 999) + let unknown = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) + guard case .unsupported? = unknown else { + return XCTFail("an unknown native status must report failure rather than deadline expiry") } XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") + } } @MainActor @@ -189,7 +156,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try resolveAlertBeforeTheCommand() - let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) + let response = try handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) // The fixture keeps an animation in flight until a button is answered or its backstop, which // outlasts resolution plus activation, stops it, and an in-flight animation is what XCTest waits @@ -212,7 +179,7 @@ extension RunnerTests { try assertAlertActivationDisablesInterruptionHandling(alert) let deadline = Date().addingTimeInterval(RunnerTests.alertActivationDeadline) - let response = handleAlert(alert, action: "accept", deadline: deadline) + let response = try handleAlert(alert, action: "accept", deadline: deadline) XCTAssertEqual(consultedInterruptions, [], "alert activation consulted XCTest's interruption handling") XCTAssertTrue(response.ok, String(describing: response.error)) @@ -225,24 +192,42 @@ extension RunnerTests { private func assertAlertActivationDisablesInterruptionHandling(_ alert: RunnerAlert) throws { let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame + try withAlertTapStub { + AlertTapDeadlineStub.status = .deadlineExceeded + let outcome = try activateAlertButton( + alert, button: button, action: "accept", frame: frame, deadline: .distantFuture + ) + XCTAssertNil(outcome) + XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true) + } + } + + @MainActor + private func launchReplacementAlert() throws -> RunnerAlert { + app.launchArguments = ["--agent-device-alert-replacement-regression"] + app.launch() + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) + return try resolveAlertBeforeTheCommand() + } + + @MainActor + private func tearDownReplacementAlert() { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + @MainActor + private func withAlertTapStub(_ body: () throws -> Void) throws { let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector)) - let original = method_getImplementation(method) - AlertTapDeadlineStub.status = .deadlineExceeded - AlertTapDeadlineStub.interruptionHandlingDisabled = nil - method_setImplementation(method, method_getImplementation(stub)) + let original = method_setImplementation(method, method_getImplementation(stub)) defer { AlertTapDeadlineStub.status = nil AlertTapDeadlineStub.interruptionHandlingDisabled = nil method_setImplementation(method, original) } - - let outcome = activateAlertButton( - alert, button: button, action: "accept", frame: frame, deadline: .distantFuture - ) - XCTAssertNil(outcome) - XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true) + try body() } @MainActor @@ -273,7 +258,7 @@ extension RunnerTests { while Date() < deadline, !fixtureAlert.exists { if allow.exists { let authorization = try requireResolvedAlert(deadline: deadline) - let response = handleAlert(authorization, action: "accept", deadline: deadline) + let response = try handleAlert(authorization, action: "accept", deadline: deadline) XCTAssertTrue(response.ok, String(describing: response.error)) } else { Thread.sleep(forTimeInterval: 0.25) @@ -322,7 +307,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.exists, "the replacement must remain visible") XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") let current = try resolveAlertBeforeTheCommand() - let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) + let inspection = try handleAlert(current, action: "get", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) XCTAssertTrue(inspection.ok) XCTAssertEqual(inspection.data?.items?.sorted(), ["Cancel", "OK"]) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift index 85591434c9..cbdefc8023 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -93,7 +93,7 @@ extension RunnerTests { defer { terminateCrowdedScreen() } let alert = try requireResolvedAlert(within: RunnerTests.defaultAlertCommandTimeout) - let inspection = handleAlert(alert, action: "get", deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + let inspection = try handleAlert(alert, action: "get", deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) XCTAssertTrue(inspection.ok) XCTAssertEqual(inspection.data?.message, "Relaunch to continue") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift index 74b65986a2..aa4de3af57 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift @@ -42,6 +42,7 @@ extension RunnerTests { RunnerAppScreenCaptureFailure(.unresolvedScreen).rawValue, "APP_SCREEN_UNRESOLVED" ) + XCTAssertEqual(RunnerAppScreenCaptureFailure(.offLitPanel).rawValue, "APP_SCREEN_OFF_LIT_PANEL") // The display resolved and the image did not: a distinct code, because an operator chasing a // panel problem is chasing the wrong thing here. This one has no bridge counterpart, so it is // named on the Swift reason rather than mapped from one. @@ -136,6 +137,24 @@ extension RunnerTests { XCTAssertEqual(refusal.rawValue, "APP_SCREEN_UNRESOLVED") } + /// A requested app whose windows are all on the dark panel is refused: the system surface would + /// answer with a capture of a different app. + func testObservedScreenCaptureRefusesAnAppOffTheLitPanelWithoutAskingTheSystemSurface() { + var askedSystemSurface = false + let outcome = selectObservedScreenCapture( + resolving: { .failure(RunnerAppScreenCaptureFailure(.offLitPanel)) }, + fallingBack: { + askedSystemSurface = true + return .failure(.unrenderableImage) + } + ) + XCTAssertFalse(askedSystemSurface) + guard case .failure(let refusal) = outcome else { + return XCTFail("an app off the lit panel must not become a capture") + } + XCTAssertEqual(refusal, .offLitPanel) + } + func testObservedScreenCaptureKeepsTheSessionAppCaptureItResolved() { var askedSystemSurface = false let captured = CapturedAppScreen( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index a91f188fee..3e6666d6cd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { + switch try coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { case .input(let input): return input case .absent: @@ -65,7 +65,7 @@ extension RunnerTests { let typeCommand = try runnerCommandFixture( #"{"command":"type","commandId":"type-after-penalized-non-text-target","text":"must-not-type","textEntryMode":"append"}"# ) - let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + let typeResponse = try executeTypeCommand(activeApp: app, command: typeCommand) XCTAssertFalse(typeResponse.ok) XCTAssertEqual(typeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") @@ -99,7 +99,7 @@ extension RunnerTests { #"{"command":"type","commandId":"fill-penalized-non-text-point","text":"must-not-replace","textEntryMode":"replace","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY)}"# ) let failureCountBefore = currentXCTestFailureCount() - let fillResponse = executeTypeCommand(activeApp: app, command: fillCommand) + let fillResponse = try executeTypeCommand(activeApp: app, command: fillCommand) XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertFalse(fillResponse.ok, String(describing: fillResponse.error)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index b73dde9bf2..95627b9b7c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -109,7 +109,7 @@ extension RunnerTests { ] for testCase in cases { RunnerWindowFrameReads.begin(answering: testCase.frames) - let resolved = resolveRunnerWindow(app: app) + let resolved = try resolveRunnerWindow(app: app) XCTAssertEqual(resolved.window != nil, testCase.hasWindow, testCase.name) XCTAssertEqual(RunnerWindowFrameReads.recorded(), testCase.reads, testCase.name) XCTAssertEqual( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index 7c603af668..d2a3644c56 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -76,16 +76,22 @@ extension RunnerTests { let plans = requests.compactMap { $0["gesturePlan"] as? [String: Any] } let pointers = plans.flatMap { $0["pointers"] as? [[String: Any]] ?? [] } let samples = pointers.flatMap { $0["samples"] as? [[String: Any]] ?? [] } + let litPanels = requests.compactMap { $0["litPanel"] as? [String: Any] } let commands = try entries.map { try decodeProductionRunnerRequest($0.request, $0.name) } let command = try XCTUnwrap(commands.first, "no production request") let plan = try XCTUnwrap( commands.compactMap(\.gesturePlan).first, "no production request carries a gesturePlan" ) + let litPanel = try XCTUnwrap( + commands.compactMap(\.litPanel).first, + "no production request carries a litPanel, so a foldable's surface route would go unpinned" + ) let step = try JSONDecoder().decode(SequenceStep.self, from: Data(#"{"kind":"tap"}"#.utf8)) let sample = try XCTUnwrap(plan.pointers.first?.samples.first) assertEveryStoredField(of: command, appearsIn: requests, "Command") assertEveryStoredField(of: step, appearsIn: steps, "SequenceStep") + assertEveryStoredField(of: litPanel, appearsIn: litPanels, "RunnerLitPanelFact") assertEveryStoredField(of: plan, appearsIn: plans, "RunnerGesturePlan") assertEveryStoredField( of: plan.viewport, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index f881f622e4..2b987ae36a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -11,6 +11,22 @@ extension RunnerTests { try assertInAppBackDispatch(navigationBar: false) } + @MainActor + func testNavigationBarBackDoesNotChooseAControlOffTheLitPanel() throws { + let status = try launchBackFixture(["--agent-device-back-regression"], navigationBar: true) + defer { + setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"uptime"}"#)) + endBackFixture() + } + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"backInApp","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try tapInAppBackControl(app: app)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + XCTAssertNotEqual(status.label, "Back performed") + } + /// A navigation bar button keeps XCTest's element tap, which aims at its computed hittable point. @MainActor func testNavigationBarBackKeepsTheElementTap() throws { @@ -22,7 +38,7 @@ extension RunnerTests { let status = try launchBackFixture(["--agent-device-back-regression"]) defer { endBackFixture() } - let outcome = try withRejectedAppTree(framesReadable: false) { tapInAppBackControl(app: app) } + let outcome = try withRejectedAppTree(framesReadable: false) { try tapInAppBackControl(app: app) } guard soleScreenRouteIsOpen(for: app) else { guard case .unavailable = outcome else { @@ -43,7 +59,7 @@ extension RunnerTests { _ = try launchBackFixture([]) defer { endBackFixture() } - let outcome = try withRejectedAppTree(framesReadable: false) { tapInAppBackControl(app: app) } + let outcome = try withRejectedAppTree(framesReadable: false) { try tapInAppBackControl(app: app) } guard case .unavailable = outcome else { return XCTFail("a top-leading tap on an app without a back control must not read as performed") @@ -61,7 +77,7 @@ extension RunnerTests { defer { endBackFixture() } let outcome = try withRejectedAppTree(framesReadable: true, rejections: 1) { - tapInAppBackControl(app: app) + try tapInAppBackControl(app: app) } guard case .performed = outcome else { @@ -94,7 +110,7 @@ extension RunnerTests { ) let outcome = try withRejectedAppTree(framesReadable: true, rejections: 1) { - tapInAppBackControl(app: app) + try tapInAppBackControl(app: app) } guard case .performed = outcome else { return XCTFail("a refused sole-screen route must still dispatch back through the resolved window") @@ -156,7 +172,7 @@ extension RunnerTests { private func withRejectedAppTree( framesReadable: Bool, rejections: Int = .max, - _ body: () -> T + _ body: () throws -> T ) throws -> T { var restorations: [() -> Void] = [] defer { restorations.reversed().forEach { $0() } } @@ -188,7 +204,7 @@ extension RunnerTests { imp_removeBlock(unavailableFrame) } } - return body() + return try body() } @MainActor @@ -217,7 +233,7 @@ extension RunnerTests { } } - guard case .performed = tapInAppBackControl(app: app) else { + guard case .performed = try tapInAppBackControl(app: app) else { return XCTFail("the selected back control must dispatch") } let changed = expectation(for: NSPredicate(format: "label == %@", "Back performed"), evaluatedWith: status) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift index 5e103b00c4..2e91292030 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -21,10 +21,10 @@ extension RunnerTests { } @MainActor - func testSequenceAcceptsDoubleTapKind() { + func testSequenceAcceptsDoubleTapKind() throws { // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. - let response = executeSequenceForTest(steps: [ + let response = try executeSequenceForTest(steps: [ sequenceStep(kind: "doubleTap", x: nil) ]) XCTAssertEqual(response.ok, false) @@ -35,7 +35,7 @@ extension RunnerTests { @MainActor func testSequenceRejectsUnknownKind() throws { - let response = executeSequenceForTest(steps: [ + let response = try executeSequenceForTest(steps: [ sequenceStep(kind: "tap", x: 1, y: 2), sequenceStep(kind: "pinch", x: 3, y: 4), ]) @@ -46,16 +46,16 @@ extension RunnerTests { } @MainActor - func testSequenceRejectsEmpty() { - let response = executeSequenceForTest(steps: []) + func testSequenceRejectsEmpty() throws { + let response = try executeSequenceForTest(steps: []) XCTAssertEqual(response.ok, false) XCTAssertEqual(response.error?.code, "INVALID_ARGS") } @MainActor - func testSequenceRejectsTooManySteps() { + func testSequenceRejectsTooManySteps() throws { let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } - let response = executeSequenceForTest(steps: steps) + let response = try executeSequenceForTest(steps: steps) XCTAssertEqual(response.ok, false) XCTAssertEqual(response.error?.code, "INVALID_ARGS") XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) @@ -156,9 +156,9 @@ extension RunnerTests { /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without /// reaching the device executor (which is never invoked when validation rejects). @MainActor - private func executeSequenceForTest(steps: [SequenceStep]) -> Response { + private func executeSequenceForTest(steps: [SequenceStep]) throws -> Response { let command = makeSequenceCommand(steps: steps) - return executeSequence(command: command, activeApp: app) + return try executeSequence(command: command, activeApp: app) } /// Build a sequence Command via JSON so the test does not depend on the memberwise init's diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift index 7d872aa45b..b884ebf27b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift @@ -10,13 +10,15 @@ final class ElementAttributesFixture: NSObject, XCUIElementSnapshot { let label: String let children: [XCUIElementSnapshot] let identifier: String + @objc let displayID: NSNumber? - init(_ elementType: XCUIElement.ElementType, _ frame: CGRect, label: String = "", identifier: String = "", children: [XCUIElementSnapshot] = []) { + init(_ elementType: XCUIElement.ElementType, _ frame: CGRect, label: String = "", identifier: String = "", children: [XCUIElementSnapshot] = [], displayID: NSNumber? = nil) { self.elementType = elementType self.frame = frame self.label = label self.children = children self.identifier = identifier + self.displayID = displayID } var value: Any? { nil } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index e5cf577f63..63880cb5da 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -89,23 +89,23 @@ extension RunnerTests { } @MainActor - func testCoordinateTapThatWasNotDispatchedDoesNotFallBackToXCTest() { + func testCoordinateTapThatWasNotDispatchedDoesNotFallBackToXCTest() throws { for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { mainOwned.accessibilityHealth = health for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { - let attempt = performSynthesizedGesture(app, kind: .coordinateTap, context: context) { nil } + let attempt = try performSynthesizedGesture(app, kind: .coordinateTap, context: context) { nil } XCTAssertEqual(synthesizedGestureRoute(attempt), "refused", "axHealth=\(health.rawValue)") } } } @MainActor - func testFailedCoordinateTapSynthesisFallsBackToXCTestAtEveryAccessibilityHealth() { + func testFailedCoordinateTapSynthesisFallsBackToXCTestAtEveryAccessibilityHealth() throws { for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { mainOwned.accessibilityHealth = health for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { let label = "axHealth=\(health.rawValue) context=\(context == nil ? "unresolved" : "resolved")" - let attempt = performSynthesizedGesture( + let attempt = try performSynthesizedGesture( app, kind: .coordinateTap, context: context, @@ -117,7 +117,7 @@ extension RunnerTests { } @MainActor - func testSynthesizedGestureFallbackFollowsItsKindAndTheResolvedAccessibilityHealth() { + func testSynthesizedGestureFallbackFollowsItsKindAndTheResolvedAccessibilityHealth() throws { mainOwned.accessibilityHealth = .healthy let unavailable = synthesizedGestureTestContext(accessibilityHealth: .unavailable) let unknown = synthesizedGestureTestContext(accessibilityHealth: .unknown) @@ -130,7 +130,7 @@ extension RunnerTests { (.coordinateLongPress, unknown, "xctestFallback"), ] for (kind, context, expected) in cases { - let attempt = performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) + let attempt = try performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) XCTAssertEqual(synthesizedGestureRoute(attempt), expected, "kind=\(kind.rawValue)") switch attempt { case .xctestFallback(let message, let hint), .refused(_, let message, let hint): @@ -139,7 +139,7 @@ extension RunnerTests { case .performed: break } - let performed = performSynthesizedGesture(app, kind: kind, context: context) { .performed } + let performed = try performSynthesizedGesture(app, kind: kind, context: context) { .performed } XCTAssertEqual(synthesizedGestureRoute(performed), "performed", "kind=\(kind.rawValue)") } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 9ca703caee..7df11aa361 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -407,7 +407,7 @@ extension RunnerTests { RunnerInterfaceOrientation.portrait ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .performed? = outcome else { return XCTFail("expected a portrait tap to be dispatched, got \(String(describing: outcome))") } @@ -430,7 +430,7 @@ extension RunnerTests { RunnerInterfaceOrientation.portrait ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40) guard case .performed? = outcome else { return XCTFail("expected a window-less portrait tap to be dispatched, got \(String(describing: outcome))") } @@ -445,7 +445,7 @@ extension RunnerTests { let (_, cleanup) = try launchWithUnreadableDisplay(landscape: true, windowQueryRejected: true) defer { cleanup() } - let outcome = synthesizedTapAt(app: app, x: 40, y: 40) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40) guard case .unsupported(let message, _)? = outcome else { return XCTFail("expected a refusal before dispatch, got \(String(describing: outcome))") } @@ -471,7 +471,7 @@ extension RunnerTests { synthesizedRotationFrame(app: app, fallback: windowFrame, interfaceOrientation: orientation), windowFrame ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .performed? = outcome else { return XCTFail("expected the tap to be dispatched about the window, got \(String(describing: outcome))") } @@ -499,7 +499,7 @@ extension RunnerTests { placedWindow, "portrait skips the box read (only landscape and upside-down read it), so it must not refuse" ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .unsupported(let message, _)? = outcome else { return XCTFail("expected a refusal before dispatch, got \(String(describing: outcome))") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 76bf0c0e77..1758544ac5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -77,7 +77,7 @@ extension RunnerTests { ]) ) let failuresBeforeType = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) return response } @@ -226,7 +226,7 @@ extension RunnerTests { let label = "\(testCase.seed.count) characters, all select-alls dropped" let seeded = try replaceSynthesizedFieldText(field, text: testCase.seed, commandId: "seed-\(testCase.seed.count)") XCTAssertTrue(seeded.ok, String(describing: seeded.error)) - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -253,6 +253,27 @@ extension RunnerTests { } } + /// Clears through the penalized synthesized route at `frame`'s center, where the pre-tap lookup + /// found `input`. + @MainActor + private func clearPenalized(at frame: CGRect, finding input: TextInputAtPoint) throws -> TextEntryResult { + try typeTextReliably( + app: app, + target: TextEntryTarget( + element: nil, + refreshPoint: CGPoint(x: frame.midX, y: frame.midY), + prefersFocusedElement: false, + contentTranslation: .none, + inputAtRefreshPoint: input + ), + text: "", + delaySeconds: 0, + repairMode: .replacement, + xCTestChannelPenalized: true, + synthesizer: RecordingTextEntrySynthesizer() + ) + } + /// Launches the fixture whose field moves away when focused and leaves a neighbouring field under /// the point the focus tap hit, the way a React Native bottom sheet extends above the keyboard. /// The field is not focused yet, so the replacement's own tap starts the move. @@ -324,21 +345,7 @@ extension RunnerTests { ] for (handle, inputAtRefreshPoint) in found { - let result = typeTextReliably( - app: app, - target: TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), - prefersFocusedElement: false, - contentTranslation: .none, - inputAtRefreshPoint: inputAtRefreshPoint - ), - text: "", - delaySeconds: 0, - repairMode: .replacement, - xCTestChannelPenalized: true, - synthesizer: RecordingTextEntrySynthesizer() - ) + let result = try clearPenalized(at: neighbourFrame, finding: inputAtRefreshPoint) XCTAssertEqual(result.failure, .notFocused, handle) XCTAssertEqual(String(describing: neighbour.value ?? ""), "neighbour", handle) @@ -357,23 +364,12 @@ extension RunnerTests { XCTAssertEqual(rebound.identifier, "agent-device-text-entry-neighbour") let neighbourFrame = neighbour.frame - let result = typeTextReliably( - app: app, - target: TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), - prefersFocusedElement: false, - contentTranslation: .none, - inputAtRefreshPoint: TextInputAtPoint( - element: rebound, - identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier) - ) - ), - text: "", - delaySeconds: 0, - repairMode: .replacement, - xCTestChannelPenalized: true, - synthesizer: RecordingTextEntrySynthesizer() + let result = try clearPenalized( + at: neighbourFrame, + finding: TextInputAtPoint( + element: rebound, + identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier) + ) ) XCTAssertNil(result.failure) @@ -392,23 +388,12 @@ extension RunnerTests { let neighbourFrame = neighbour.frame let failures = currentXCTestFailureCount() - let result = typeTextReliably( - app: app, - target: TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), - prefersFocusedElement: false, - contentTranslation: .none, - inputAtRefreshPoint: TextInputAtPoint( - element: app.textFields.element, - identity: TextEntryInputIdentity(elementType: .textField, identifier: "") - ) - ), - text: "", - delaySeconds: 0, - repairMode: .replacement, - xCTestChannelPenalized: true, - synthesizer: RecordingTextEntrySynthesizer() + let result = try clearPenalized( + at: neighbourFrame, + finding: TextInputAtPoint( + element: app.textFields.element, + identity: TextEntryInputIdentity(elementType: .textField, identifier: "") + ) ) XCTAssertFalse(didRecordXCTestFailure(since: failures)) @@ -453,7 +438,7 @@ extension RunnerTests { textInputProbeBlindForTesting = true let frame = textField.frame - guard case .unavailable(.emptyTree) = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) else { + guard case .unavailable(.emptyTree) = try probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) else { return XCTFail("A blind tree must be unavailable to every probe caller") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift index a3b4fe7a5a..b3749e1f02 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift @@ -298,7 +298,7 @@ extension RunnerTests { "x": frame.midX, "y": frame.midY, "appBundleId": "com.callstack.agentdevice.runner", ])) let failures = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failures)) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "TEXT_ENTRY_MISMATCH") @@ -338,7 +338,7 @@ extension RunnerTests { "x": frame.midX, "y": frame.midY, "appBundleId": "com.callstack.agentdevice.runner", ])) let failures = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failures)) XCTAssertTrue(response.ok, String(describing: response.error)) XCTAssertEqual(response.data?.message, "typed") @@ -435,7 +435,7 @@ extension RunnerTests { ) let failuresBeforeType = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) XCTAssertTrue(response.ok, String(describing: response.error)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index 9b94a9df90..8b2fc7aad9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) let failures = currentXCTestFailureCount() let startedAt = Date() - waitForTextEntryReadinessAfterTap(app: app, element: field) + try waitForTextEntryReadinessAfterTap(app: app, element: field) let elapsed = Date().timeIntervalSince(startedAt) XCTAssertFalse(didRecordXCTestFailure(since: failures)) XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) @@ -54,7 +54,7 @@ extension RunnerTests { ) let readsBefore = try XCTUnwrap(Int(valueReads.label)) - let response = executeTypeCommand(activeApp: app, command: fill) + let response = try executeTypeCommand(activeApp: app, command: fill) let readsAfter = try XCTUnwrap(Int(valueReads.label)) XCTAssertTrue(response.ok, String(describing: response.error)) @@ -76,15 +76,15 @@ extension RunnerTests { let valueReads = app.staticTexts["agent-device-text-entry-value-reads"] let frame = field.frame let point = CGPoint(x: frame.midX, y: frame.midY) - func countingReads(_ body: () -> T) throws -> (T, Int) { + func countingReads(_ body: () throws -> T) throws -> (T, Int) { let before = try XCTUnwrap(Int(valueReads.label)) - let value = body() + let value = try body() return (value, try XCTUnwrap(Int(valueReads.label)) - before) } let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false, contentTranslation: .none) - let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) } - let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } + let (lookup, lookupReads) = try countingReads { try textInputCandidatesAt(app: app, point: point, contentTranslation: .none).first } + let (input, resolutionReads) = try countingReads { try resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") XCTAssertEqual(input?.snapshot?.identifier, "agent-device-hardware-keyboard-input") @@ -146,7 +146,7 @@ extension RunnerTests { let readsBefore = try XCTUnwrap(Int(valueReads.label)) let fillStartedAt = Date() - let response = executeTypeCommand(activeApp: app, command: fill) + let response = try executeTypeCommand(activeApp: app, command: fill) let readsAfter = try XCTUnwrap(Int(valueReads.label)) try XCTSkipUnless( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 20f62e56b9..b01c2ebe1f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -733,7 +733,7 @@ extension RunnerTests { defer { tearDownSynthesizedReplacementField() } let synthesizer = RecordingTextEntrySynthesizer() let frame = field.frame - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -778,10 +778,10 @@ extension RunnerTests { // `_XCTestCaseInterruptionException`, which XCTest records as a test failure even when production // code catches it. Springboard is always running, so the query resolves to no input at the point. @MainActor - func testSynthesizedReplacementTypesNothingWhenItCannotReadTheFieldBack() { + func testSynthesizedReplacementTypesNothingWhenItCannotReadTheFieldBack() throws { let synthesizer = RecordingTextEntrySynthesizer() let failures = currentXCTestFailureCount() - let result = typeTextReliably( + let result = try typeTextReliably( app: springboard, target: TextEntryTarget( element: nil, @@ -809,8 +809,8 @@ extension RunnerTests { // fail closed: it used to fall through to the vacuous-typing early return and report // `verified: true` for a clear that never ran. @MainActor - func testEmptyReplacementWithoutResolvableTargetFailsClosed() { - let result = typeTextReliably( + func testEmptyReplacementWithoutResolvableTargetFailsClosed() throws { + let result = try typeTextReliably( app: springboard, target: TextEntryTarget(element: nil, refreshPoint: nil, prefersFocusedElement: false, contentTranslation: .none), text: "", @@ -830,7 +830,7 @@ extension RunnerTests { func testSynthesizedReplacementCommitSkipsSubmitKeyText() { for expectedText in ["ada@example.test\n", "ada@example.test\r"] { XCTAssertEqual( - awaitSynthesizedReplacementCommit( + try awaitSynthesizedReplacementCommit( app: XCUIApplication(), target: TextEntryTarget(element: nil, refreshPoint: CGPoint(x: 10, y: 20), prefersFocusedElement: false, contentTranslation: .none), expectedText: expectedText diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift index f42559ae14..44e05347f0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -23,14 +23,14 @@ extension RunnerTests { let unknown = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .unknown)) XCTAssertNil(unknown.frameOnScreen) - retapAt(read: unknown, app: app, element: field) + try retapAt(read: unknown, app: app, element: field) XCTAssertEqual(presses.label, "presses=0", "an unknown content space dispatched a point") - retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)), app: app, element: field) + try retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)), app: app, element: field) XCTAssertEqual(presses.label, "presses=0", "a full-screen read dispatched a point off the field") let vector = CGSize(width: witnessFrame.midX - fieldFrame.midX, height: witnessFrame.midY - fieldFrame.midY) - retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))), app: app, element: field) + try retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))), app: app, element: field) XCTAssertEqual(presses.label, "presses=1", "the retap dispatched the raw snapshot center, not the screen-space one") } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index 5d1fe3a468..1a618c3621 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait") - _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: .none) + _ = try? self.probeTextInputs(app: self.app, point: point, contentTranslation: .none) completed.fulfill() } guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else { @@ -174,8 +174,8 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) - _ = probeTextInputs(app: app, point: point, contentTranslation: .none) + _ = try? textInputCandidatesAt(app: app, point: point, contentTranslation: .none) + _ = try? probeTextInputs(app: app, point: point, contentTranslation: .none) record(expected) } XCTAssertEqual(observed, 2) @@ -192,7 +192,7 @@ extension RunnerTests { XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") - let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) + let outcome = try probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) guard case .unavailable = outcome else { return XCTFail("A suppressed AX issue must discard the matching candidate") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 75764dcd42..e692526a79 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -10,7 +10,7 @@ extension RunnerTests { #"{"command":"type","commandId":"type-without-focus","text":"hello","textEntryMode":"append"}"# ) - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: XCUIApplication(bundleIdentifier: "com.example.agentdevice.missing-input"), command: command ) @@ -31,7 +31,7 @@ extension RunnerTests { try skipUnlessSoftwareKeyboardIsHidden() let failureCountBefore = currentXCTestFailureCount() - let typeResponse = executeTypeCommand( + let typeResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard","text":"hardware-keyboard","textEntryMode":"append"}"# @@ -46,7 +46,7 @@ extension RunnerTests { // The tap witness is one-shot: a second bare type without a new tap has no target. let unfocusedFailureCountBefore = currentXCTestFailureCount() - let unfocusedResponse = executeTypeCommand( + let unfocusedResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard-unfocused","text":"-again","textEntryMode":"append"}"# @@ -63,7 +63,7 @@ extension RunnerTests { // keyboard here is a product change, not an environment fact. XCTAssertFalse(isKeyboardVisible(app: app)) let appendFailureCountBefore = currentXCTestFailureCount() - let appendResponse = executeTypeCommand( + let appendResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard-again","text":"-again","textEntryMode":"append"}"# @@ -83,7 +83,7 @@ extension RunnerTests { try skipUnlessSoftwareKeyboardIsHidden() let failureCountBefore = currentXCTestFailureCount() - let submitResponse = executeTypeCommand( + let submitResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard-submit","text":"\n"}"# @@ -102,7 +102,7 @@ extension RunnerTests { try skipUnlessSoftwareKeyboardIsHidden() let failureCountBefore = currentXCTestFailureCount() - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: textField, @@ -146,7 +146,7 @@ extension RunnerTests { let typeCommand = try runnerCommandFixture( #"{"command":"type","commandId":"type-disappearing-input","text":"ab","delayMs":50,"textEntryMode":"append"}"# ) - let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + let typeResponse = try executeTypeCommand(activeApp: app, command: typeCommand) XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertFalse(typeResponse.ok) @@ -163,7 +163,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-auto-submit") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-auto-submit", text: "123456", at: textField) ) @@ -183,7 +183,7 @@ extension RunnerTests { func testFillReplacesAValueLongerThanOneClearBurst() throws { let textField = try launchRemovableInputFixture("--agent-device-text-entry-long-prefill") - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-long-prefill", text: "Ada Lovelace", at: textField) ) @@ -197,7 +197,7 @@ extension RunnerTests { func testFillClearsAValueLongerThanOneClearBurst() throws { let textField = try launchRemovableInputFixture("--agent-device-text-entry-long-prefill") - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "clear-long-prefill", text: "", at: textField) ) @@ -216,7 +216,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - _ = executeTypeCommand( + _ = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-clear-replaced-input", text: "", at: textField) ) @@ -234,7 +234,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-auto-submit-without-successor") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-auto-submit-no-successor", text: "123456", at: textField) ) @@ -253,7 +253,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-disappear-after-input") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-disappearing-input", text: "123456", at: textField) ) @@ -272,7 +272,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-replace-after-input") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-replaced-input", text: "123456", at: textField) ) @@ -294,7 +294,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-disappearing-masked-input", text: "123456", at: textField) ) @@ -314,7 +314,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-replaced-masked-input", text: "123456", at: textField) ) @@ -349,7 +349,7 @@ extension RunnerTests { defer { textEntryWarmupPauseForTesting = nil } let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-masked-replaced-in-pause", text: "123456", at: textField) ) @@ -375,7 +375,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-unnamed-successor", text: "123456", at: textField) ) @@ -398,7 +398,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-digit-count-auto-submit", text: "123456", at: textField) ) @@ -425,7 +425,7 @@ extension RunnerTests { let frameBefore = textField.frame let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-moves-on-input", text: "hello", at: textField) ) @@ -451,7 +451,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-unnamed-replaced", text: "123456", at: textField) ) @@ -510,7 +510,7 @@ extension RunnerTests { let failureCountBefore = currentXCTestFailureCount() // The target the `type` command builds when it cannot resolve an input but the keyboard is up: // no element, no refresh point, focused-element preference. - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -564,7 +564,7 @@ extension RunnerTests { try tapHardwareKeyboardInput(commandId: "tap-masked-input") let failureCountBefore = currentXCTestFailureCount() - let fillResponse = executeTypeCommand( + let fillResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"fill-masked-input","text":"not-a-secret-123","textEntryMode":"replace"}"# diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 289012ccf7..3439148f11 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -35,6 +35,11 @@ private final class DisplayWindowsFixture: NSObject { init(_ window: NSObject) { firstMatch = window } } +private final class WindowListFixture: NSObject { + @objc let allElementsBoundByIndex: [NSObject] + init(_ windows: [NSObject]) { allElementsBoundByIndex = windows } +} + private final class DisplayApplicationFixture: NSObject { @objc let windows: NSObject convenience init(_ window: NSObject) { @@ -247,5 +252,164 @@ extension RunnerTests { ) } } + // MARK: - The lit panel's window (ADR 0025) + + /// The inner panel as the host names it: 669x951 native points, presented turned as 951x669. + private static let innerPanel = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) + private static let innerHome = CGRect(x: 0, y: 0, width: 951, height: 669) + /// SpringBoard's first window: the status bar on the outer panel, whichever panel is lit. + private static let outerStatusBar = CGRect(x: 452, y: 0, width: 226, height: 155.3) + + private func resolveLitPanelWindow( + _ windows: [NSObject], + panel: RunnerLitPanel = innerPanel + ) -> (failure: RunnerApplicationScreenFailure, window: AnyObject?, frame: CGRect, screen: AnyObject?) { + var window: AnyObject? + var frame = CGRect.null + var screen: AnyObject? + let failure = RunnerResolveLitPanelWindow( + DisplayApplicationFixture(windows: WindowListFixture(windows)), panel, &window, &frame, &screen) + return (failure, window, frame, screen) + } + + func testLitPanelWindowSkipsTheFirstWindowOnTheDarkPanel() { + let statusBar = DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1) + let home = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let later = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let walk = resolveLitPanelWindow([statusBar, home, later]) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.window === home) + XCTAssertEqual(walk.frame, Self.innerHome) + XCTAssertFalse(later.resolved, "the walk stops at the first full-panel window") + } + + func testLitPanelWindowAcceptsThePanelBoxEitherWayUp() { + for frame in [CGRect(x: 0, y: 0, width: 669, height: 951), Self.innerHome] { + let window = DisplayWindowFixture(frame: frame, displayID: 3) + let walk = resolveLitPanelWindow([window]) + XCTAssertEqual(walk.failure, .none, "\(frame)") + XCTAssertTrue(walk.window === window) + } + } + + /// A full-size window on the dark panel is the trap: geometry alone would book a screen nobody + /// is looking at. + func testLitPanelWindowRefusesWindowsThatAreAllOnTheDarkPanel() { + let walk = resolveLitPanelWindow([ + DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1), + DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 1), + ]) + XCTAssertEqual(walk.failure, .offLitPanel) + XCTAssertNil(walk.window) + XCTAssertEqual(resolveLitPanelWindow([]).failure, .unresolvedWindow, "no window at all") + } + + /// An unread window list is not an app without windows: it must not reach the system-surface + /// fallback, which only `UnresolvedWindow` takes. + func testLitPanelWindowRefusesAWindowListItCannotRead() { + var window: AnyObject? + var frame = CGRect.null + var screen: AnyObject? + let failure = RunnerResolveLitPanelWindow( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel, &window, &frame, &screen) + XCTAssertEqual(failure, .unresolvedScreen) + XCTAssertNil(window) + } + + /// A window the walk could not place may itself be the panel's surface, so a full-panel window + /// read after it is not booked. + func testLitPanelWindowRefusesAFullPanelWindowReadAfterAnUnplacedOne() { + let home = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let walk = resolveLitPanelWindow([ScreenlessWindowFixture(), home]) + XCTAssertEqual(walk.failure, .unresolvedScreen) + XCTAssertNil(walk.window) + } + + func testLitPanelWindowRefusesAFrameReadExceptionBeforeAnyLaterWindow() { + let laterWindows: [[NSObject]] = [ + [], + [DisplayWindowFixture(frame: Self.innerHome, displayID: 3)], + [DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1)], + ] + for later in laterWindows { + let walk = resolveLitPanelWindow([FramelessWindowFixture()] + later) + XCTAssertEqual(walk.failure, .unresolvedScreen) + XCTAssertNil(walk.window) + XCTAssertTrue(walk.frame.isNull) + } + } + + /// An app that occupies part of the lit panel has no full-panel box; being on the lit display is + /// still the fact the first-window route could not establish. + func testLitPanelWindowBooksTheLargestWindowOnTheLitDisplay() { + let strip = DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 3) + let sheet = DisplayWindowFixture(frame: CGRect(x: 0, y: 90, width: 300, height: 400), displayID: 3) + let walk = resolveLitPanelWindow([ + DisplayWindowFixture(frame: Self.innerHome, displayID: 1), strip, sheet, + ]) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.window === sheet) + } + + /// A window the walk could not place, or one past its cap, could be the panel's surface, so a + /// lesser window is not booked in its place. + func testLitPanelWindowRefusesALesserWindowWhenTheListWasNotFullyPlaced() { + let sheet = DisplayWindowFixture(frame: CGRect(x: 0, y: 90, width: 300, height: 400), displayID: 3) + XCTAssertEqual( + resolveLitPanelWindow([ScreenlessWindowFixture(), sheet]).failure, .unresolvedScreen) + let beyondCap = (0.."}}, + {"name": "ios-foldable.interactions-tap.synthesized", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "synthesized": true, "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-find-text.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "findText", "text": "Search", "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-keyboard-dismiss.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "keyboardDismiss", "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-read-text.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-snapshot.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, {"name": "ios-simulator.alert.accept", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "alert", "action": "accept", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, {"name": "ios-simulator.alert.dismiss", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "alert", "action": "dismiss", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, {"name": "ios-simulator.alert.get", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "alert", "action": "get", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 0e744dc228..297c18dc5e 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -8,6 +8,7 @@ The touch-overlay export diagnosis was corrected on 2026-09-22 under #2707. The Android emulator pose amendment is dated 2026-10-10. The no-panel refusal correction (#3761) is dated 2026-10-18. The guest readback amendment is dated 2026-10-26. +The runner surface amendment is dated 2026-10-31. This ADR covers iPhone Duo (iOS 27.1, `iPhone19,4`) and Apple devices with more than one integrated CoreDevice display. The amendment below also covers foldable Android emulators. @@ -148,12 +149,18 @@ The vitest twin and runner unit-test lane decode this golden table. Contract drift therefore fails CI on both sides. Capture first asks for the target app's window, then the system surface's window. +On a multi-panel device, it asks the system surface only when the app has no window at all. SpringBoard owns the home-screen window. Thus, a session without an app, or with a stopped app, can still resolve a window's display. The result always names a display occupied by a window. It never defaults to `XCUIScreen.main`, which is dark on a Duo when the app uses the inner panel. -A capture fails closed only in three cases. Neither window resolves. A resolved window names no display. A resolved display yields an unrenderable image. -The typed reasons are `unresolvedWindow`, `unresolvedScreen`, and `unrenderableImage` (`RunnerAppScreenCapture.swift`). +A capture fails closed with one of four typed reasons (`RunnerAppScreenCapture.swift`). + +- `unresolvedWindow` (`APP_SCREEN_WINDOW_UNRESOLVED`): neither window resolves. +- `unresolvedScreen` (`APP_SCREEN_UNRESOLVED`): a resolved window names no display. On a multi-panel device, it also covers a window list that cannot be read and a window the lit-panel walk cannot place. +- `unrenderableImage` (`APP_SCREEN_CAPTURE_UNRENDERABLE`): a resolved display yields an unrenderable image. +- `offLitPanel` (`APP_SCREEN_OFF_LIT_PANEL`): on a multi-panel device, every window of the target app is on the dark panel. + The query uses both `exists` and frame evidence to refuse an app without a window. Such an app returns `(0,0 0x0)` from `frame` without an exception. Geometry alone cannot prove that no window exists. @@ -708,3 +715,28 @@ It is refused as `single-panel-device`, the same typed reason as a single-panel Keyframes and panel geometry remain Apple-only. The emulator console supports only fixed postures. Android reports the app window on the next snapshot instead of a panel. + +## Amendment: the lit panel chooses the runner's window (2026-10-31) + +SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. + +Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. A multi-panel table read that names the lit panel but spends the request budget refuses with `COMMAND_FAILED`, `reason: panel-probe-budget-exhausted` and `dispatched: no`, without a runner error code. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. + +With a named panel, the runner walks at most eight windows to book one; element search reads every window that reports the lit display. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. A window list that cannot be read is `APP_SCREEN_UNRESOLVED`, so it never reaches the system-surface fallback. + +The request books the window by bundle identity and request identity before it acts, and releases the booking when the request ends. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search rereads the complete window list for each read, including windows beyond the booking cap, because mutations can open another window. If the list cannot be read or contains no proven lit windows, the search throws `APP_SCREEN_UNRESOLVED` instead of answering no match. A lit-panel refusal during text-entry confirmation propagates with its typed code. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. + +A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. Collapsed-tab fallback candidates use the shared proven-lit element-query owner with that capture target's pinned display ID. A failed root walk propagates `APP_SCREEN_UNRESOLVED`. A `customActions` request with no explicit backend pin implies the private AX pin, so its refusal names `customActions`. + +Rejected alternatives: + +- **Largest window overall: rejected.** It guesses panel power from size. +- **Pruning the application tree by geometry: rejected.** A dark 466x678 window fits inside the inner panel's box, and a turned inner window does not. +- **Rooting the snapshot at the booked window: rejected.** The first full-panel SpringBoard window has no icons, and an app's keyboard lives in another window. +- **A device-wide panel cached in the runner: rejected.** It is stale after a fold. + +Consequences: + +- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` must equal that frame. Snapshot points are translated by that frame's origin, as for a placed iPad window, and a synthesized turn is about the lit panel's screen in the app's interface orientation, not the window's shape: an app beside the inner panel's 80 pt status strip still maps into the whole screen. `SYNTHESIZED_RECORD displayID=` must equal `lit=`. +- A recording keeps the panel that its `recordStart` named. The runner cannot read panel power, so it does not notice a fold during the recording: SpringBoard keeps windows on the panel that went dark, and later frames show that dark panel. Start a new recording after a fold. +- Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index 3fcb985970..60a123b8db 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -178,6 +178,7 @@ const DRIVERS: Record Promise> = { 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_CAPTURE_UNRENDERABLE': () => replyFailure('APP_SCREEN_CAPTURE_UNRENDERABLE'), + 'ios-runner.reply.APP_SCREEN_OFF_LIT_PANEL': () => replyFailure('APP_SCREEN_OFF_LIT_PANEL'), 'ios-runner.reply.ELEMENT_NOT_FOUND': () => replyFailure('ELEMENT_NOT_FOUND'), 'ios-runner.reply.ELEMENT_OFFSCREEN': () => replyFailure('ELEMENT_OFFSCREEN'), 'ios-runner.reply.AMBIGUOUS_MATCH': () => replyFailure('AMBIGUOUS_MATCH'), diff --git a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts index 7c881cc936..65e4fd1853 100644 --- a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts +++ b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts @@ -426,6 +426,7 @@ test('runner_endpoint_probe_exhausted is decided by the typed reason, not the me test('a refused screen capture keeps the runner reason and stays off the wire code', () => { assert.deepEqual([...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].sort(), [ 'APP_SCREEN_CAPTURE_UNRENDERABLE', + 'APP_SCREEN_OFF_LIT_PANEL', 'APP_SCREEN_UNRESOLVED', 'APP_SCREEN_WINDOW_UNRESOLVED', ]); diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index cf544dcfc5..d860b6e345 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -26,8 +26,10 @@ export { SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, buildRunnerSequenceCommand, parseRunnerSequenceResult, + resolveRunnerRequestSignal, type RunnerCommand, } from './runner-contract.ts'; +export { runnerCommandResolvesSurface } from './runner-command-traits.ts'; export { isRunnerMainThreadTimeoutError } from './runner-error-classification.ts'; export { withAppleRunnerProvider, diff --git a/packages/apple-runner/src/runner-command-traits.ts b/packages/apple-runner/src/runner-command-traits.ts index b419962847..ed11e98248 100644 --- a/packages/apple-runner/src/runner-command-traits.ts +++ b/packages/apple-runner/src/runner-command-traits.ts @@ -92,6 +92,56 @@ export const RUNNER_COMMAND_TRAITS = { shutdown: DEFAULT_TRAITS, } as const satisfies Record; +/** + * Whether the runner resolves a screen for the command: a window to book a gesture on, a viewport, + * a display to sample, or windows to search for an element. Such a request for a multi-panel device + * names the lit panel (ADR 0025). This table is the one list of lit-panel-scoped commands. + */ +const RUNNER_COMMAND_RESOLVES_SURFACE = { + tap: true, + mouseClick: false, + longPress: true, + drag: true, + remotePress: false, + type: true, + swipe: true, + scroll: true, + desktopScroll: false, + findText: true, + querySelector: true, + readText: true, + snapshot: true, + screenshot: true, + backInApp: true, + backSystem: true, + home: false, + rotate: false, + gesture: true, + gestureViewport: true, + appSwitcher: true, + actionButton: false, + volumeUp: false, + volumeDown: false, + keyboardDismiss: true, + keyboardReturn: false, + alert: false, + sequence: true, + recordStart: true, + recordStop: false, + status: false, + uptime: false, + appState: false, + pasteboardWrite: false, + activate: false, + terminate: false, + targetReset: false, + shutdown: false, +} as const satisfies Record; + +export function runnerCommandResolvesSurface(command: RunnerCommand): boolean { + return RUNNER_COMMAND_RESOLVES_SURFACE[command.command]; +} + export function isReadOnlyRunnerCommand(command: RunnerCommand): boolean { return readRunnerCommandTraits(command).readOnly; } diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 24c7bfa56c..21fcddb643 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -163,6 +163,11 @@ export type RunnerCommand = { fullscreen?: boolean; inlineScreenshot?: boolean; synthesized?: boolean; + /** + * The lit panel's display and native box in points, sent only for a multi-panel device on a + * command that resolves a screen (ADR 0025). + */ + litPanel?: { displayId: number; widthPt: number; heightPt: number }; steps?: RunnerSequenceStep[]; }; @@ -270,12 +275,17 @@ export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE = 'SCROLL_KEYBOARD_OCC const RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_WINDOW_UNRESOLVED'; const RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_UNRESOLVED'; const RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE = 'APP_SCREEN_CAPTURE_UNRENDERABLE'; +const RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE = 'APP_SCREEN_OFF_LIT_PANEL'; -/** Every runner code meaning "this capture did not happen", covering the display and the image. */ +/** + * Every runner code meaning "this capture did not happen", covering the display, the image, and the + * panel: on a foldable, a target app whose windows are all on the dark panel is refused. + */ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = new Set([ RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE, + RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE, ]); /** diff --git a/packages/platform-apple/src/__tests__/display-fixtures.ts b/packages/platform-apple/src/__tests__/display-fixtures.ts index 75135bf6f4..33a3f340f5 100644 --- a/packages/platform-apple/src/__tests__/display-fixtures.ts +++ b/packages/platform-apple/src/__tests__/display-fixtures.ts @@ -41,6 +41,53 @@ export const IPHONE_DUO_CLOSED = { }, }; +/** + * The same Duo with the hinge open: the inner panel is lit and the outer one is dark. The `rot90` + * the inner panel reports in a portrait hold is what makes its interaction space 951x669 for a + * 669x951 native panel (ADR 0025). + */ +export const IPHONE_DUO_OPEN = { + result: { + displays: [ + { + active: false, + backlightState: 'off', + bounds: [ + [0, 0], + [1398, 2034], + ], + currentOrientation: 'rot0', + displayId: 1, + name: 'LCD', + nativeOrientation: 'rot0', + nativeSize: [1398, 2034], + pointScale: 3, + primary: true, + type: { integrated: {} }, + uniqueId: '2C570360-1D02-40A1-BFBF-11197BB1159C', + }, + { + active: true, + backlightState: 'activeOn', + bounds: [ + [0, 0], + [2007, 2853], + ], + currentOrientation: 'rot90', + displayId: 3, + name: 'LCD-1', + nativeOrientation: 'rot0', + nativeSize: [2007, 2853], + pointScale: 3, + primary: false, + type: { integrated: {} }, + uniqueId: '162CC14D-0995-4D13-9C83-B609458243C8', + }, + ], + orientation: { currentDeviceOrientation: 'portrait' }, + }, +}; + /** Measured on a real iPhone 17: CoreDevice omits `active` entirely for one panel. */ export const IPHONE_17_SINGLE_PANEL = { result: { diff --git a/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts b/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts new file mode 100644 index 0000000000..b78e96a010 --- /dev/null +++ b/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts @@ -0,0 +1,91 @@ +import { + assertProducedRunnerRequests, + bindTestCommandHost, + IOS_SIMULATOR, +} from '@agent-device/test-kit'; +import { test, vi } from 'vitest'; +import type { Interactor } from '@agent-device/contracts/interactor-types'; +import type { RunnerCommand } from '@agent-device/apple-runner'; +import type { DeviceInfo } from '@agent-device/kernel/device'; + +vi.mock('../foldable/fold-helper-cache.ts', () => ({ + ensureFoldHelperBinary: vi.fn(async () => ({ path: '/cache/fold-helper' })), +})); +vi.mock('../core/devicectl.ts', async (importOriginal) => { + const actual = await importOriginal(); + // The mock factory is hoisted above this file's imports, so the panel table it answers with is + // read through the dynamic import the mock itself controls. + const { IPHONE_DUO_OPEN } = await import('./display-fixtures.ts'); + return { + ...actual, + runIosDevicectlJsonRequest: vi.fn(async () => ({ + ok: true as const, + payload: IPHONE_DUO_OPEN, + })), + }; +}); + +import { createAppleInteractor } from '../interactor.ts'; +import { recordingRunnerProvider, type RecordedRunnerCall } from './recording-runner-provider.ts'; + +bindTestCommandHost(); + +// The multi-panel half of the Apple runner request pin: the same production entry points as +// ./index-runner-requests.test.ts, driven for a device whose +// panel table reports two integrated panels, so the wire shape that carries the lit panel is pinned +// too. That file's entries are the single-panel requests, and they must stay panel-free. + +const APP = 'com.example.app'; + +/** A device the `fold` fact admits: a foldable iPhone simulator in the default device set. */ +const DUO_SIMULATOR: DeviceInfo = { ...IOS_SIMULATOR, name: 'iPhone Duo' }; + +type InteractorDrive = readonly [DeviceInfo, (interactor: Interactor) => Promise]; + +const INTERACTOR_SITES: Record = { + 'ios-foldable.interactions-tap.synthesized': [DUO_SIMULATOR, (i) => i.tap(10, 20)], + 'ios-foldable.interactor-snapshot.lit-panel': [ + DUO_SIMULATOR, + (i) => i.snapshot({ appBundleId: APP, interactiveOnly: true }), + ], + // `keyboardDismiss` settles by fingerprinting screenshots, so it resolves a screen even though it + // reads nothing: a clip or a settle loop taken from the dark panel reports success on nothing. + 'ios-foldable.interactor-keyboard-dismiss.lit-panel': [ + DUO_SIMULATOR, + (i) => i.keyboardDismiss!(), + ], + // `readText` resolves the surface window to place the point it was handed, so the point means the + // same screen on a foldable that the ref it came from was measured in. + 'ios-foldable.interactor-read-text.lit-panel': [ + DUO_SIMULATOR, + (i) => i.readTextAtPoint!({ x: 10, y: 20 }, { appBundleId: APP }), + ], + // An element search reads only the lit panel's windows, so the dark panel's duplicate cannot answer. + 'ios-foldable.interactor-find-text.lit-panel': [ + DUO_SIMULATOR, + (i) => i.findText!('Search', { appBundleId: APP }), + ], + // A request that resolves no screen carries no panel: alert, lifecycle, readiness and clipboard + // traffic must stay byte-for-byte the request it was. + 'ios-foldable.alert.get': [DUO_SIMULATOR, (i) => i.readAlert!({ appBundleId: APP })], +}; + +test('a multi-panel device pins the runner request that carries its lit panel', async () => { + const captured: Array = []; + for (const [name, [device, drive]] of Object.entries(INTERACTOR_SITES)) { + const calls: RecordedRunnerCall[] = []; + await drive( + createAppleInteractor( + device, + { appBundleId: APP }, + { env: process.env, cwd: process.cwd() }, + recordingRunnerProvider(calls, { + alert: { message: 'Allow', items: ['Allow'] }, + }), + ), + ); + if (calls.length !== 1) throw new Error(`${name} sent ${calls.length} runner requests`); + captured.push([name, calls[0]!.command]); + } + assertProducedRunnerRequests(import.meta.filename, captured); +}); diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index b3db960c66..476cd6b774 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -1,4 +1,5 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; +import type { RunnerCommand } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; @@ -409,6 +410,23 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi }); } +/** + * The panel a runner request names, only when panel power decided exactly one panel. Unlike a + * capture, a surface request does not fall back to the `primary` panel: it can keep the runner's + * historic route instead of guessing. Display 0 is the runtime's "no display" identity. + */ +export function appleLitPanel(inventory: AppleDisplayInventory): RunnerCommand['litPanel'] { + if (!inventory.multiScreen || inventory.unresolved || !inventory.activeDisplay) return undefined; + const litPanels = inventory.displays.filter((display) => display.power === 'lit'); + const panel = litPanels.length === 1 ? litPanels[0]! : undefined; + if (!panel || panel.displayId <= 0) return undefined; + return { + displayId: panel.displayId, + widthPt: panel.widthPx / panel.pointScale, + heightPt: panel.heightPx / panel.pointScale, + }; +} + /** * Resolves the display a capture must target. * diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 01673064f1..3ed071c725 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -1,4 +1,12 @@ -import { bindAppleRunnerHost } from '@agent-device/apple-runner'; +import { + bindAppleRunnerHost, + resolveRunnerRequestSignal, + runnerCommandResolvesSurface, + runAppleRunnerCommand as runHostBoundAppleRunnerCommand, + type AppleRunnerCommandOptions, + type RunnerCommand, +} from '@agent-device/apple-runner'; +import type { DeviceInfo } from '@agent-device/kernel/device'; import { appleRunnerHost } from './runner-host.ts'; /** @@ -11,6 +19,27 @@ import { appleRunnerHost } from './runner-host.ts'; */ bindAppleRunnerHost(appleRunnerHost); +/** + * Sends `command` to the runner. A command that resolves a screen names the lit panel for a + * multi-panel device. The panel probe loads on first use, because this module is in the eager closure + * of the Apple façades (ADR 0019). + */ +export async function runAppleRunnerCommand( + device: DeviceInfo, + command: RunnerCommand, + options: AppleRunnerCommandOptions = {}, +): Promise> { + if (!runnerCommandResolvesSurface(command)) { + return await runHostBoundAppleRunnerCommand(device, command, options); + } + const { nameRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); + return await runHostBoundAppleRunnerCommand( + device, + await nameRunnerRequestLitPanel(device, command, resolveRunnerRequestSignal(options)), + options, + ); +} + export { applyXctestRunnerAppIconFromDerivedPath, cleanupRunnerLeasesForOwner, @@ -38,7 +67,6 @@ export { resolveRunnerSandboxBuildArgs, resolveRunnerSigningBuildSettings, runApplePressSeries, - runAppleRunnerCommand, stopAllIosRunnerSessions, stopIosRunnerSession, verifyLeaseRunnerPidIdentity, diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 0d695b074e..85420a7236 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -15,6 +15,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { runAppleRunnerCommand } from '../core/runner-client.ts'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/tool-provider.ts'; import { createAppleInteractor } from '../interactor.ts'; +import { readFoldablePanelTable } from './orientation.ts'; import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; import { IPHONE_17_SINGLE_PANEL, IPHONE_DUO_CLOSED } from '../__tests__/display-fixtures.ts'; @@ -200,6 +201,30 @@ test('a foldable rotation that CoreDevice never reports is refused as unverified expect(sim.helpers).toHaveLength(1); }); +/** A rotation reads the panel table itself: the surface probe's retry window never holds it. */ +test('a rotation is not routed by a pending panel-table retry', async () => { + const device = simulator(); + let tableFailures = 0; + const sim = fakeSimulator(device, DUO_DISPLAYS); + const failing = createLocalAppleToolProvider({ + runCommand: async (_command, args) => { + if (args.slice(0, 4).join(' ') === 'devicectl device info displays') { + tableFailures += 1; + throw new AppError('COMMAND_FAILED', 'devicectl does not report displays', { + timeoutMs: 2_000, + }); + } + throw new Error(`unexpected tool call: ${args.join(' ')}`); + }, + }); + await withAppleToolProvider(failing, () => readFoldablePanelTable(device, { timeoutMs: 5_000 })); + expect(tableFailures).toBe(1); + + await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); + expect(sim.helpers).toEqual([['rotate', 'landscape-left']]); + expect(mockRunner).not.toHaveBeenCalled(); +}); + test('a stale CoreDevice sample cannot verify a rotation that did not happen', async () => { const sim = fakeSimulator(simulator(), DUO_DISPLAYS, { followsHid: false, diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index f9c78cc97d..a67802bab1 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -2,13 +2,18 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { createTtlMemo } from '@agent-device/kernel/ttl-memo'; import { + IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS, IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS, } from '../core/config.ts'; import { readAppleDeviceOrientation } from '../core/device-orientation.ts'; -import { queryAppleDisplayInventory } from '../core/display-inventory.ts'; +import { + queryAppleDisplayInventory, + type AppleDisplayInventory, +} from '../core/display-inventory.ts'; import { simulatorAddressFor } from '../core/simctl.ts'; import { createFoldHelper } from './fold-helper.ts'; import { appleFoldableFacts } from './runtime.ts'; @@ -25,6 +30,24 @@ const ORIENTATION_UNVERIFIED_HINT = /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); +// A device whose panel count was never read routes as single-panel. A full-budget read that failed +// for it is not repeated for a few seconds, because every surface command would pay it again. +const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); +// Reads overlap, so a failure may finish after a read that started later and already resolved. +// Each read takes a sequence number, and a failure never overrides a newer resolved read. +let panelTableReads = 0; +const latestResolvedRead = new Map(); + +function recordPanelTable( + device: DeviceInfo, + inventory: AppleDisplayInventory, + read: number, +): void { + multiPanelByDevice.set(device.id, inventory.multiScreen); + unansweredPanelTables.delete(device.id); + latestResolvedRead.set(device.id, Math.max(read, latestResolvedRead.get(device.id) ?? 0)); +} + /** * Whether a rotation must go through guest HID: a simulator the `fold` fact admits that has more * than one integrated panel. XCUIDevice rotation has no effect there, and its readback repeats @@ -37,12 +60,47 @@ export async function isAppleFoldableSimulator( if (!appleFoldableFacts(device).setFoldPose.available) return false; const known = multiPanelByDevice.get(device.id); if (known !== undefined) return known; + const read = ++panelTableReads; const inventory = await queryAppleDisplayInventory(device, { signal }); if (inventory.unresolved) return false; - multiPanelByDevice.set(device.id, inventory.multiScreen); + recordPanelTable(device, inventory, read); return inventory.multiScreen; } +export type FoldablePanelTableRead = + | { kind: 'multi-panel'; inventory: AppleDisplayInventory } + /** A device the `fold` fact refuses, one with one panel, or one whose panel count is unknown. */ + | { kind: 'single-panel' } + /** A device known to have more than one panel whose table did not answer this time. */ + | { kind: 'unreadable' }; + +/** + * One panel-table read for a surface request. It records the panel count it reads, so a + * single-panel simulator pays one read per daemon. + */ +export async function readFoldablePanelTable( + device: DeviceInfo, + options: { signal?: AbortSignal; timeoutMs: number }, +): Promise { + if (!appleFoldableFacts(device).setFoldPose.available) return { kind: 'single-panel' }; + const known = multiPanelByDevice.get(device.id); + if (known === false) return { kind: 'single-panel' }; + if (known === undefined && unansweredPanelTables.get(device.id)) return { kind: 'single-panel' }; + const read = ++panelTableReads; + const inventory = await queryAppleDisplayInventory(device, options); + if (inventory.unresolved) { + if (known) return { kind: 'unreadable' }; + const newerReadResolved = (latestResolvedRead.get(device.id) ?? 0) > read; + const fullBudget = options.timeoutMs >= IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS; + if (fullBudget && !options.signal?.aborted && !newerReadResolved) { + unansweredPanelTables.set(device.id, true); + } + return { kind: 'single-panel' }; + } + recordPanelTable(device, inventory, read); + return inventory.multiScreen ? { kind: 'multi-panel', inventory } : { kind: 'single-panel' }; +} + /** Rotates a foldable simulator through guest HID and verifies the rotation through CoreDevice. */ export async function setAppleFoldableOrientation( device: DeviceInfo, diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts new file mode 100644 index 0000000000..f1461b60bb --- /dev/null +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -0,0 +1,321 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test, vi } from 'vitest'; +import { bindTestCommandHost } from '@agent-device/test-kit'; +import { withAppleRunnerProvider, type AppleRunnerProvider } from '@agent-device/apple-runner'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { AppError } from '@agent-device/kernel/errors'; + +vi.mock('../foldable/fold-helper-cache.ts', () => ({ + ensureFoldHelperBinary: vi.fn(async () => ({ path: '/cache/fold-helper' })), +})); +vi.mock('../core/devicectl.ts', async (importOriginal) => ({ + ...(await importOriginal()), + runIosDevicectlJsonRequest: vi.fn(async () => ({ + ok: true as const, + payload: { result: { displays: [] } }, + })), +})); + +import { runIosDevicectlJsonRequest } from '../core/devicectl.ts'; +import { runAppleRunnerCommand } from '../core/runner-client.ts'; +import { iosRunnerOverrides } from '../interactions.ts'; +import { + recordingRunnerProvider, + type RecordedRunnerCall, +} from '../__tests__/recording-runner-provider.ts'; +import { isAppleFoldableSimulator } from './orientation.ts'; +import { resolveRunnerRequestLitPanel } from './runner-panel.ts'; +import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; +import { + IPHONE_17_SINGLE_PANEL, + IPHONE_DUO_CLOSED, + IPHONE_DUO_OPEN, +} from '../__tests__/display-fixtures.ts'; + +bindTestCommandHost(); + +const mockPanelTable = vi.mocked(runIosDevicectlJsonRequest); + +beforeEach(() => { + mockPanelTable.mockReset(); + mockPanelTable.mockResolvedValue({ ok: true as const, payload: { result: { displays: [] } } }); +}); + +/** + * A fresh simulator id per case: the panel count is cached per simulator for the life of the + * process, and that cache is one of the facts under test. + */ +let simulatorCount = 0; +function simulator(overrides: Partial = {}): DeviceInfo { + simulatorCount += 1; + return { ...IOS_SIMULATOR, id: `${IOS_SIMULATOR.id}-${simulatorCount}`, ...overrides }; +} + +function answering(payload: unknown): void { + mockPanelTable.mockImplementation(async () => ({ ok: true as const, payload })); +} + +test('a multi-panel request names the lit panel in the units the runner compares a window against', async () => { + answering(IPHONE_DUO_CLOSED); + // A closed Duo lights its outer panel: 1398x2034 px at pointScale 3, which is the 466x678 the + // runner finds as that panel's SpringBoard window. + assert.deepEqual(await resolveRunnerRequestLitPanel(simulator(), {}), { + displayId: 1, + widthPt: 466, + heightPt: 678, + }); +}); + +test('each request reads the panel table once, so a fold moves the surface', async () => { + const device = simulator(); + answering(IPHONE_DUO_CLOSED); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); + assert.equal(mockPanelTable.mock.calls.length, 1, 'the panel count comes from the same read'); + answering(IPHONE_DUO_OPEN); + assert.equal( + (await resolveRunnerRequestLitPanel(device, {}))?.displayId, + 3, + 'an opened device must be asked again, not answered from the panel that just went dark', + ); + assert.equal(mockPanelTable.mock.calls.length, 2); +}); + +test('a device with one panel sends no panel and is never asked again', async () => { + const device = simulator(); + answering(IPHONE_17_SINGLE_PANEL); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + const reads = mockPanelTable.mock.calls.length; + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(mockPanelTable.mock.calls.length, reads, 'one panel is hardware, probed once'); +}); + +test('a failed first panel read backs off temporarily and probes again after expiry', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000); + try { + const device = simulator(); + mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(mockPanelTable.mock.calls.length, 1); + now.mockReturnValue(6_001); + answering(IPHONE_DUO_OPEN); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 3); + assert.equal(mockPanelTable.mock.calls.length, 2); + } finally { + now.mockRestore(); + } +}); + +test('a cancelled panel-table read leaves the next request free to ask', async () => { + const device = simulator(); + mockPanelTable.mockRejectedValueOnce(new Error('devicectl aborted by the caller')); + const cancelled = new AbortController(); + cancelled.abort(); + assert.equal(await resolveRunnerRequestLitPanel(device, { signal: cancelled.signal }), undefined); + answering(IPHONE_DUO_CLOSED); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); +}); + +test('a multi-panel device whose panel power decides nothing sends no panel', async () => { + const device = simulator(); + // Both panels lit: panel power settles no single panel, so the runner keeps its historic route. + answering({ + result: { + displays: IPHONE_DUO_CLOSED.result.displays.map((display) => ({ + ...display, + active: true, + backlightState: 'activeOn', + })), + }, + }); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + answering(IPHONE_DUO_OPEN); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 3); + assert.equal( + mockPanelTable.mock.calls.length, + 2, + 'ambiguous power does not cache a single-panel verdict', + ); +}); + +test('a multi-panel device whose panel table is incomplete sends no panel', async () => { + const device = simulator(); + // The parser read one of two integrated panels, so the unread one may be the lit one. + const [outer, inner] = IPHONE_DUO_CLOSED.result.displays; + answering({ + result: { + displays: [outer, { ...inner, nativeSize: undefined }], + }, + }); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + answering(IPHONE_DUO_OPEN); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 3); + assert.equal(mockPanelTable.mock.calls.length, 2, 'an incomplete table remains multi-panel'); +}); + +test('a device the fold fact refuses never costs a panel table read', async () => { + // A physical iPhone is folded by hand (ADR 0025) and no physical foldable leaf is in the + // supported matrix, so its runner keeps the surface route the request always sent. + assert.equal(await resolveRunnerRequestLitPanel(IOS_DEVICE, {}), undefined); + assert.equal(mockPanelTable.mock.calls.length, 0); +}); + +test('a resolved read lifts the back-off a failed one opened', async () => { + const device = simulator(); + mockPanelTable.mockRejectedValueOnce(new Error('devicectl does not report displays')); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + answering(IPHONE_DUO_CLOSED); + // A rotation reads the table on its own route; its answer clears the back-off for surface reads. + assert.equal(await isAppleFoldableSimulator(device, undefined), true); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); +}); + +test('the probe takes the request timeout when it is the shorter one', async () => { + answering(IPHONE_DUO_CLOSED); + await resolveRunnerRequestLitPanel(simulator(), { requestTimeoutMs: 1_200 }); + assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 1_200); + await resolveRunnerRequestLitPanel(simulator(), { requestTimeoutMs: 60_000 }); + assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 5_000); +}); + +test('a surface command probes the panel table within its own timeout', async () => { + mockPanelTable.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { ok: true as const, payload: IPHONE_DUO_CLOSED }; + }); + const device = simulator({ name: 'iPhone Duo' }); + const calls: RecordedRunnerCall[] = []; + await withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20, timeoutMs: 1_200 }), + ); + assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 1_200); + assert.equal(calls[0]?.command.litPanel?.displayId, 1); + const sentTimeoutMs = calls[0]?.command.timeoutMs ?? Number.NaN; + assert.ok(sentTimeoutMs > 0 && sentTimeoutMs < 1_200, 'the runner gets what the read left'); +}); + +test('a panel read that spends the whole request budget refuses instead of sending', async () => { + mockPanelTable.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { ok: true as const, payload: IPHONE_DUO_CLOSED }; + }); + const device = simulator({ name: 'iPhone Duo' }); + const calls: RecordedRunnerCall[] = []; + await assert.rejects( + withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20, timeoutMs: 20 }), + ), + (error: AppError) => { + assert.equal(error.details?.reason, 'panel-probe-budget-exhausted'); + assert.equal(error.details?.runnerErrorCode, undefined); + assert.equal(error.details?.dispatched, 'no'); + return true; + }, + ); + assert.equal(calls.length, 0, 'a request with no time left never reaches the runner'); +}); + +test('a known multi-panel device refuses a surface command when its panel table goes unread', async () => { + const device = simulator({ name: 'iPhone Duo' }); + const calls: RecordedRunnerCall[] = []; + const run = () => + withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20 }), + ); + answering(IPHONE_DUO_CLOSED); + await run(); + assert.equal(calls[0]?.command.litPanel?.displayId, 1); + + mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); + await assert.rejects(run(), (error: AppError) => { + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'panel-table-unreadable'); + assert.equal(error.details?.runnerErrorCode, 'APP_SCREEN_UNRESOLVED'); + assert.equal(error.details?.dispatched, 'no'); + return true; + }); + assert.equal(calls.length, 1, 'the refused command never reached the runner'); +}); + +test('a slow failed read does not back off over a newer read that resolved', async () => { + const device = simulator(); + let failSlowRead!: (error: Error) => void; + mockPanelTable.mockImplementationOnce( + () => new Promise((_resolve, reject) => (failSlowRead = reject)), + ); + const slow = resolveRunnerRequestLitPanel(device, {}); + answering(IPHONE_DUO_CLOSED); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); + failSlowRead(new Error('devicectl does not report displays')); + assert.equal(await slow, undefined); + assert.equal( + (await resolveRunnerRequestLitPanel(device, {}))?.displayId, + 1, + 'the stale failure must not hold the next request on the historic route', + ); +}); + +test('every press-series chunk uses the lit-panel naming path', async () => { + answering(IPHONE_DUO_OPEN); + const device = simulator(); + const calls: RecordedRunnerCall[] = []; + const base = recordingRunnerProvider(calls); + const provider: AppleRunnerProvider = { + ...base, + runCommand: async (target, command, options) => { + await base.runCommand(target, command, options); + return { + completedSteps: command.steps?.length ?? 0, + sequenceResults: command.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], + }; + }, + }; + // A long interval forces separate sequence chunks; each dispatch must name its current panel. + mockPanelTable.mockResolvedValueOnce({ ok: true, payload: IPHONE_DUO_OPEN }); + mockPanelTable.mockResolvedValueOnce({ ok: true, payload: IPHONE_DUO_CLOSED }); + await withAppleRunnerProvider(provider, { deviceId: device.id }, async () => { + const { overrides } = iosRunnerOverrides( + device, + { appBundleId: 'com.example.app' }, + process.env, + ); + await overrides.pressPoint!( + { x: 100, y: 200 }, + { + button: 'primary', + count: 4, + intervalMs: 10_000, + holdMs: 0, + jitterPx: 0, + doubleTap: false, + }, + ); + }); + assert.ok(calls.length > 1, 'the regression must exercise more than one chunk'); + assert.ok(calls.every(({ command }) => command.command === 'sequence' && command.litPanel)); + assert.equal(calls[0]?.command.litPanel?.displayId, 3); + assert.equal(calls[1]?.command.litPanel?.displayId, 1); + assert.equal(mockPanelTable.mock.calls.length, calls.length); +}); + +test('an unnamed panel keeps the historic wire timeout and caller signal', async () => { + let time = 1_000; + const clock = vi.spyOn(Date, 'now').mockImplementation(() => time); + try { + mockPanelTable.mockImplementation(async () => { + time += 50; + return { ok: true, payload: IPHONE_17_SINGLE_PANEL }; + }); + const device = simulator(); + const calls: RecordedRunnerCall[] = []; + const signal = new AbortController().signal; + await withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20, timeoutMs: 1_200 }, { signal }), + ); + assert.equal(calls[0]?.command.timeoutMs, 1_200); + assert.equal(calls[0]?.command.litPanel, undefined); + assert.equal(calls[0]?.options.signal, signal); + } finally { + clock.mockRestore(); + } +}); diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts new file mode 100644 index 0000000000..0e8010172d --- /dev/null +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -0,0 +1,79 @@ +import type { RunnerCommand } from '@agent-device/apple-runner'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; + +import { IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS } from '../core/config.ts'; +import { appleLitPanel } from '../core/display-inventory.ts'; +import { readFoldablePanelTable } from './orientation.ts'; + +const PANEL_TABLE_UNREADABLE_HINT = + 'The panel table is unreadable, so no lit panel can be named and nothing was sent. Retry; if it persists, run xcrun devicectl device info displays --device .'; + +/** Refuses a request whose lit panel cannot be named; nothing was sent. */ +function panelTableRefusal(device: DeviceInfo, message: string): AppError { + return new AppError('COMMAND_FAILED', message, { + deviceId: device.id, + reason: 'panel-table-unreadable', + runnerErrorCode: 'APP_SCREEN_UNRESOLVED', + dispatched: 'no', + hint: PANEL_TABLE_UNREADABLE_HINT, + }); +} + +/** + * The lit panel a runner request names, read fresh per request because a fold changes it (ADR + * 0025). The read never outlasts the request's `timeoutMs`. Undefined keeps the historic + * first-window route, for a device not known to have more than one panel. + */ +export async function resolveRunnerRequestLitPanel( + device: DeviceInfo, + options: { signal?: AbortSignal; requestTimeoutMs?: number }, +): Promise { + const read = await readFoldablePanelTable(device, { + signal: options.signal, + timeoutMs: Math.min( + options.requestTimeoutMs ?? IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, + IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, + ), + }); + if (read.kind === 'unreadable') { + throw panelTableRefusal( + device, + `The panel table of ${device.name} did not answer, so the lit panel cannot be named`, + ); + } + return read.kind === 'multi-panel' ? appleLitPanel(read.inventory) : undefined; +} + +/** + * `command` with the lit panel named and the request budget the read left, measured on one clock. + * A read that leaves no budget is refused before anything is sent. + * Only a named panel charges probe time to the wire timeout; unnamed-panel requests keep the historic timeout and share the caller's AbortSignal. + */ +export async function nameRunnerRequestLitPanel( + device: DeviceInfo, + command: RunnerCommand, + signal: AbortSignal | undefined, +): Promise { + const startedAt = Date.now(); + const litPanel = await resolveRunnerRequestLitPanel(device, { + signal, + requestTimeoutMs: command.timeoutMs, + }); + if (!litPanel) return command; + if (command.timeoutMs === undefined) return { ...command, litPanel }; + const timeoutMs = command.timeoutMs - (Date.now() - startedAt); + if (timeoutMs < 1) { + throw new AppError( + 'COMMAND_FAILED', + `Reading the panel table of ${device.name} spent the request's ${command.timeoutMs} ms, so nothing was sent`, + { + deviceId: device.id, + reason: 'panel-probe-budget-exhausted', + dispatched: 'no', + hint: 'Increase the request timeout to leave time for the command after reading the panel table.', + }, + ); + } + return { ...command, litPanel, timeoutMs }; +}